diff --git a/.github/ci-suites.json b/.github/ci-suites.json index 8e18cf9fe18..a9bc643c275 100644 --- a/.github/ci-suites.json +++ b/.github/ci-suites.json @@ -16,6 +16,10 @@ "group": "desktop-inbox", "task": "ci:e2e-desktop-inbox" }, + { + "group": "browser-extension", + "task": "ci:e2e-browser-extension" + }, { "group": "seo", "task": "ci:e2e-seo" diff --git a/.github/scripts/ci-plan.gql b/.github/scripts/ci-plan.gql index db015cb2441..b2a0e6c184f 100644 --- a/.github/scripts/ci-plan.gql +++ b/.github/scripts/ci-plan.gql @@ -3,7 +3,7 @@ query($base: String!, $head: String!) { affectedTasks(base: $base, head: $head, tasks: [ "test", "build", "type-check", "ci:integration", "ci:desktop-live", "ci:e2e-scim", "ci:e2e-cli", "ci:e2e-stop-after", "ci:e2e-desktop-inbox", - "ci:e2e-project-files", "ci:e2e-mobile", "ci:e2e-seo" + "ci:e2e-browser-extension", "ci:e2e-project-files", "ci:e2e-mobile", "ci:e2e-seo" ]) { items { name diff --git a/.github/scripts/http-e2e.sh b/.github/scripts/http-e2e.sh index 978b2f0488c..7eaf06c1774 100755 --- a/.github/scripts/http-e2e.sh +++ b/.github/scripts/http-e2e.sh @@ -3,7 +3,7 @@ # CI groups, which reuse the production build from `e2e-build`. # The job extracts it into apps/sim/.next before running these suites. # -# Usage: http-e2e.sh (run from apps/sim) +# Usage: http-e2e.sh (run from apps/sim) # # The job provides DATABASE_URL, BETTER_AUTH_SECRET and ENCRYPTION_KEY; each group sets the rest of # its app's environment here. Reports and server logs land in $RUNNER_TEMP/e2e. @@ -20,7 +20,7 @@ # telemetry flush runs detached and still writes .next/dev). set -euo pipefail -group=${1:?usage: http-e2e.sh } +group=${1:?usage: http-e2e.sh } report_dir="$RUNNER_TEMP/e2e" ready_timeout_seconds=600 mkdir -p "$report_dir" @@ -198,7 +198,7 @@ case "$group" in ;; # The desktop background executor's protocol: device registration, the SSE doorbell over Redis - # pub/sub, presence, leased claims, Stop and isolation. The only group whose app gets Redis. + # pub/sub, presence, leased claims, Stop and isolation. desktop-inbox) export REDIS_URL=redis://127.0.0.1:6379 export NEXT_PUBLIC_FORCE_HOSTED=false @@ -227,6 +227,31 @@ case "$group" in bun --no-env-file scripts/test-organization-search-e2e.ts ;; + browser-extension) + export REDIS_URL=redis://127.0.0.1:6379 + export NEXT_PUBLIC_FORCE_HOSTED=false + export KNOWLEDGE_MEMBER_ACCESS=true CREDENTIAL_GROUPS=true ACCESS_CONTROL_ENABLED=true + export NEXT_PUBLIC_ACCESS_CONTROL_ENABLED=true + export API_ENCRYPTION_KEY="$ENCRYPTION_KEY" + bunx --no-install playwright install --with-deps chromium + extension_identity=$(bun - <<'JS' + import { generateKeyPairSync, createHash } from 'node:crypto' + const { publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) + const der = publicKey.export({ type: 'spki', format: 'der' }) + const id = [...createHash('sha256').update(der).digest('hex').slice(0, 32)] + .map(c => String.fromCharCode(97 + parseInt(c, 16))).join('') + process.stdout.write(`${der.toString('base64')}\n${id}`) +JS + ) + export SIM_EXTENSION_KEY="${extension_identity%%$'\n'*}" + export BROWSER_EXTENSION_IDS="${extension_identity#*$'\n'}" + start_app browser-extension 3007 'browser extension' record-http-status + EXTENSION_APP_E2E_BASE_URL="$local_app_url" \ + EXTENSION_APP_E2E_DATABASE_URL="$DATABASE_URL" \ + EXTENSION_APP_E2E_REPORT_PATH="$report_dir/browser-extension/report.json" \ + bun run test:browser-extension:e2e + ;; + project-files) PROJECT_FILES_E2E_ADMIN_DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \ PROJECT_FILES_E2E_REDIS_URL=redis://127.0.0.1:6379 \ diff --git a/.github/workflows/browser-extension.yml b/.github/workflows/browser-extension.yml new file mode 100644 index 00000000000..5ce53766683 --- /dev/null +++ b/.github/workflows/browser-extension.yml @@ -0,0 +1,58 @@ +name: browser-extension + +on: + pull_request: + paths: + - '.github/workflows/browser-extension.yml' + - 'apps/extension/**' + - 'packages/browser-extension-protocol/**' + - 'apps/sim/app/extension/**' + - 'apps/sim/app/workspace/**/home/**' + - 'apps/sim/stores/mothership-drafts/**' + - 'apps/sim/scripts/test-browser-extension-e2e.ts' + - 'apps/sim/lib/core/security/**' + - 'apps/sim/lib/mothership/chat/**' + - 'apps/sim/lib/api/contracts/mothership-chat-context.ts' + - 'apps/sim/proxy.ts' + - 'apps/sim/next.config.ts' + - 'apps/sim/package.json' + - 'packages/emcn/**' + - 'packages/logger/**' + - 'packages/utils/**' + - 'bun.lock' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: browser-extension-${{ github.ref }} + cancel-in-progress: true + +jobs: + transport: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 + with: + node-version: '24' + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.4 + - run: bun install --frozen-lockfile + - run: bunx playwright install --with-deps chromium + working-directory: apps/extension + - run: bun run type-check && bun run build && bun run test:e2e + working-directory: apps/extension + env: + EXTENSION_REPORT_PATH: test-results/extension-report.json + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: browser-extension-results + path: apps/extension/test-results + retention-days: 7 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7abb0de6951..e75db8a95c1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -217,9 +217,9 @@ jobs: --health-interval 5s --health-timeout 5s --health-retries 10 - # Desktop inbox and Project Files use Redis. + # Desktop inbox, browser extension, and Project Files use Redis. redis: - image: ${{ (matrix.group == 'desktop-inbox' || matrix.group == 'project-files') && 'redis:8.2-alpine' || '' }} + image: ${{ (matrix.group == 'desktop-inbox' || matrix.group == 'browser-extension' || matrix.group == 'project-files') && 'redis:8.2-alpine' || '' }} ports: - 6379:6379 options: >- @@ -296,7 +296,9 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-reports-${{ matrix.group }} - path: ${{ runner.temp }}/e2e/**/*.json + path: | + ${{ runner.temp }}/e2e/**/*.json + ${{ runner.temp }}/e2e/browser-extension/**/*.png if-no-files-found: ignore retention-days: 7 diff --git a/apps/extension/DESIGN.md b/apps/extension/DESIGN.md new file mode 100644 index 00000000000..7f81bb04311 --- /dev/null +++ b/apps/extension/DESIGN.md @@ -0,0 +1,159 @@ +# Sim browser extension + +Implementation scope and research, 2026-10-29. + +## Product contract + +The extension is another surface of Sim. A native Chrome side panel hosts the actual +Sim composer and transcript, using existing authenticated Mothership requests, +conversation persistence, streaming recovery, tool permissions, and organization +or workspace authorization. The panel keeps Fast mode and omits the model selector. +A conversation can continue in the full app. + +The user signs in in a normal Sim tab and lands in their organization chat. Choosing +a workspace is optional; members without Build access use Assistant. They can open +or create a conversation and explicitly attach the current page or highlighted text. +The composer’s + menu offers Current page and Selected text. Pending snapshots sit +inside the composer as compact removable chips alongside file attachments. Opening one +shows the captured text, source, capture time, and conversation-retention disclosure. +Partial captures remain labeled in the attachment row. Site access lives in the +composer’s Site access action, which opens an extension-owned dialog; successful delivery adds no persistent status banner. +Snapshots are immutable: navigating to another page cannot silently change an attached +source. Removing an attachment, changing organization or workspace, or starting a new conversation +clears pending page context. Several explicitly attached pages can support comparison. +Assistant accepts page snapshots while retaining its existing restriction on +workspace-resource contexts. + +The extension does not promise complete capture of virtualized documents, canvas +editors, restricted browser pages, inaccessible cross-origin frames, or PDFs. +Extraction failure is visible; the model must never infer absent page content. +Browser interaction, autonomous navigation, screenshot OCR, and native-app control +are separate capabilities requiring their own permission and execution contracts. +Existing Sim server tools remain subject to their existing authorization and approval. + +## Ownership and data flow + +1. `apps/extension`: packaged MV3 service worker, side-panel host, bounded content + extraction, per-site permission controls, build artifacts, browser verification. +2. `packages/browser-extension-protocol`: versioned, strictly validated messages and + immutable page snapshots shared by the extension, app, and server. +3. `apps/sim/app/extension`: session gate, organization landing, optional workspace picker, app providers, + real chat components, history and a narrow parent-window bridge. +4. Existing chat context pipeline: `browser_page` input validation, durable message + projection, restored context chips, untrusted-content formatting, Assistant admission. +5. Existing Sim API and Mothership: authentication, authorization, billing, tool + execution and streaming; no alternate inference service or extension API key. + +The content script returns text to its extension caller. The extension verifies the +captured document still belongs to the intended tab, then sends a typed snapshot to +the exact hosted frame. The frame validates its parent origin and identity, and puts +the snapshot into pending UI context. The existing send path persists it with the +message and passes quoted content to Mothership. A page snapshot grants no access to +other workspace resources or browser-control tools. + +## Permission and lifetime rules + +- Required browser capabilities: side panel, scripting, active tab, context menus, + and short-lived session storage for selection handoff. Required host access is + limited to the configured Sim origin. +- Other HTTP(S) origins are optional grants requested for the current site from a + user gesture. Each grant can be revoked. Keeping the panel open is not a grant + to read every subsequently selected tab. +- No history, bookmarks, cookie API, debugger, network interception, native + messaging, external messaging, or arbitrary remote code execution. +- No always-running page reader. Extraction is tied to an explicit attachment action. +- Page bytes are not written to extension local/sync storage. Temporary context-menu + handoff uses trusted-context session storage, expiration and navigation cleanup. +- Sent snapshots become chat data and follow Sim's chat retention and deletion + policy. This must be disclosed in the interface and store listing. No claim of + zero retention is made once a message is sent. +- No page content or full page URL in extension telemetry or routine logs. + +## Boundary controls + +- Package all privileged JavaScript. The hosted iframe has no extension API access. +- Only exact configured Sim origins and exact configured extension IDs are trusted. + Both directions validate source window, origin, protocol version and message shape. + There is no arbitrary fetch, JavaScript, navigation or extension-API command bridge. +- `BROWSER_EXTENSION_IDS` is a bounded comma-separated list of Chrome IDs. Missing + or malformed configuration disables embedding. Only `/extension` and its children + receive the corresponding frame policy; ordinary app pages keep their frame denial. +- Extension pages use a restrictive CSP; the hosted app keeps its app CSP with only + the embedding ancestor list changed. Sign-in happens in a top-level tab. +- Allow HTTPS production app origins; HTTP is development-only on loopback. +- Capture uses bounded traversal and output size, skips hidden and editable/form + content, and records truncation. It never serializes the entire DOM or reads + cookies, browser storage, input values, or page-world JavaScript state. +- Validate snapshots again server-side, including headless execution paths. Bound + individual and aggregate text/byte sizes. Only HTTP(S) source URLs without embedded + credentials are accepted; URL secrets should not be carried as source metadata. +- Snapshot content is explicitly untrusted quoted JSON. This is a model cue, not + an authorization mechanism: tool access remains independently enforced by Sim. + +## Verification planned before implementation + +The credible failure modes are cross-origin/frame message forgery; schema-version +confusion; hostile URL metadata; hidden/form data leakage; unbounded DOM traversal; +selection or extraction delivered after navigation; stale context on workspace +switch; persistence mutating the captured text; Assistant silently dropping context; +and an embed exception accidentally allowing unrelated app pages to be framed. + +Use a real Chromium extension against controlled HTTP pages for extraction, +navigation, permission and message-boundary checks. Emit a caller-configured JSON +report and screenshot/trace artifacts. Security-boundary tests may cover header +isolation and durable-context semantics where the running app needs unavailable +services. Do not replace real Chrome APIs with mocks and describe that as E2E. + +The authenticated live-app acceptance gate covers organization chat, Assistant, and +permitted workspaces: capture a real page, receive a grounded Mothership answer, +reopen the conversation, stop/reconnect streaming, and switch scopes. Verify revoked +access, live policy changes, retained drafts, and late history reads after completion. +An absent backend or account is an unverified gate, not a pass. + +## Research and decisions + +- [Glean sidebar](https://docs.glean.com/user-guide/apps/extension-sidebar) and + [page permissions](https://docs.glean.com/user-guide/apps/why-does-the-chrome-extension-ask-for-those-permissions): + page context can use address and text without indexing the page. Adopt explicit + page attachment alongside the existing Sim knowledge and tool experience. +- [Chrome activeTab](https://developer.chrome.com/docs/extensions/develop/concepts/activeTab) + and [optional permissions](https://developer.chrome.com/docs/extensions/reference/api/permissions): + temporary access follows user invocation; cross-origin navigation requires fresh + authority. Request persistent access per site only when selected by the user. +- [Chrome extension security](https://developer.chrome.com/docs/extensions/develop/security-privacy/stay-secure) + and [OWASP extension guidance](https://cheatsheetseries.owasp.org/cheatsheets/Browser_Extension_Vulnerabilities_Cheat_Sheet.html): + minimize privileges and treat all incoming page messages as attacker-controlled. + Do not expose a generic privileged bridge. +- [Service worker lifecycle](https://developer.chrome.com/docs/extensions/develop/concepts/service-workers/lifecycle): + workers are disposable. Stream from the app document; bound temporary handoffs + independently of worker globals. +- [Storage and cookies](https://developer.chrome.com/docs/extensions/develop/concepts/storage-and-cookies): + extension frames have special partition and cookie behavior. Reuse Sim sessions, + verify enterprise policy and blocked-cookie cases, and show a top-level sign-in + recovery path instead of copying credentials into extension storage. +- [MV3 remote-code policy](https://developer.chrome.com/docs/webstore/program-policies/mv3-requirements): + isolated remote frames are permitted subject to review and user-data rules. All + privileged capture and message logic remains packaged and inspectable. +- [User-data disclosures](https://developer.chrome.com/docs/webstore/program-policies/user-data-faq): + local processing of website content still requires disclosure. Publish actual + data handling and permission justifications with the store submission. + +## Release scope and remaining expansion + +Planning estimate for one engineer familiar with Sim: 5–8 engineering days for +the core surface, context pipeline and security boundaries; 2–4 more days for +authenticated deployment QA, enterprise cookie-policy checks and release packaging. +These are estimates for production readiness, not a promise that a working local +build has completed those gates. Chrome Web Store review is an external timeline. +Richer document adapters and screenshot capture are roughly another 1–2 weeks; +autonomous browser actions need a separate design and estimate. + +The first implementation targets Chrome MV3 and a configured Sim deployment. It +includes the shared chat UI and engine, explicit page/selection context, Assistant +context support, organization/workspace/history navigation, permission controls and verification. +It does not constitute store publication or proof of every third-party site's DOM. + +Follow-on work: screenshot/image attachment through existing authorized upload paths; +document-specific adapters with measured coverage; Firefox/Safari shells; managed +enterprise distribution; and separately authorized browser-action tools. Each must +retain the same source, identity, consent and lifecycle boundaries. diff --git a/apps/extension/README.md b/apps/extension/README.md new file mode 100644 index 00000000000..5b3763cda59 --- /dev/null +++ b/apps/extension/README.md @@ -0,0 +1,67 @@ +# Sim browser extension + +The Chrome extension attaches an explicitly captured page or selection to the existing Sim chat. The native side panel hosts the authenticated `/extension` app surface. Chat opens in the user's organization when available; choosing a workspace is optional. Members without access to Home use the organization's Assistant. Chat rendering, execution, permissions, billing, and history belong to Sim and Mothership. + +## Build and load + +From the repository root: + +```sh +bun install --frozen-lockfile +bun run --cwd apps/extension build +``` + +In Chrome, open `chrome://extensions`, enable Developer mode, choose **Load unpacked**, and select `apps/extension/dist`. Copy the extension ID into the Sim server's `BROWSER_EXTENSION_IDS` configuration. It accepts comma-separated Chrome extension IDs. Restart the app after changing that configuration. Unconfigured IDs cannot embed the app. + +Click the Sim toolbar icon to open chat and attach the current page. **+ → Current page** adds a fresh snapshot; **Selected text** attaches the highlighted passage. Attachments appear as chips inside the composer. Attach up to three pages or selections to summarize, extract facts, or compare sources. To quote a passage, select it and choose **Ask Sim about selection** from the context menu. Review or remove the attachment before sending. **+ → Site access** opens the extension-owned permission dialog: **Allow access to [site]** grants access only to that site; **Remove access to [site]** revokes it. Persistent site permission never enables background capture. + +For local development: + +```sh +SIM_EXTENSION_DEV=1 SIM_EXTENSION_ORIGIN=http://localhost:3000 bun run --cwd apps/extension build +``` + +`SIM_EXTENSION_ORIGIN` is baked into the package. Production builds require an HTTPS origin. Explicit development builds may use HTTP on loopback. `SIM_EXTENSION_KEY` optionally supplies the public manifest key for a stable extension ID; never supply a private signing key. Chrome Web Store signing and publisher credentials stay outside this repository. + +## Data and permissions + +- Required capabilities: `activeTab`, `scripting`, `sidePanel`, `storage`, `contextMenus`, and access to the configured Sim origin. +- Optional HTTP/HTTPS site access is requested only by a menu action naming that site. Chrome's `activeTab` access is temporary and does not follow arbitrary new sites. +- Page capture runs on demand in Chrome's isolated world. It excludes editable fields, inputs, hidden elements, scripts, and embedded documents. It reads only the top frame. PDF viewers and canvas documents may have no accessible text. +- Extraction stops at 50,000 text characters, 150,000 serialized UTF-8 bytes, 15,000 visited nodes, or 150ms. The composer exposes truncation. +- Source URLs omit credentials, query parameters, and fragments. Human-readable text can still contain sensitive information; it is the user-selected material they review and send into their conversation. +- Snapshots stay in memory until handed to Sim and acknowledged or expired after 30 seconds. Extension storage contains only a short-lived intent with a tab/document identity and a SHA-256 URL digest, never raw URLs, captured text, or credentials. Closing a tab or navigating invalidates its pending intent. Sim applies its normal conversation retention once a message is sent. +- No browser history, bookmarks, cookie access, debugger access, remote extension JavaScript, arbitrary network proxy, or background content collection is requested. Incognito use is disabled. +- The iframe bridge checks both the exact configured origin and the exact frame, then validates versioned messages. The hosted composer may request only page capture, selection capture, or the native site-access dialog. Requests require the current conversation nonce and Chrome transient user activation; arbitrary URLs, tabs, script execution, and network operations are never accepted. Permission grants still require a click in the extension-owned dialog. +- Captures and queued toolbar intents are scoped to the receiving conversation. Changing conversation, workspace, or account discards an unfinished handoff. Captured attachments remain explicit snapshots when the source tab changes. +- The complete hosted document, including portaled source dialogs and menus, is blocked from session replay. Clipboard writing is delegated only to the configured Sim origin. Clipboard reading and microphone capture are not delegated; voice input remains available in the main app. + +## Verification + +```sh +bun run --cwd apps/extension type-check +EXTENSION_REPORT_PATH=test-results/extension-report.json bun run --cwd apps/extension test:e2e +bun run --cwd apps/extension build +``` + +The E2E suite uses real Chromium, the packaged extension, and a disposable HTTP app fixture. It pregrants only the fixture page origin in a temporary package to exercise extraction without automating the browser's permission dialog. It checks HttpOnly session-cookie transport, origin/frame spoofing, sensitive-field exclusion, source URL minimization, selected excerpts, UTF-8 limits, stale-document handoffs, denied host access, and absence of page bytes in extension storage. The cookie check uses a fixture, not Better Auth or SSO. JSON results and a screenshot are written beside the caller-supplied report path. The E2E suite builds into a temporary directory and leaves shipping `dist` untouched. + +Release verification must additionally exercise the native toolbar and shortcut, Chrome's real grant/revoke prompt, sign-in/SSO and expiry against the running Sim app, side-panel closing/reopening, workspace changes, narrow widths, and the real Mothership streaming response. The fixture suite does not establish those integrations. The design document records launch scope and release criteria. + +The authenticated acceptance suite runs against a real local app, Better Auth, and a disposable Postgres database whose name contains `test`. Enable `KNOWLEDGE_MEMBER_ACCESS`, `CREDENTIAL_GROUPS`, `ACCESS_CONTROL_ENABLED`, and `NEXT_PUBLIC_ACCESS_CONTROL_ENABLED` on that app to exercise restricted organization members: + +```sh +EXTENSION_APP_E2E_BASE_URL=http://127.0.0.1:3007 \ +EXTENSION_APP_E2E_DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/extension_test \ +EXTENSION_APP_E2E_REPORT_PATH=../extension/test-results/real-app-report.json \ +SIM_EXTENSION_KEY="$PUBLIC_EXTENSION_KEY" \ +bun run --cwd apps/sim test:browser-extension:e2e +``` + +Configure the matching extension ID in the running app. The suite builds into a temporary directory, creates its own accounts/workspaces, and removes them afterward; it leaves the shipping `dist` untouched. It verifies real authentication, workspace denial, page limits, navigation, source links, clipboard writes, actual replay-recorder exclusion, themes, widths, and sign-out. CI provisions this isolated environment in the shared HTTP acceptance matrix, using the same production build as the other app suites. The packaged extension transport suite runs separately. Set `EXTENSION_APP_E2E_LIVE_CHAT=1` when the app is connected to a working local Mothership instance to additionally verify live rendering, saved page context, queue editing, and drafts through chat admission. Provider credentials belong only in that server's environment. Reports and screenshots are written beside the supplied report path. Native toolbar/shortcut/context-menu and permission-prompt checks remain manual release checks. + +## Distribution + +Publish only the production contents of `dist`, after rebuilding against the deployed Sim origin and configuring the issued store extension ID on that server. The store listing and privacy disclosure must describe that page text is sent to Sim when attached and used in a conversation, and link to the applicable Sim privacy policy. Keep the publisher account protected with hardware-backed MFA and limited release access. + +Research: [Chrome extension security](https://developer.chrome.com/docs/extensions/develop/security-privacy/stay-secure), [activeTab](https://developer.chrome.com/docs/extensions/develop/concepts/activeTab), [storage and cookies](https://developer.chrome.com/docs/extensions/develop/concepts/storage-and-cookies), [Manifest V3 policies](https://developer.chrome.com/docs/webstore/program-policies/mv3-requirements), and [OWASP extension vulnerabilities](https://cheatsheetseries.owasp.org/cheatsheets/Browser_Extension_Vulnerabilities_Cheat_Sheet.html). diff --git a/apps/extension/package.json b/apps/extension/package.json new file mode 100644 index 00000000000..796a3a2d843 --- /dev/null +++ b/apps/extension/package.json @@ -0,0 +1,37 @@ +{ + "name": "@sim/extension", + "version": "0.1.0", + "private": true, + "type": "module", + "license": "Apache-2.0", + "scripts": { + "build": "bun scripts/build.ts", + "type-check": "tsc --noEmit", + "lint": "biome check --write --unsafe .", + "lint:check": "biome check .", + "format": "biome format --write .", + "format:check": "biome format .", + "test:e2e": "bun scripts/test-e2e.ts" + }, + "dependencies": { + "@sim/browser-extension-protocol": "workspace:*", + "@sim/emcn": "workspace:*", + "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*", + "react": "19.2.4", + "react-dom": "19.2.4", + "zod": "4.3.6" + }, + "devDependencies": { + "@playwright/test": "1.61.1", + "@sim/tsconfig": "workspace:*", + "@types/bun": "1.4.3", + "@types/chrome": "0.3.6", + "@types/react": "^19", + "@types/react-dom": "^19", + "esbuild": "0.28.2", + "postcss": "^8.5.30", + "postcss-load-config": "6.0.1", + "typescript": "^7.0.2" + } +} diff --git a/apps/extension/scripts/build.ts b/apps/extension/scripts/build.ts new file mode 100644 index 00000000000..ebabf41ea13 --- /dev/null +++ b/apps/extension/scripts/build.ts @@ -0,0 +1,123 @@ +import { createPublicKey } from 'node:crypto' +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { build } from 'esbuild' +import postcss from 'postcss' +import loadPostcssConfig from 'postcss-load-config' + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const origin = new URL(process.env.SIM_EXTENSION_ORIGIN ?? 'https://www.sim.ai') +const development = process.env.SIM_EXTENSION_DEV === '1' +const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(origin.hostname) +if ( + origin.username || + origin.password || + origin.pathname !== '/' || + origin.search || + origin.hash || + (origin.protocol !== 'https:' && !(development && loopback && origin.protocol === 'http:')) +) { + throw new Error( + 'SIM_EXTENSION_ORIGIN must be an HTTPS origin; explicit development builds may use HTTP loopback.' + ) +} +const key = process.env.SIM_EXTENSION_KEY +if (key) { + try { + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(key) || key.length > 4_096) + throw new Error('Invalid encoding') + const parsed = createPublicKey({ key: Buffer.from(key, 'base64'), format: 'der', type: 'spki' }) + if ( + parsed.asymmetricKeyType !== 'rsa' || + parsed.export({ format: 'der', type: 'spki' }).toString('base64') !== key + ) + throw new Error('Invalid public key') + } catch { + throw new Error('SIM_EXTENSION_KEY must be a base64 RSA public key in SPKI format.') + } +} + +const outputArgument = process.argv.find((argument) => argument.startsWith('--outdir=')) +const outputDirectory = outputArgument?.slice('--outdir='.length) +if (outputArgument && !outputDirectory) throw new Error('--outdir requires a directory.') +const dist = resolve(root, outputDirectory ?? 'dist') +if (!outputArgument) await rm(dist, { recursive: true, force: true }) +await mkdir(dist, { recursive: true }) +await build({ + absWorkingDir: root, + entryPoints: { background: 'src/background.ts', sidepanel: 'src/sidepanel.tsx' }, + outdir: dist, + bundle: true, + platform: 'browser', + format: 'iife', + target: 'chrome120', + minify: !development, + sourcemap: development ? 'external' : false, + tsconfig: resolve(root, 'tsconfig.json'), + loader: { '.module.css': 'local-css', '.woff2': 'file' }, + define: { + __SIM_ORIGIN__: JSON.stringify(origin.origin), + 'process.env.NODE_ENV': '"production"', + 'process.env': '{}', + 'globalThis.process': 'undefined', + }, + plugins: [ + { + name: 'extension-tailwind', + setup(builder) { + builder.onResolve({ filter: /SeasonSansUprightsVF\.woff2$/ }, () => ({ + path: resolve(root, '../sim/app/_styles/fonts/season/SeasonSansUprightsVF.woff2'), + })) + builder.onLoad({ filter: /shell\.css$/ }, async ({ path }) => { + const config = await loadPostcssConfig({}, resolve(root, '../sim')) + const result = await postcss(config.plugins).process(await readFile(path, 'utf8'), { + from: path, + }) + return { contents: result.css, loader: 'css', resolveDir: dirname(path) } + }) + }, + }, + ], +}) +await cp(resolve(root, 'src/sidepanel.html'), resolve(dist, 'sidepanel.html')) +for (const [size, file] of [ + [16, 'favicon-16x16.png'], + [32, 'favicon-32x32.png'], + [128, 'android-chrome-512x512.png'], +] as const) { + await cp(resolve(root, '../sim/public/favicon', file), resolve(dist, `icon-${size}.png`)) +} +await writeFile( + resolve(dist, 'manifest.json'), + JSON.stringify( + { + manifest_version: 3, + name: development ? 'Sim (Development)' : 'Sim', + version: '0.1.0', + description: + 'Ask Sim about the page you are reading, with your existing Sim conversations and tools.', + minimum_chrome_version: '120', + permissions: ['activeTab', 'scripting', 'sidePanel', 'storage', 'contextMenus'], + host_permissions: [`${origin.origin}/*`], + optional_host_permissions: ['https://*/*', 'http://*/*'], + background: { service_worker: 'background.js' }, + side_panel: { default_path: 'sidepanel.html' }, + action: { default_title: 'Open Sim' }, + icons: { 16: 'icon-16.png', 32: 'icon-32.png', 128: 'icon-128.png' }, + commands: { + 'open-sim': { + suggested_key: { default: 'Alt+Shift+S', mac: 'Command+Shift+S' }, + description: 'Open Sim and attach the current page', + }, + }, + content_security_policy: { + extension_pages: `default-src 'none'; script-src 'self'; style-src 'self'; font-src 'self'; img-src 'self'; frame-src ${origin.origin}; connect-src 'none'; object-src 'none'; base-uri 'none'`, + }, + incognito: 'not_allowed', + ...(key ? { key } : {}), + }, + null, + 2 + ) +) diff --git a/apps/extension/scripts/test-e2e.ts b/apps/extension/scripts/test-e2e.ts new file mode 100644 index 00000000000..4bb45960f88 --- /dev/null +++ b/apps/extension/scripts/test-e2e.ts @@ -0,0 +1,661 @@ +import { spawnSync } from 'node:child_process' +import { generateKeyPairSync } from 'node:crypto' +import { existsSync } from 'node:fs' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { type BrowserContext, chromium, expect, type Frame, type Page } from '@playwright/test' +import { pageSnapshotSchema } from '@sim/browser-extension-protocol' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const logger = createLogger('BrowserExtensionE2E') +const reportPath = process.env.EXTENSION_REPORT_PATH +if (!reportPath) throw new Error('Set EXTENSION_REPORT_PATH to the output JSON report path.') +const output = resolve(root, reportPath) +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] +const fixture = Bun.serve({ + hostname: '127.0.0.1', + port: 0, + fetch(request) { + const url = new URL(request.url) + if (url.pathname === '/sign-in') + return new Response('Fixture session created', { + headers: { 'set-cookie': 'fixture-session=present; Path=/; HttpOnly; SameSite=Lax' }, + }) + if (url.pathname === '/extension') + return new Response( + ` +

Sim bridge fixture

Packaged extension transport verification

[]
+

${request.headers.get('cookie')?.includes('fixture-session=present') ? 'present' : 'missing'}

+ + `, + { headers: { 'content-type': 'text/html' } } + ) + return new Response( + `Private work page +

Customer escalation

The customer needs a replacement by Friday.

Reference: visible-reference-482

+ +
EDITABLE_SENTINEL
+
CSS_HIDDEN_SENTINEL
+ + `, + { headers: { 'content-type': 'text/html' } } + ) + }, +}) +const temporary = await mkdtemp(resolve(tmpdir(), 'sim-extension-e2e-')) +const extensionPath = resolve(temporary, 'extension') +let context: BrowserContext | undefined + +function assert(value: unknown, message: string): asserts value { + if (!value) throw new Error(message) +} +async function check(name: string, run: () => Promise) { + const started = performance.now() + logger.info(`Checking ${name}`) + try { + await run() + checks.push({ name, status: 'passed', durationMs: Math.round(performance.now() - started) }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: Math.round(performance.now() - started), + error: getErrorMessage(error), + }) + throw error + } +} + +try { + await check('The build rejects private key material before creating output', async () => { + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) + const rejectedOutput = resolve(temporary, 'rejected-key') + const result = spawnSync(process.execPath, ['scripts/build.ts', `--outdir=${rejectedOutput}`], { + cwd: root, + env: { + ...process.env, + SIM_EXTENSION_KEY: privateKey.export({ type: 'pkcs8', format: 'der' }).toString('base64'), + }, + encoding: 'utf8', + }) + assert(result.status !== 0, 'The build accepted private key material') + assert( + result.stderr.includes('RSA public key in SPKI format'), + 'The build failed for an unrelated reason' + ) + assert(!existsSync(rejectedOutput), 'Invalid key material reached build output') + }) + const origin = `http://127.0.0.1:${fixture.port}` + const pageOrigin = `http://localhost:${fixture.port}` + const built = spawnSync(process.execPath, ['scripts/build.ts', `--outdir=${extensionPath}`], { + cwd: root, + env: { ...process.env, SIM_EXTENSION_ORIGIN: origin, SIM_EXTENSION_DEV: '1' }, + encoding: 'utf8', + }) + assert(built.status === 0, `Extension build failed: ${built.stderr}`) + const manifest = JSON.parse(await readFile(resolve(extensionPath, 'manifest.json'), 'utf8')) + // The test package pregrants its disposable fixture site, equivalent to an accepted site prompt. + manifest.host_permissions.push(`${pageOrigin}/*`) + await writeFile(resolve(extensionPath, 'manifest.json'), JSON.stringify(manifest)) + context = await chromium.launchPersistentContext(resolve(temporary, 'profile'), { + channel: 'chromium', + headless: true, + args: [`--disable-extensions-except=${extensionPath}`, `--load-extension=${extensionPath}`], + }) + const worker = context.serviceWorkers()[0] ?? (await context.waitForEvent('serviceworker')) + context.setDefaultTimeout(15_000) + const extensionId = worker.url().split('/')[2] + const signIn = await context.newPage() + await signIn.goto(`${origin}/sign-in`) + await signIn.close() + const panel: Page = await context.newPage() + await panel.goto(`chrome-extension://${extensionId}/sidepanel.html`) + await panel.getByTitle('Sim chat').waitFor() + const frame: Frame = panel.frames()[1] ?? (await panel.waitForEvent('frameattached')) + const app = panel.frames().find((candidate) => candidate.url().startsWith(origin)) ?? frame + assert(app, 'Hosted frame did not load') + await app.waitForSelector('#received') + await check('A normal-tab HttpOnly session cookie reaches the hosted frame', async () => { + assert( + (await app.locator('#session').textContent()) === 'present', + 'Existing session cookie was not sent to the hosted app' + ) + }) + const page = await context.newPage() + await page.goto(`${pageOrigin}/ticket?access_token=URL_SECRET#secret`) + await page.bringToFront() + let receiverReady = false + const capture = async (name = 'Attach page') => { + if (!receiverReady) { + const intent = await pendingIntent() + await worker.evaluate(async (intent) => { + await chrome.storage.session.set({ [`captureIntent:${intent.windowId}:initial`]: intent }) + }, intent) + return + } + await app.evaluate( + ({ sessionId, action, origin }) => { + parent.postMessage( + { channel: 'sim-extension', version: 1, type: 'action', sessionId, action }, + origin + ) + }, + { + sessionId: receiverSessionId, + action: name === 'Selection' ? 'capture-selection' : 'capture-page', + origin: `chrome-extension://${extensionId}`, + } + ) + } + const snapshots = async () => + pageSnapshotSchema.array().parse(JSON.parse(await app.locator('#received').innerText())) + let receiverSessionId = generateId() + await app.evaluate((sessionId) => { + document.body.dataset.sessionId = sessionId + }, receiverSessionId) + const ready = () => { + receiverReady = true + return app.evaluate( + ({ targetOrigin, sessionId }) => { + document.body.dataset.sessionId = sessionId + parent.postMessage( + { channel: 'sim-extension', version: 1, type: 'ready', sessionId }, + targetOrigin + ) + }, + { targetOrigin: `chrome-extension://${extensionId}`, sessionId: receiverSessionId } + ) + } + const pendingIntent = () => + worker.evaluate(async () => { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }) + if (tab.id === undefined || !tab.url) throw new Error('Missing source tab') + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(tab.url)) + const urlDigest = Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, '0') + ).join('') + const [result] = await chrome.scripting.executeScript({ + target: { tabId: tab.id, frameIds: [0] }, + func: () => null, + }) + return { + tabId: tab.id, + windowId: tab.windowId, + kind: 'page', + createdAt: Date.now(), + documentId: result.documentId, + urlDigest, + } + }) + + const settleMessages = async () => { + const barrierId = generateId() + await panel.evaluate( + ({ origin, barrierId }) => { + const target = document.querySelector('iframe')?.contentWindow + if (!target) throw new Error('Missing hosted frame') + target.postMessage({ type: 'e2e-barrier', id: barrierId }, origin) + }, + { origin, barrierId } + ) + await app.waitForFunction((id) => document.body.dataset.barrierId === id, barrierId) + } + + await check('Capture stays local until the trusted hosted frame is ready', async () => { + await capture() + await panel.getByRole('status').filter({ hasText: 'Waiting for Sim' }).waitFor() + assert((await snapshots()).length === 0, 'Capture crossed before readiness') + await panel.evaluate( + (sessionId) => + new Promise((resolve) => { + window.addEventListener('message', function listener(event) { + if (event.source !== window) return + window.removeEventListener('message', listener) + resolve() + }) + window.postMessage( + { channel: 'sim-extension', version: 1, type: 'ready', sessionId }, + '*' + ) + }), + receiverSessionId + ) + await settleMessages() + assert((await snapshots()).length === 0, 'Wrong-origin readiness was accepted') + const wrongFrameDelivered = panel.evaluate( + () => + new Promise((resolve) => { + window.addEventListener('message', function listener(event) { + if ( + event.source === window || + event.source === document.querySelector('iframe')?.contentWindow + ) + return + window.removeEventListener('message', listener) + resolve() + }) + }) + ) + await app.evaluate((sessionId) => { + const child = document.createElement('iframe') + child.srcdoc = ` + + diff --git a/apps/extension/src/sidepanel.tsx b/apps/extension/src/sidepanel.tsx new file mode 100644 index 00000000000..2166938f23a --- /dev/null +++ b/apps/extension/src/sidepanel.tsx @@ -0,0 +1,301 @@ +import { useEffect, useRef, useState } from 'react' +import { + appToExtensionMessageSchema, + type ExtensionToAppMessage, + type PageSnapshot, +} from '@sim/browser-extension-protocol' +import { Chip, ChipModal, ChipModalBody, ChipModalHeader } from '@sim/emcn' +import { getErrorMessage } from '@sim/utils/errors' +import { createRoot } from 'react-dom/client' +import { captureActivePage } from '@/capture' +import { CAPTURE_INTENT_KEY, CAPTURE_INTENT_TTL_MS, SIM_ORIGIN } from '@/config' +import { captureIntentSchema } from '@/intent' +import '@/shell.css' + +interface SidePanelProps { + origin: string +} + +function SidePanel({ origin }: SidePanelProps) { + const frameRef = useRef(null) + const pendingRef = useRef(null) + const sessionRef = useRef(null) + const deliveredRef = useRef(null) + const captureGenerationRef = useRef(0) + const intentCutoffRef = useRef(0) + const busyRef = useRef(false) + const [accessOpen, setAccessOpen] = useState(false) + const [status, setStatus] = useState('') + const [site, setSite] = useState<{ origin: string; allowed: boolean } | null>(null) + + function deliver() { + const snapshot = pendingRef.current + const sessionId = sessionRef.current + if (!snapshot || !sessionId || deliveredRef.current === snapshot.id) return + if (Date.now() - Date.parse(snapshot.capturedAt) > CAPTURE_INTENT_TTL_MS) { + pendingRef.current = null + setStatus('The capture expired. Attach the page again.') + return + } + const message: ExtensionToAppMessage = { + channel: 'sim-extension', + version: 1, + type: 'context', + sessionId, + snapshot, + } + frameRef.current?.contentWindow?.postMessage(message, SIM_ORIGIN) + deliveredRef.current = snapshot.id + } + + function invalidateSession() { + intentCutoffRef.current = Date.now() + setAccessOpen(false) + if (pendingRef.current || busyRef.current) + setStatus('The conversation changed before delivery. Attach the page again.') + captureGenerationRef.current++ + pendingRef.current = null + deliveredRef.current = null + sessionRef.current = null + busyRef.current = false + } + + async function capture( + kind: 'page' | 'selection', + expected?: { tabId: number; documentId: string; urlDigest: string } + ) { + if (busyRef.current) return + if (pendingRef.current) { + setStatus('Waiting for Sim to receive the previous capture. Try again after it appears.') + return + } + busyRef.current = true + const generation = ++captureGenerationRef.current + setStatus('Reading the page…') + try { + const snapshot = await captureActivePage(kind, expected) + if (generation !== captureGenerationRef.current) return + pendingRef.current = snapshot + setStatus('Waiting for Sim to receive the attachment…') + deliver() + } catch (error) { + if (generation === captureGenerationRef.current) + setStatus( + getErrorMessage( + error, + 'Could not read this page. Click the Sim toolbar icon and try again.' + ) + ) + } finally { + if (generation === captureGenerationRef.current) { + busyRef.current = false + } + } + } + + useEffect(() => { + let disposed = false + let appTheme: 'light' | 'dark' | undefined + const systemTheme = window.matchMedia('(prefers-color-scheme: dark)') + const applyTheme = () => { + const theme = appTheme ?? (systemTheme.matches ? 'dark' : 'light') + document.documentElement.classList.toggle('dark', theme === 'dark') + document.documentElement.classList.toggle('light', theme === 'light') + } + applyTheme() + systemTheme.addEventListener('change', applyTheme) + let metadataGeneration = 0 + const updateSite = async () => { + const generation = ++metadataGeneration + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }) + let next: { origin: string; allowed: boolean } | null = null + if (tab?.url && !tab.incognito) { + const url = new URL(tab.url) + if (['http:', 'https:'].includes(url.protocol) && url.origin !== SIM_ORIGIN) { + next = { + origin: url.origin, + allowed: await chrome.permissions.contains({ origins: [`${url.origin}/*`] }), + } + } + } + if (!disposed && generation === metadataGeneration) setSite(next) + } + const update = () => { + void updateSite().catch(() => { + if (!disposed) setSite(null) + }) + } + let consumingIntent = false + const consumeIntent = async () => { + if (disposed || consumingIntent || busyRef.current || pendingRef.current) return + consumingIntent = true + try { + const window = await chrome.windows.getCurrent() + if (window.id === undefined) return + const stored = await chrome.storage.session.get(null) + if (disposed || busyRef.current || pendingRef.current) return + const keys = Object.keys(stored).filter((key) => + key.startsWith(`${CAPTURE_INTENT_KEY}:${window.id}:`) + ) + const intents = keys + .flatMap((key) => { + const intent = captureIntentSchema.safeParse(stored[key]) + return intent.success && intent.data.windowId === window.id ? [intent.data] : [] + }) + .sort((left, right) => right.createdAt - left.createdAt) + if (keys.length) await chrome.storage.session.remove(keys) + const intent = intents[0] + if (intent && intent.createdAt <= intentCutoffRef.current) { + if (!disposed) + setStatus( + 'A capture from the previous conversation was discarded. Attach the page again.' + ) + return + } + if (disposed || !intent) return + if (Date.now() - intent.createdAt > CAPTURE_INTENT_TTL_MS) { + setStatus('The capture expired. Attach the page again.') + return + } + await capture(intent.kind, intent) + } catch { + if (!disposed) setStatus('Could not receive the capture request. Attach the page again.') + } finally { + consumingIntent = false + } + } + const storageChanged = ( + changes: Record, + area: string + ) => { + if (area === 'session' && Object.values(changes).some((change) => change.newValue)) + void consumeIntent() + } + const message = (event: MessageEvent) => { + if (event.origin !== SIM_ORIGIN || event.source !== frameRef.current?.contentWindow) return + const parsed = appToExtensionMessageSchema.safeParse(event.data) + if (!parsed.success) return + if (parsed.data.type === 'theme') { + appTheme = parsed.data.theme + applyTheme() + } else if (parsed.data.type === 'ready') { + if (sessionRef.current && sessionRef.current !== parsed.data.sessionId) invalidateSession() + sessionRef.current = parsed.data.sessionId + const connected: ExtensionToAppMessage = { + channel: 'sim-extension', + version: 1, + type: 'connected', + sessionId: parsed.data.sessionId, + } + frameRef.current?.contentWindow?.postMessage(connected, SIM_ORIGIN) + deliver() + } else if (parsed.data.sessionId !== sessionRef.current) { + return + } else if (parsed.data.type === 'action') { + if (!navigator.userActivation.isActive) return + if (parsed.data.action === 'site-access') setAccessOpen(true) + else void capture(parsed.data.action === 'capture-page' ? 'page' : 'selection') + } else if (parsed.data.type === 'unavailable') { + invalidateSession() + } else if (pendingRef.current?.id === parsed.data.snapshotId) { + pendingRef.current = null + setStatus('') + void consumeIntent() + } + } + const expire = window.setInterval(() => { + if ( + pendingRef.current && + Date.now() - Date.parse(pendingRef.current.capturedAt) > CAPTURE_INTENT_TTL_MS + ) { + pendingRef.current = null + setStatus('The capture expired. Attach the page again.') + } + void consumeIntent() + }, 5_000) + const tabUpdated = ( + _tabId: number, + change: chrome.tabs.OnUpdatedInfo, + tab: chrome.tabs.Tab + ) => { + if (!tab.active || (change.url === undefined && change.status === undefined)) return + update() + } + window.addEventListener('message', message) + chrome.storage.onChanged.addListener(storageChanged) + chrome.tabs.onActivated.addListener(update) + chrome.tabs.onUpdated.addListener(tabUpdated) + chrome.permissions.onAdded.addListener(update) + chrome.permissions.onRemoved.addListener(update) + update() + void consumeIntent() + return () => { + disposed = true + captureGenerationRef.current++ + pendingRef.current = null + sessionRef.current = null + window.clearInterval(expire) + systemTheme.removeEventListener('change', applyTheme) + window.removeEventListener('message', message) + chrome.storage.onChanged.removeListener(storageChanged) + chrome.tabs.onActivated.removeListener(update) + chrome.tabs.onUpdated.removeListener(tabUpdated) + chrome.permissions.onAdded.removeListener(update) + chrome.permissions.onRemoved.removeListener(update) + } + }, []) + + async function toggleSite() { + if (!site) return + const origins = [`${site.origin}/*`] + try { + const changed = site.allowed + ? await chrome.permissions.remove({ origins }) + : await chrome.permissions.request({ origins }) + if (!changed) + setStatus('Site access was not changed. You can still use the Sim toolbar icon.') + else setStatus('') + } catch { + setStatus('Could not change site access. Use Chrome’s extension settings.') + } + } + + return ( +
+

+ {status} +

+