diff --git a/.github/actions/ci-scope/action.yml b/.github/actions/ci-scope/action.yml new file mode 100644 index 000000000..cfa3b9eb9 --- /dev/null +++ b/.github/actions/ci-scope/action.yml @@ -0,0 +1,38 @@ +name: Classify CI changes +description: Check the full PR or push diff; missing history runs all checks. +outputs: + runtime_changed: + description: Whether runtime tests and Python packaging are needed. + value: ${{ steps.scope.outputs.runtime_changed }} + desktop_changed: + description: Whether Desktop or bundled runtime inputs changed. + value: ${{ steps.scope.outputs.desktop_changed }} +runs: + using: composite + steps: + - id: scope + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + run_all() { + echo "runtime_changed=true" >> "$GITHUB_OUTPUT" + echo "desktop_changed=true" >> "$GITHUB_OUTPUT" + } + if [[ "$EVENT_NAME" != "pull_request" && "$EVENT_NAME" != "push" ]] || + [[ -z "$BASE_SHA" || "$BASE_SHA" =~ ^0+$ ]] || + ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null || + ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then + run_all + exit 0 + fi + if [[ "$EVENT_NAME" == "pull_request" ]]; then + if ! BASE_SHA="$(git merge-base "$BASE_SHA" "$HEAD_SHA")"; then + run_all + exit 0 + fi + fi + git diff --no-renames --name-only -z "$BASE_SHA" "$HEAD_SHA" \ + | python scripts/ci_scope.py >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/desktop-ci.yml b/.github/workflows/desktop-ci.yml index aa96cf4d5..64984735d 100644 --- a/.github/workflows/desktop-ci.yml +++ b/.github/workflows/desktop-ci.yml @@ -12,7 +12,7 @@ permissions: concurrency: group: desktop-ci-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: quality: @@ -28,37 +28,9 @@ jobs: with: fetch-depth: 0 - - name: Detect desktop-impacting changes + - name: Classify changes id: scope - env: - EVENT_NAME: ${{ github.event_name }} - BEFORE_SHA: ${{ github.event.before }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - run: | - if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then - echo "desktop_changed=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - if [[ "$EVENT_NAME" == "pull_request" ]]; then - compare_from="$BASE_SHA" - else - compare_from="$BEFORE_SHA" - fi - - if [[ -z "$compare_from" || "$compare_from" =~ ^0+$ ]]; then - echo "desktop_changed=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - if ! git cat-file -e "${compare_from}^{commit}" 2>/dev/null; then - echo "desktop_changed=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - git diff --no-renames --name-only -z "$compare_from" "$HEAD_SHA" \ - | python3 scripts/desktop_ci_scope.py >> "$GITHUB_OUTPUT" + uses: ./.github/actions/ci-scope - name: Skip unaffected Desktop checks if: steps.scope.outputs.desktop_changed != 'true' @@ -128,6 +100,12 @@ jobs: with: components: clippy, rustfmt + - name: Cache Rust dependencies + if: steps.scope.outputs.desktop_changed == 'true' + uses: Swatinem/rust-cache@v2 + with: + workspaces: desktop/src-tauri + - name: Install frontend dependencies if: steps.scope.outputs.desktop_changed == 'true' working-directory: desktop @@ -154,8 +132,8 @@ jobs: working-directory: desktop/src-tauri run: | cargo fmt --check - cargo clippy --all-targets -- -D warnings - cargo test --all-targets + cargo clippy --locked --all-targets -- -D warnings + cargo test --locked --all-targets bundle: name: Bundle ${{ matrix.name }} @@ -254,6 +232,11 @@ jobs: - name: Set up Rust uses: dtolnay/rust-toolchain@stable + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: desktop/src-tauri + - name: Install frontend dependencies working-directory: desktop run: npm ci diff --git a/.github/workflows/linting.yaml b/.github/workflows/linting.yaml index ce9be58db..5177b4d17 100644 --- a/.github/workflows/linting.yaml +++ b/.github/workflows/linting.yaml @@ -8,9 +8,16 @@ on: branches: - main +permissions: + contents: read + +concurrency: + group: lint-ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: lint-and-format: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: @@ -18,14 +25,20 @@ jobs: uses: actions/checkout@v4 - name: Set up Python + id: python uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Install dependencies + - name: Cache pre-commit environments + uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml', 'scripts/ci/requirements.lock') }} + + - name: Install pre-commit run: | - python -m pip install --upgrade pip - pip install pre-commit + python -m pip install -c scripts/ci/requirements.lock pre-commit - name: Run pre-commit run: pre-commit run --all-files --show-diff-on-failure diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 3a3431211..9ef2a84f0 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -31,6 +31,19 @@ jobs: python-version: "3.12" cache: pip + - name: Set up Node for packaged web assets + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: desktop/package-lock.json + + - name: Build packaged browser client + working-directory: desktop + run: | + npm ci + npm run build:web + - name: Build release distributions run: | python -m pip install --upgrade pip build packaging twine diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index e06960d8c..080a51d78 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -9,9 +9,13 @@ on: permissions: contents: read +concurrency: + group: python-ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 strategy: fail-fast: false @@ -21,22 +25,40 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Classify changes + id: scope + uses: ./.github/actions/ci-scope - name: Set up Python + if: steps.scope.outputs.runtime_changed == 'true' uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip + cache-dependency-path: scripts/ci/requirements.lock - name: Install package and test tools + if: steps.scope.outputs.runtime_changed == 'true' run: | - python -m pip install --upgrade pip - python -m pip install -e ".[test]" + python -m pip install -r scripts/ci/requirements.lock + python -m pip install --no-deps --no-build-isolation -e ".[test]" - name: Run tests + if: steps.scope.outputs.runtime_changed == 'true' run: | python -m pip check - python -m pytest -q + python -m pytest -q --durations=10 --junitxml=test-results/python.xml + + - name: Upload test results + if: always() && steps.scope.outputs.runtime_changed == 'true' + uses: actions/upload-artifact@v4 + with: + name: python-${{ matrix.python-version }}-results + path: test-results/ + retention-days: 7 windows-lifecycle: name: Windows lifecycle locks and platform isolation @@ -46,21 +68,33 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Classify changes + id: scope + uses: ./.github/actions/ci-scope - name: Set up Python + if: steps.scope.outputs.runtime_changed == 'true' uses: actions/setup-python@v5 with: python-version: "3.12" cache: pip + cache-dependency-path: scripts/ci/requirements.lock - name: Install package and test tools + if: steps.scope.outputs.runtime_changed == 'true' + shell: bash run: | - python -m pip install --upgrade pip - python -m pip install -e ".[test]" + python -m pip install -r scripts/ci/requirements.lock + python -m pip install --no-deps --no-build-isolation -e ".[test]" + python -m pip check - name: Verify shared leases, startup recovery, and scheduler leadership + if: steps.scope.outputs.runtime_changed == 'true' run: >- - python -m pytest -q + python -m pytest -q --durations=10 --junitxml=test-results/windows-lifecycle.xml tests/application/test_application_lease.py tests/application/test_automation_scheduler_leadership.py tests/application/test_session_deletion_service.py @@ -70,36 +104,77 @@ jobs: # backend) that the ubuntu job can only skip. This is the sole place # they actually execute. - name: Verify Windows ACLs and Job Object sandbox + if: steps.scope.outputs.runtime_changed == 'true' run: >- - python -m pytest -q + python -m pytest -q --durations=10 --junitxml=test-results/windows-platform.xml tests/test_private_storage_windows.py tests/test_harness_sandbox.py tests/test_exec_sandbox_wiring.py + tests/app_server/test_windows_task.py + tests/app_server/test_state_backup.py + tests/test_provider_oauth.py + tests/test_provider_protocols.py + tests/app_server/test_service.py + tests/app_server/test_service_discovery.py + + - name: Upload test results + if: always() && steps.scope.outputs.runtime_changed == 'true' + uses: actions/upload-artifact@v4 + with: + name: windows-lifecycle-results + path: test-results/ + retention-days: 7 package: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - name: Check out repository uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Classify changes + id: scope + uses: ./.github/actions/ci-scope - name: Set up Python + if: steps.scope.outputs.runtime_changed == 'true' uses: actions/setup-python@v5 with: python-version: "3.12" cache: pip + cache-dependency-path: scripts/ci/requirements.lock + + - name: Set up Node for packaged web assets + if: steps.scope.outputs.runtime_changed == 'true' + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: desktop/package-lock.json + + - name: Build packaged browser client + if: steps.scope.outputs.runtime_changed == 'true' + working-directory: desktop + run: | + npm ci + npm run build:web - name: Build distributions + if: steps.scope.outputs.runtime_changed == 'true' run: | - python -m pip install --upgrade pip build packaging twine - python -m build + python -m pip install -r scripts/ci/requirements.lock + python -m build --no-isolation python -m twine check dist/* - name: Verify distribution metadata and installed runtime + if: steps.scope.outputs.runtime_changed == 'true' run: python scripts/verify_python_distribution.py --dist-dir dist - name: Upload distributions + if: steps.scope.outputs.runtime_changed == 'true' uses: actions/upload-artifact@v4 with: name: python-distributions diff --git a/.github/workflows/security-ci.yml b/.github/workflows/security-ci.yml index 1aa6fb1a3..f67ff8ac0 100644 --- a/.github/workflows/security-ci.yml +++ b/.github/workflows/security-ci.yml @@ -14,7 +14,7 @@ permissions: concurrency: group: security-ci-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: secret-scan: @@ -46,8 +46,15 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Classify changes + id: scope + uses: ./.github/actions/ci-scope - name: Set up Node + if: steps.scope.outputs.runtime_changed == 'true' uses: actions/setup-node@v4 with: node-version: "22" @@ -55,6 +62,7 @@ jobs: cache-dependency-path: desktop/package-lock.json - name: Set up Python + if: steps.scope.outputs.runtime_changed == 'true' uses: actions/setup-python@v5 with: python-version: "3.12" @@ -64,9 +72,18 @@ jobs: desktop/sidecar-requirements.lock - name: Set up Rust + if: steps.scope.outputs.runtime_changed == 'true' uses: dtolnay/rust-toolchain@stable + - name: Cache Rust audit tools + if: steps.scope.outputs.runtime_changed == 'true' + uses: Swatinem/rust-cache@v2 + with: + workspaces: desktop/src-tauri + key: audit-0.22.2 + - name: Audit Node dependencies + if: steps.scope.outputs.runtime_changed == 'true' working-directory: desktop run: | npm ci @@ -75,6 +92,7 @@ jobs: npm sbom --sbom-format cyclonedx > build/security/node-sbom.json - name: Audit Python package dependencies + if: steps.scope.outputs.runtime_changed == 'true' run: | python -m pip install --upgrade pip pip-audit==2.10.1 mkdir -p desktop/build/security @@ -87,7 +105,14 @@ jobs: --output desktop/build/security/python-sbom.json rm -rf desktop/build/python-audit-env + - name: Audit pinned CI dependencies + if: steps.scope.outputs.runtime_changed == 'true' + run: >- + python -m pip_audit --requirement scripts/ci/requirements.lock + --no-deps --disable-pip + - name: Audit locked App Server environment + if: steps.scope.outputs.runtime_changed == 'true' working-directory: desktop run: | npm run setup:sidecar @@ -96,15 +121,18 @@ jobs: --output build/security/sidecar-sbom.json - name: Audit Rust dependencies + if: steps.scope.outputs.runtime_changed == 'true' run: | cargo install cargo-audit --version 0.22.2 --locked cargo audit --file desktop/src-tauri/Cargo.lock - name: Audit dependency licenses + if: steps.scope.outputs.runtime_changed == 'true' working-directory: desktop run: npm run audit:licenses - name: Upload audit reports + if: steps.scope.outputs.runtime_changed == 'true' uses: actions/upload-artifact@v4 with: name: dependency-security-reports diff --git a/.github/workflows/web-ci.yml b/.github/workflows/web-ci.yml new file mode 100644 index 000000000..9e32122de --- /dev/null +++ b/.github/workflows/web-ci.yml @@ -0,0 +1,62 @@ +name: Web client acceptance + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: web-ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + browser: + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Classify changes + id: scope + uses: ./.github/actions/ci-scope + - uses: actions/setup-python@v5 + if: steps.scope.outputs.runtime_changed == 'true' + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/ci/requirements.lock + - uses: actions/setup-node@v4 + if: steps.scope.outputs.runtime_changed == 'true' + with: + node-version: "22" + cache: npm + cache-dependency-path: desktop/package-lock.json + - name: Install test runtime + if: steps.scope.outputs.runtime_changed == 'true' + run: | + python -m pip install -r scripts/ci/requirements.lock + python -m pip install --no-deps --no-build-isolation -e ".[test]" + python -m pip check + - name: Install browser tooling + if: steps.scope.outputs.runtime_changed == 'true' + working-directory: desktop + run: | + npm ci + npx playwright install --with-deps chromium + - name: Run deterministic browser acceptance + if: steps.scope.outputs.runtime_changed == 'true' + working-directory: desktop + env: + DEEPCODE_TEST_PYTHON: python + run: npm run test:web + - uses: actions/upload-artifact@v4 + if: always() && steps.scope.outputs.runtime_changed == 'true' + with: + name: web-test-results + path: desktop/test-results/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index 06273589f..72bd3bf53 100644 --- a/.gitignore +++ b/.gitignore @@ -21,8 +21,6 @@ build/ site/ desktop/src-tauri/target/ desktop/src-tauri/gen/ -desktop/src-tauri/binaries/* -!desktop/src-tauri/binaries/.gitkeep desktop/build/sidecar/ # Logs / Reports @@ -84,6 +82,9 @@ memory-bank/ # project files deepcode_lab/ uploads/ +/app_server/web_assets/ +/desktop/test-results/ +/desktop/playwright-report/ # Secrets belong in DeepCode's credential store or environment variables. # NEVER commit real API keys or local user configuration. diff --git a/MANIFEST.in b/MANIFEST.in index 62c88db4c..937120d49 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,6 +12,7 @@ recursive-include core *.py recursive-include core/skills/builtin * recursive-include core/application/goal_prompts *.md recursive-include app_server *.py +recursive-include app_server/web_assets * recursive-include protocol *.json recursive-include utils *.py recursive-include tools *.py diff --git a/README.md b/README.md index f4f720768..aaaa4a858 100644 --- a/README.md +++ b/README.md @@ -78,11 +78,10 @@ *Work with DeepCode in a visual workspace for Sessions, goals, tool activity, code changes, and verification.* -DeepCode has one Agent runtime and two interfaces: an interactive CLI for -terminal workflows and a Tauri Desktop workbench for visual Sessions, review, -and settings. Both open the same local Projects, Session history, models, -Skills, permissions, Goals, and Automations. See the -[`Desktop source guide`](desktop/README.md) to run the application locally. +DeepCode provides TUI, Desktop, and Web clients over one shared local service. +They share Projects, Session history, models, Skills, permissions, Goals, and +Automations. Use `deepcode`, `deepcode desktop`, or `deepcode web`; see +[Quick start](#quick-start) for installation and your first task. --- @@ -133,6 +132,14 @@ Skills, permissions, Goals, and Automations. See the - [Automate repeatable engineering work](#automate-repeatable-engineering-work) - [Paper2Code](#paper2code) - [⚡ Quick start](#quick-start) + - [Launch commands at a glance](#launch-commands-at-a-glance) + - [Install the runtime](#install-the-runtime) + - [Configure a model](#configure-a-model) + - [Start the TUI](#start-the-tui) + - [Start Desktop](#start-desktop) + - [Start Web](#start-web) + - [Complete your first task](#complete-your-first-task) + - [Manage the shared background service](#manage-the-shared-background-service) - [🧭 Using DeepCode](#using-deepcode) - [⚙️ Headless and automation](docs/HEADLESS_AND_AUTOMATION.md) - [🔬 Paper2Code](#paper2code-1) @@ -151,6 +158,29 @@ Skills, permissions, Goals, and Automations. See the ## News +**2026-09-09 · TUI, Desktop, and Web share one local background service** + +- **Choose your interface.** Run `deepcode` for the TUI, `deepcode desktop` + for the native app, or `deepcode web` to open the workbench in your browser. + All three connect to the same local service; Web needs no DeepCode account. +- **Continue the same task across interfaces.** Closing a client leaves running + work in the background. Reconnect from another interface to follow the same + conversation, review tool activity, and respond to approvals. The computer + must remain awake, and pending approvals still need your response. +- **Configure and verify your models.** Save cloud or local model connections + for use across all three interfaces. Browse available models, check a model's + response, and test streaming and tool calls before using a custom server. + See [Models and providers](docs/guide/models.md). +- **Manage your background service and prepare for upgrades.** Use + `deepcode service` commands to check status, read logs, or stop after current + work finishes. Create a runtime snapshot before upgrading and restore it when + needed; keep project files in your usual version control or backup. + See [Upgrading DeepCode](docs/UPGRADE_AND_RESTORE.md). + +Start with the updated [Quick start](#quick-start) and +[first coding task tutorial](docs/guide/getting-started.md). +([#212](https://github.com/HKUDS/DeepCode/pull/212)) + **2026-09-06 · DeepCode v2.2.0: Session context-window caps, compaction that leaves a memory, and a fully localized Desktop** - **Cap a Session's context window.** `/context 64k` in the TUI or the @@ -604,9 +634,9 @@ you finish real software engineering work more reliably. ## Core capabilities -DeepCode provides a complete local Coding Agent workflow. CLI and Desktop are -two ways to use the same Agent, Sessions, models, Skills, permissions, and task -state. +DeepCode provides a complete local Coding Agent workflow. TUI, Desktop, and Web +use the same Agent, Sessions, models, Skills, permissions, and task state through +the shared local service.

DeepCode Agent Harness and verification loop @@ -725,7 +755,14 @@ Automation does not launch a separate, reduced Agent. It uses the same Sessions, models, Skills, permissions, approvals, and recovery behavior, and keeps the history of every run. -### Paper2Code +### Shared background service + +Desktop, Web, TUI, `exec`, `loop`, and MCP task calls use the same local service. +Starting a client starts the service when needed. Closing a client leaves its +accepted tasks running; explicitly stop the service to shut down the runtime. +See [Quick start](#quick-start) for installation and launch commands. + +## Paper2Code Paper2Code was DeepCode's original research direction and remains its dedicated workflow for research reproduction. @@ -739,156 +776,188 @@ run, be inspected, and keep improving. ## Quick start -DeepCode has two interfaces with separate installation paths. Choose one to get -started; both use the same Agent runtime and canonical Session history. +Use DeepCode to explore a repository, make changes, and run tests. Choose the +terminal, desktop app, or browser—your projects and conversations are available +in all three. -> `uv tool install --python 3.12 deepcode-hku` installs the CLI and shared -> Python runtime. It does **not** install the Tauri Desktop application. +### Launch commands at a glance -### Option A — Install the CLI +After installation, open the interface you prefer: -Install `uv` first if it is not already available. On Windows PowerShell: +| Interface | Start command | +|---|---| +| **TUI** — work in your terminal | `deepcode` | +| **Desktop** — use a native app | `deepcode desktop` | +| **Web** — work in your browser | `deepcode web` | -```powershell -winget install --id astral-sh.uv --exact -``` +DeepCode starts its local background service automatically. You can close an +interface and return to your conversation later. + +### Install the runtime -Open a new terminal after the first `uv` installation, then run: +#### Published package + +With `uv` installed, run: ```console uv tool install --python 3.12 deepcode-hku deepcode init ``` -The explicit Python selection is intentional: DeepCode requires Python 3.12+ -and must not fall back to an unsupported legacy package on an older interpreter. -If an existing uv tool environment still contains DeepCode 1.x, migrate it with -`uv tool upgrade --python 3.12 deepcode-hku`. +On Windows, you can install `uv` with `winget install --id astral-sh.uv --exact`, +then open a new terminal. DeepCode requires Python 3.12 or later. + +To use the native app, also install DeepCode Desktop from +[GitHub Releases](https://github.com/HKUDS/DeepCode/releases). To try changes +that have not been released yet, install from source below. + +

+Install from source + +#### Current source checkout + +You will need Git, `uv`, and Node.js 22 or later. Run these commands in your terminal: + +```console +git clone https://github.com/HKUDS/DeepCode.git +cd DeepCode +npm --prefix desktop ci +npm --prefix desktop run build:web +uv tool install --python 3.12 --force . +deepcode init +``` + +If you already have the repository, start in its root directory and skip cloning. +You can now run `deepcode` from any directory. For source Desktop, also install +Rust and the platform dependencies described in the [Desktop guide](desktop/README.md). + +
+ +If your terminal cannot find `deepcode`, run `uv tool update-shell` and open a +new terminal. + +### Configure a model + +Choose either the graphical setup or the terminal commands below. -Create a model connection once. `--api-key` opens a non-echoing prompt: +**In Desktop or Web:** open **Settings → AI providers**, select **Add provider**, +and enter your provider's API key. Click **Save and check** to load its models. +Under **Agent model**, choose a model and click **Save and verify model** to try +a short request. Then choose the connection and model in your conversation's +message box. + +
+ Choose a provider and model in DeepCode settings +
+ +**In your terminal:** this example connects to OpenRouter. Enter your API key +when prompted; it will not be displayed as you type. ```console -deepcode provider set personal-openrouter --template openrouter --label "OpenRouter · Personal" --api-key -deepcode provider models personal-openrouter --refresh -deepcode provider test personal-openrouter --model +deepcode provider set my-openrouter --template openrouter --api-key +deepcode provider models my-openrouter --refresh ``` -Enter the repository you want DeepCode to work in and start the interactive -Agent: +Choose a model from the list. Replace `MODEL_ID` below with its exact ID and +send a short test request: ```console -cd -deepcode +deepcode provider test my-openrouter --model MODEL_ID ``` -`deepcode init` creates minimal user configuration under `~/.deepcode/`. -Credentials are stored separately in user-private storage and are never written -to Session history. `pipx install deepcode-hku` and `pip install deepcode-hku` -are also supported in an appropriate Python 3.12+ environment. +You can use the same connection in all three interfaces. For other providers, +local models, and tool-calling checks, see [Models and providers](docs/guide/models.md). + +### Start the TUI -### Option B — Install Desktop +Open a terminal in the project you want to work on. On your first visit, use +`--trust` to allow DeepCode to work in that folder. With the connection configured +above: -Desktop release bundles are distributed separately from the Python package. -Check [GitHub Releases](https://github.com/HKUDS/DeepCode/releases) for a signed -installer for your platform. If no installer is attached, use the source setup -below. +```console +cd /path/to/your-project +deepcode --trust --connection my-openrouter --model MODEL_ID +``` -#### macOS and Linux from source +Replace the path with your project directory; on Windows, for example, +`C:\projects\my-app`. For later visits, run `deepcode` in that directory and use +`/model` to choose your connection and model. Use `/help` to see available commands. -Install the platform dependencies from the -[Tauri 2 prerequisite guide](https://v2.tauri.app/start/prerequisites/), plus -Git, Python 3.12+, `uv`, Node.js 22+, and stable Rust. Then run: +### Start Desktop -```bash -git clone https://github.com/HKUDS/DeepCode.git -cd DeepCode -uv venv --python 3.12 -uv pip install --python .venv/bin/python -e . -.venv/bin/deepcode init -cd desktop -npm ci -npm run setup:sidecar -npm run build:sidecar -cd .. -mkdir -p ~/.local/bin -ln -sf "$(pwd)/scripts/deepcode-desktop" ~/.local/bin/deepcode-desktop -export PATH="$HOME/.local/bin:$PATH" -deepcode-desktop +```console +deepcode desktop ``` -The final link is a one-time source launcher installation. Afterwards, -`deepcode-desktop` starts this checkout from any directory, provided -`~/.local/bin` is on `PATH`. Add the export to your shell profile if it is not -already configured. The command launches Desktop; add or select the repository -you want to work on from the Project sidebar. - -#### Windows from source - -Windows requires Microsoft Edge WebView2 and the Visual Studio 2022 Build Tools -workload **Desktop development with C++**. Accept the UAC prompt raised by Build -Tools: - -```powershell -winget install --id Git.Git --exact -winget install --id astral-sh.uv --exact -winget install --id OpenJS.NodeJS.LTS --exact -winget install --id Rustlang.Rustup --exact -winget install --id Microsoft.VisualStudio.2022.BuildTools --exact ` - --override "--wait --passive --norestart --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended" +Add your project folder, confirm that you trust it, and click **New thread**. +Choose a model in the message box, then describe what you want to do. + +If you installed from source, this command opens the development app. Its first +launch may take longer while dependencies are prepared; keep the terminal open +while you use it. See the [Desktop guide](desktop/README.md) for platform setup +and custom installation paths. + +### Start Web + +```console +deepcode web ``` -Close PowerShell, open a new window, and verify the toolchains: +DeepCode opens a browser tab. Add your project folder on this computer, confirm +that you trust it, and create a thread. Select a model and send your first message. +No DeepCode account is needed. + +To copy the access link and open it yourself, use `deepcode web --no-open`. +Open the link within 60 seconds. If it expires or the page asks you to sign in +again, run `deepcode web` to get a fresh link. + +### Complete your first task -```powershell -git --version -uv --version -node --version -rustup default stable-msvc -rustc --version -cargo --version +Start by asking DeepCode to help you understand your project: + +```text +Explain how this project is organized and how to run its tests. +Do not change any files yet. ``` -Clone, prepare, and start Desktop: +Then ask for a small change, describing the behavior you want and how to check it: -```powershell -git clone https://github.com/HKUDS/DeepCode.git -Set-Location DeepCode -uv venv --python 3.12 -uv pip install --python .venv\Scripts\python.exe -e . -.venv\Scripts\deepcode.exe init -Set-Location desktop -npm ci -$env:DEEPCODE_PYTHON = (Resolve-Path ..\.venv\Scripts\python.exe) -npm run setup:sidecar -npm run build:sidecar -npm run tauri -- dev +```text +Add tests for the configuration loader's handling of missing and invalid +values. Follow the existing test style, run the relevant tests, and summarize +the changes and results. ``` -Keep that PowerShell window open while Desktop is running. See the -[Desktop source guide](desktop/README.md#windows-powershell) for subsequent -launches and troubleshooting. +Adapt the task to your repository. Follow the tool activity as DeepCode works, +respond to any approval requests, and review the changed files and test output. +You can send a follow-up message to adjust the task, or use the stop button +(`/stop` in the TUI) to interrupt it. -#### Configure the Desktop model +Your conversation is saved automatically. In Desktop/Web, select it from the +project's thread list. In the TUI, use `/resume` to pick a saved conversation. +You can switch interfaces and continue the same task. -Open **Settings → AI providers** after Desktop starts. +For a complete exercise in a new folder, follow +[Your first coding task](docs/guide/getting-started.md). It takes you through +creating a Python function, testing it, and returning to the conversation later. -
- Configure an AI provider and model in DeepCode Desktop +### Manage the shared background service - Provider credentials, model discovery, and inference verification stay in one Desktop workflow. -
+Closing an interface leaves running tasks in the background. Pending approvals +still need your response, and the computer must remain awake for work to continue. + +Most days, you can just launch your preferred interface. When you need to inspect +or stop the background service, use the command for that action: -1. Select **Add provider**, choose the service, and enter an API key or its - environment-variable name. -2. Select **Save and check** to verify the credential and load the provider's - model catalog without sending repository content. -3. Under **Agent model**, choose an exact model ID and select **Save and verify - model**. This final check sends only a minimal inference request. -4. Add or open a Project, create a Session, choose the model, Thinking effort, - and access level, then describe the task in natural language. +| What you want to do | Command | +|---|---| +| Check whether DeepCode is running | `deepcode service status` | +| Read recent logs | `deepcode service logs --lines 100` | +| Stop after current work finishes | `deepcode service stop --drain --timeout 60` | +| Start automatically after signing into your computer | `deepcode service install --at-login` | -> The interface changes how the work is presented, not the Agent, policy, -> configuration, or Session history behind it. +For updates and backups, follow [Upgrading DeepCode](docs/UPGRADE_AND_RESTORE.md). +If something does not work, start with [Troubleshooting](docs/guide/troubleshooting.md). ## Using DeepCode @@ -900,10 +969,10 @@ skills and memory, and headless automation — with worked examples. Every task lives in a durable Session attached to its original Project. Open a Project in Desktop or start `deepcode` from its directory, then create a new -Session or resume an existing one. The same history can move between Desktop -and CLI without export or conversion. +Session or resume an existing one. The same history is available in TUI, +Desktop, and Web without export or conversion. -| What you want to do | Desktop | Interactive CLI | +| What you want to do | Desktop / Web | Interactive CLI | |---|---|---| | Start a Session | **New thread** | `/new [title]` | | Resume local history | Select it under the Project | `/resume` | @@ -924,9 +993,11 @@ and CLI without export or conversion. | Rename or delete a Session | Session context menu | `/rename ` · `/delete <id>` | Session history, tool activity, approvals, Goal state, and verification evidence -remain together. A Session has one live writer at a time: Desktop and the CLI -may hold it open together, but if one is mid-Turn the other refuses new input -with a message naming the holder instead of corrupting shared history. Archiving hides a Session without deleting its history; +remain together. TUI, Desktop, and Web can observe the same Session while the +shared service serializes its execution. Normal input during a Turn steers the +active work; explicit queueing requests a subsequent Turn. The interface reports +how the input was accepted. Closing a client leaves accepted work running. +Archiving hides a Session without deleting its history; permanent deletion removes the Session records but never repository files. ### Connections and models @@ -1247,6 +1318,9 @@ On Windows PowerShell, activate with `.\.venv\Scripts\Activate.ps1`. ### Verification +To reproduce CI's Python environment and understand each check, see the +[CI guide](docs/CI.md). + ```bash uvx pre-commit run --all-files python -m compileall -q app_server cli core tools workflows diff --git a/README_ZH.md b/README_ZH.md index 78c1749ea..e24929a27 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -77,11 +77,10 @@ *在可视化工作台中使用 DeepCode，管理 Session 和目标，并查看工具活动、代码修改与验证过程。* </div> -DeepCode 只有一套 Agent 运行时，同时提供两种使用界面：面向终端 -工作流的交互式 CLI，以及用于 Session、审查和设置的 Tauri Desktop。 -两端打开同一份本地 Project、Session 历史、模型、Skills、权限、Goals 与 -Automations。从源码启动请参考 -[`Desktop 运行指南`](desktop/README.md)。 +DeepCode 提供 TUI、Desktop 和 Web 三种客户端，连接同一个本地共享后台， +共用 Project、Session 历史、模型、Skills、权限、Goals 与 Automations。 +分别使用 `deepcode`、`deepcode desktop` 和 `deepcode web` 启动；安装及首次任务 +见[快速开始](#快速开始)。 --- @@ -132,6 +131,14 @@ Automations。从源码启动请参考 - [让可重复的工程工作自动化](#让可重复的工程工作自动化) - [Paper2Code](#paper2code) - [⚡ 快速开始](#快速开始) + - [三种启动入口](#三种启动入口) + - [安装 DeepCode](#安装-deepcode) + - [配置并选择模型](#配置并选择模型) + - [启动 TUI](#启动-tui) + - [启动 Desktop](#启动-desktop) + - [启动 Web](#启动-web) + - [完成并验收第一个任务](#完成并验收第一个任务) + - [管理共享后台与排查问题](#管理共享后台与排查问题) - [🧭 使用 DeepCode](#使用-deepcode) - [⚙️ Headless 与 Automation](docs/HEADLESS_AND_AUTOMATION.md#中文) - [🔬 Paper2Code](#paper2code-1) @@ -150,6 +157,25 @@ Automations。从源码启动请参考 ## 新闻 +**2026-09-09 · TUI、Desktop 与 Web 共用一个本地后台** + +- **选择你习惯的界面。** 运行 `deepcode` 启动 TUI，`deepcode desktop` + 打开桌面应用，或用 `deepcode web` 在浏览器中打开工作台。三种界面连接 + 同一个本地后台，Web 无需注册 DeepCode 账号。 +- **换个界面，继续同一个任务。** 关闭客户端后，正在执行的任务继续在后台 + 运行。你可以从其他界面重新连接，查看同一段对话、工具活动并处理审批。 + 任务继续执行需要电脑保持唤醒，遇到审批时仍需你确认。 +- **配置模型，并验证它能否正常工作。** 保存的云端或本地模型连接可在三种 + 界面中使用。你可以查看可用模型、发送简短请求检查响应，并在使用自定义 + 模型服务前验证流式输出和工具调用。参见[模型与服务商指南](docs/guide/models.md)。 +- **管理后台，为升级做好准备。** 使用 `deepcode service` 命令查看状态和 + 日志，或等待当前任务完成后停止服务。升级前可以创建运行时数据快照， + 需要时再恢复；项目文件仍使用你原有的版本管理或备份方式。 + 参见[升级与恢复指南](docs/UPGRADE_AND_RESTORE.md)。 + +从更新后的[快速开始](#快速开始)和[第一个编程任务教程](docs/guide/getting-started.md) +开始使用。（[#212](https://github.com/HKUDS/DeepCode/pull/212)） + **2026-09-06 · DeepCode v2.2.0：Session 上下文窗口上限、会留下记忆的压缩，以及完整本地化的 Desktop** - **给 Session 的上下文窗口设上限。** TUI 里的 `/context 64k` 或 Desktop 模型 @@ -497,7 +523,7 @@ DeepCode 的目标不是让 Agent 显得更忙，而是帮助你更可靠地完 ## 核心能力 -DeepCode 提供完整的本地 Coding Agent 工作流。CLI 和 Desktop 只是两种使用方式，它们共享同一套 Agent、Session、模型、Skills、权限和任务状态。 +DeepCode 提供完整的本地 Coding Agent 工作流。TUI、Desktop 和 Web 通过共享后台，使用同一套 Agent、Session、模型、Skills、权限和任务状态。 <p align="center"> <img src="assets/readme/verification-loop.png" alt="DeepCode Agent Harness 与验证循环" width="1080" /> @@ -584,150 +610,171 @@ Paper2Code 是 DeepCode 最初的研究方向，也是当前产品中专门面 ## 快速开始 -DeepCode 提供两种界面，并且对应两条独立安装路径。任选一种即可开始；两端 -使用相同的 Agent 运行时和规范 Session 历史。 +你可以用 DeepCode 阅读项目代码、开发功能、修复问题和运行测试。选择习惯的终端、 +桌面应用或浏览器即可，项目和对话可以在三端继续使用。 -> `uv tool install --python 3.12 deepcode-hku` 安装的是 CLI 和共享 Python -> 运行时，**不会**安装 Tauri Desktop 应用。 +### 三种启动入口 -### 方案 A：安装 CLI +安装后，选择一种方式打开 DeepCode： -如果尚未安装 `uv`，请先安装。Windows PowerShell 使用： +| 界面 | 启动命令 | +|---|---| +| **TUI**：在终端中工作 | `deepcode` | +| **Desktop**：使用桌面应用 | `deepcode desktop` | +| **Web**：在浏览器中工作 | `deepcode web` | -```powershell -winget install --id astral-sh.uv --exact -``` +DeepCode 会自动启动本地后台。你可以关闭界面，下次再回来继续同一段对话。 + +### 安装 DeepCode -首次安装 `uv` 后重新打开终端，再执行： +#### 安装已发布版本 + +准备好 `uv` 后，在终端运行： ```console uv tool install --python 3.12 deepcode-hku deepcode init ``` -这里显式选择 Python 是有意的：DeepCode 要求 Python 3.12+，不能在旧解释器上 -回退安装已经不受支持的历史版本。 -如果现有 uv tool 环境仍安装着 DeepCode 1.x，可执行 -`uv tool upgrade --python 3.12 deepcode-hku` 完成迁移。 +Windows 用户可以先运行 `winget install --id astral-sh.uv --exact` 安装 `uv`， +再重新打开终端。DeepCode 需要 Python 3.12 或更高版本。 + +如果想使用桌面应用，还需要从 [GitHub Releases](https://github.com/HKUDS/DeepCode/releases) +安装 DeepCode Desktop。想体验尚未发布的更新，可以使用下面的源码安装方式。 + +<details> +<summary>从源码安装</summary> + +#### 安装当前源码版本 + +准备 Git、`uv` 和 Node.js 22 或更高版本，然后运行： + +```console +git clone https://github.com/HKUDS/DeepCode.git +cd DeepCode +npm --prefix desktop ci +npm --prefix desktop run build:web +uv tool install --python 3.12 --force . +deepcode init +``` + +已有仓库时，从仓库根目录开始，跳过克隆步骤。安装完成后，可在任意目录运行 +`deepcode`。如果还要运行源码版 Desktop，请按照 [Desktop 指南](desktop/README.md) +准备 Rust 和对应平台的依赖。 + +</details> + +如果终端提示找不到 `deepcode`，运行 `uv tool update-shell`，再重新打开终端。 + +### 配置并选择模型 + +你可以在图形界面中配置，也可以使用终端命令，任选一种即可。 -首次创建模型连接。`--api-key` 会打开不回显的安全输入： +**在 Desktop 或 Web 中：** 打开 **Settings → AI providers**，点击 **Add provider**， +选择模型服务并填写 API Key。点击 **Save and check** 获取模型列表；在 **Agent model** +中选择模型，再点击 **Save and verify model** 发送一条简短请求，检查模型是否可用。 +随后在对话输入框中选择要使用的连接和模型。 + +<div align="center"> + <img src="assets/setting_model.png" alt="在 DeepCode 设置中选择模型服务和模型" width="100%" style="border-radius: 10px; box-shadow: 0 8px 20px rgba(45,55,72,0.2); margin: 15px 0;"/> +</div> + +**在终端中：** 下面以 OpenRouter 为例。按提示输入 API Key，输入内容不会显示在终端上。 ```console -deepcode provider set personal-openrouter --template openrouter --label "OpenRouter · Personal" --api-key -deepcode provider models personal-openrouter --refresh -deepcode provider test personal-openrouter --model <model-id> +deepcode provider set my-openrouter --template openrouter --api-key +deepcode provider models my-openrouter --refresh ``` -进入希望 DeepCode 操作的仓库，然后启动交互式 Agent： +从列表中选择一个模型，将下文中的 `MODEL_ID` 替换为它的完整 ID，再测试连接： ```console -cd <你的项目> -deepcode +deepcode provider test my-openrouter --model MODEL_ID ``` -`deepcode init` 会在 `~/.deepcode/` 下创建最小用户配置。凭证单独保存在 -用户私有存储中，不会进入 Session 历史。也可以在合适的 Python 3.12+ -环境中使用 `pipx install deepcode-hku` 或 `pip install deepcode-hku`。 +配置好的连接可以在三端使用。其他模型服务、本地模型和工具调用检查，见 +[模型配置指南](docs/guide/models.md)。 -### 方案 B：安装 Desktop +### 启动 TUI -Desktop 安装包与 Python 包分开发布。先检查 -[GitHub Releases](https://github.com/HKUDS/DeepCode/releases) 是否提供当前 -平台的签名安装包；如果没有，请使用下面的源码安装流程。 +进入希望 DeepCode 帮你处理的项目目录。第一次使用该目录时，加上 `--trust`， +表示你信任这个项目并允许 DeepCode 在其中工作。使用上面配置的连接启动： -#### macOS 与 Linux 源码安装 +```console +cd /path/to/your-project +deepcode --trust --connection my-openrouter --model MODEL_ID +``` -请先根据 [Tauri 2 前置依赖指南](https://v2.tauri.app/start/prerequisites/) -安装平台依赖，并准备 Git、Python 3.12+、`uv`、Node.js 22+ 和稳定版 Rust。 -然后执行： +将路径替换为你的项目目录；Windows 路径例如 `C:\projects\my-app`。以后回到这个 +目录，运行 `deepcode` 即可打开 TUI。输入 `/model` 选择连接和模型，输入 `/help` +查看可用命令。 -```bash -git clone https://github.com/HKUDS/DeepCode.git -cd DeepCode -uv venv --python 3.12 -uv pip install --python .venv/bin/python -e . -.venv/bin/deepcode init -cd desktop -npm ci -npm run setup:sidecar -npm run build:sidecar -cd .. -mkdir -p ~/.local/bin -ln -sf "$(pwd)/scripts/deepcode-desktop" ~/.local/bin/deepcode-desktop -export PATH="$HOME/.local/bin:$PATH" -deepcode-desktop +### 启动 Desktop + +```console +deepcode desktop ``` -以上步骤会完成一次性的源码启动器安装。以后只要 `~/.local/bin` 已加入 -`PATH`，就可以从任意目录运行 `deepcode-desktop` 启动这个源码版本。如果 -Shell 尚未配置该路径，请把上述 export 写入 Shell profile。命令只负责启动 -Desktop；需要操作的仓库仍应在 Project 侧边栏中添加或选择。 +添加你的项目文件夹，确认信任后，点击 **New thread** 开始对话。在输入框中选择 +模型，然后描述你希望完成的任务。 -#### Windows 源码安装 +如果从源码安装，该命令会打开开发版应用。首次启动需要准备依赖，可能花费较长时间； +使用期间请保持启动终端开启。平台依赖和自定义安装位置见 [Desktop 指南](desktop/README.md)。 -Windows 必须安装 Microsoft Edge WebView2，以及 Visual Studio 2022 Build -Tools 的 **Desktop development with C++** 工作负载。Build Tools 弹出 UAC -提示时请选择“是”： +### 启动 Web -```powershell -winget install --id Git.Git --exact -winget install --id astral-sh.uv --exact -winget install --id OpenJS.NodeJS.LTS --exact -winget install --id Rustlang.Rustup --exact -winget install --id Microsoft.VisualStudio.2022.BuildTools --exact ` - --override "--wait --passive --norestart --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended" +```console +deepcode web ``` -关闭 PowerShell，重新打开一个窗口并验证工具链： +DeepCode 会自动打开浏览器页面。添加这台电脑上的项目目录，确认信任，创建对话， +选择模型后即可开始使用，无需 DeepCode 账号。 + +如果希望自己复制链接打开，运行 `deepcode web --no-open`。请在 60 秒内打开生成的 +访问链接；链接过期或页面提示重新授权时，再运行一次 `deepcode web` 即可。 + +### 完成并验收第一个任务 + +进入项目后，可以先让 DeepCode 帮你了解代码： -```powershell -git --version -uv --version -node --version -rustup default stable-msvc -rustc --version -cargo --version +```text +介绍这个项目的目录结构、主要模块，以及应该如何运行测试。先不要修改文件。 ``` -克隆、准备并启动 Desktop： +了解项目后，再提出一个具体的小任务，说明期望的行为和检查方法。例如： -```powershell -git clone https://github.com/HKUDS/DeepCode.git -Set-Location DeepCode -uv venv --python 3.12 -uv pip install --python .venv\Scripts\python.exe -e . -.venv\Scripts\deepcode.exe init -Set-Location desktop -npm ci -$env:DEEPCODE_PYTHON = (Resolve-Path ..\.venv\Scripts\python.exe) -npm run setup:sidecar -npm run build:sidecar -npm run tauri -- dev +```text +为配置加载模块补充缺少配置项、配置值不合法时的测试。沿用项目现有的测试风格， +运行相关测试，并说明修改了哪些文件、测试结果如何。 ``` -Desktop 运行期间请保持这个 PowerShell 窗口开启。后续启动和故障排查请参考 -[Desktop 源码运行指南](desktop/README.md#windows-powershell)。 +请按你的项目调整任务内容。DeepCode 工作时，你可以查看工具调用、处理审批， +并检查文件修改和测试输出。需要调整方向时，直接发送补充消息；需要中断时， +点击停止按钮，或在 TUI 输入 `/stop`。 -#### 配置 Desktop 模型 +对话会自动保存。在 Desktop/Web 中，从项目的对话列表重新打开；在 TUI 中， +输入 `/resume` 选择历史对话。换一个界面，也可以接着做同一个任务。 -Desktop 启动后，打开 **Settings → AI providers**。 +如果想从空目录开始练习，请阅读[第一个编程任务](docs/guide/getting-started.md)： +跟着创建一个 Python 函数、运行测试，再学习如何恢复对话。 +更多用法见[使用指南目录](docs/guide/README.md)。 -<div align="center"> - <img src="assets/setting_model.png" alt="在 DeepCode Desktop 中配置 AI Provider 与模型" width="100%" style="border-radius: 10px; box-shadow: 0 8px 20px rgba(45,55,72,0.2); margin: 15px 0;"/> +### 管理共享后台与排查问题 - <sub>在同一个 Desktop 流程中完成凭证保存、模型发现和真实推理验证。</sub> -</div> +关闭界面后，已经开始的任务会继续在后台运行；遇到待审批操作时，仍需你回来处理。 +执行期间请保持电脑运行，避免休眠。 -1. 点击 **Add provider**，选择模型服务，并输入 API Key 或保存它的环境变量名。 -2. 点击 **Save and check**，检查凭证并读取 Provider 的模型目录；这一阶段不会 - 发送项目内容。 -3. 在 **Agent model** 中选择准确的模型 ID，然后点击 **Save and verify - model**；最后一步只会发送一次极小的真实推理请求。 -4. 添加或打开 Project，创建 Session，选择模型、Thinking 档位和权限，然后 - 用自然语言描述任务。 +日常直接打开你喜欢的界面即可。需要检查或停止后台时，按需使用以下命令： -> 使用界面只改变工作的呈现方式，不改变背后的 Agent、策略、配置和 Session -> 历史。 +| 你想做什么 | 命令 | +|---|---| +| 检查 DeepCode 是否正在运行 | `deepcode service status` | +| 查看最近的运行日志 | `deepcode service logs --lines 100` | +| 等当前工作结束后停止后台 | `deepcode service stop --drain --timeout 60` | +| 登录电脑后自动启动后台 | `deepcode service install --at-login` | + +更新版本和备份数据时，参考[升级与恢复指南](docs/UPGRADE_AND_RESTORE.md)。 +遇到启动、模型连接或页面断线问题，先查看[故障排查](docs/guide/troubleshooting.md)。 ## 使用 DeepCode @@ -735,9 +782,11 @@ Desktop 启动后，打开 **Settings → AI providers**。 每项工作都保存在与原始 Project 关联的持久 Session 中。在 Desktop 打开 Project，或者从项目目录启动 `deepcode`，然后创建新 Session 或恢复历史。 -同一份历史可以直接在 Desktop 与 CLI 之间继续，无需导出或转换。 +同一份历史可以在 TUI、Desktop 和 Web 中继续，无需导出或转换。共享后台统一执行 +任务；执行中发送的普通消息用于引导当前 Turn，显式排队用于后续 Turn。 +多个客户端可同时查看同一 Session，并处理其中的审批。 -| 需要完成的操作 | Desktop | 交互式 CLI | +| 需要完成的操作 | Desktop / Web | 交互式 CLI | |---|---|---| | 创建 Session | **New thread** | `/new [标题]` | | 恢复当前项目历史 | 在 Project 下选择 Session | `/resume` | @@ -1033,6 +1082,8 @@ Windows PowerShell 请使用 `.\.venv\Scripts\Activate.ps1` 激活环境。 ### 验证 +复现 CI 的 Python 环境、了解各项检查及排查失败，请参考 [CI 指南](docs/CI.md)。 + ```bash uvx pre-commit run --all-files python -m compileall -q app_server cli core tools workflows diff --git a/app_server/__init__.py b/app_server/__init__.py index 591c999f5..3a4ec6328 100644 --- a/app_server/__init__.py +++ b/app_server/__init__.py @@ -1,5 +1 @@ -"""DeepCode local stdio App Server.""" - -from app_server.server import AppServer - -__all__ = ["AppServer"] +"""DeepCode local service and client transports.""" diff --git a/app_server/__main__.py b/app_server/__main__.py index b932a213f..025df5209 100644 --- a/app_server/__main__.py +++ b/app_server/__main__.py @@ -1,4 +1,4 @@ -"""Run the local App Server over stdin/stdout.""" +"""Attach stdin/stdout to the local DeepCode service.""" from __future__ import annotations @@ -10,7 +10,6 @@ import time from pathlib import Path - _PROCESS_STARTED = time.perf_counter() @@ -22,14 +21,29 @@ def _trace_startup(stage: str) -> None: _trace_startup("entrypoint") -from app_server.server import AppServer -from core.application.application import DeepCodeApplication - -_trace_startup("imports-ready") - def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="DeepCode stdio App Server") + parser.add_argument( + "--service", + action="store_true", + help="manage the background service (accepts service subcommands)", + ) + parser.add_argument( + "--serve", + action="store_true", + help="run the loopback Web service (accepts serve options)", + ) + parser.add_argument( + "--web", + action="store_true", + help="start/connect to the service and open its browser client (accepts web options)", + ) + parser.add_argument( + "--managed-config", + type=Path, + help="read a private supervisor launch configuration", + ) parser.add_argument( "--database", type=Path, @@ -44,8 +58,25 @@ def build_parser() -> argparse.ArgumentParser: return parser -def main(argv: list[str] | None = None) -> int: - args = build_parser().parse_args(argv) +def main(argv: list[str] | None = None, *, shared_service: bool = True) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if "--service" in arguments: + from cli.service_cli import run as service_cli_main + + return service_cli_main([value for value in arguments if value != "--service"]) + if "--serve" in arguments: + from app_server.service import main as serve_main + + return serve_main([value for value in arguments if value != "--serve"]) + if "--web" in arguments: + from cli.web_cli import run as web_main + + return web_main([value for value in arguments if value != "--web"]) + args = build_parser().parse_args(arguments) + if args.managed_config is not None: + from app_server.managed_entry import run as managed_main + + return managed_main(args.managed_config) if args.verify_runtime: from app_server.runtime_probe import verify_runtime @@ -57,6 +88,19 @@ def main(argv: list[str] | None = None) -> int: format="%(asctime)s %(levelname)s %(name)s: %(message)s", ) source, protocol_sink = isolate_protocol_streams() + if shared_service: + from app_server.service_state import ServiceFiles + from app_server.stdio_relay import serve_relay + from core.persistence.database import default_database_path + + return serve_relay( + ServiceFiles(args.database or default_database_path()), + source, + protocol_sink, + ) + from app_server.server import AppServer + from core.application.application import DeepCodeApplication + _trace_startup("opening-application") application = DeepCodeApplication.open( args.database, diff --git a/app_server/blocking_client.py b/app_server/blocking_client.py new file mode 100644 index 000000000..f49a85f11 --- /dev/null +++ b/app_server/blocking_client.py @@ -0,0 +1,175 @@ +"""Synchronous native RPC bridge for existing CLI command handlers.""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import json +import threading +import time + +from app_server.errors import RpcError +from app_server.native_client import NativeRpcClient +from app_server.service_client import ServiceOperationError, ServiceUnavailable +from app_server.service_state import ServiceFiles +from core.application.errors import ( + ApplicationError, + ExpectedTurnMismatchError, + NoActiveTurnError, + TurnAlreadyRunningError, + TurnNotSteerableError, +) + + +class RemoteApplicationError(ApplicationError): + def __init__(self, error: RpcError): + super().__init__(str(error), details=error.data.get("details")) + self.code = error.stable_code + self.retryable = error.data.get("retryable", False) + + +class BlockingServiceClient: + """One owned I/O thread; close releases transport, never service work.""" + + def __init__( + self, files: ServiceFiles, *, surface: str = "cli", start: bool = True + ): + self.files = files + self.surface = surface + self._connection_lock = threading.RLock() + self.generation = 0 + self.loop = asyncio.new_event_loop() + self.thread = threading.Thread( + target=self.loop.run_forever, name="deepcode-cli-rpc", daemon=True + ) + self.thread.start() + self.client = NativeRpcClient(files) + self._closed = False + try: + self.info = self.run( + self.client.connect( + { + "protocolVersion": "1.0", + "clientInfo": { + "name": "deepcode-cli", + "version": "1", + "surface": surface, + }, + }, + start=start, + ) + ) + except BaseException: + self.close() + raise + + def run(self, coroutine, *, timeout: float = 40): + if self._closed: + coroutine.close() + raise RuntimeError("Service client is closed") + future = asyncio.run_coroutine_threadsafe(coroutine, self.loop) + try: + return future.result(timeout=timeout) + except concurrent.futures.TimeoutError: + future.cancel() + raise RuntimeError( + "Service request timed out; inspect its state before retrying" + ) from None + except (ServiceUnavailable, ServiceOperationError) as exc: + raise RemoteApplicationError( + RpcError(-32000, str(exc), stable_code="SERVICE_UNAVAILABLE") + ) from exc + except RpcError as exc: + details = exc.data.get("details") or {} + if exc.stable_code == ExpectedTurnMismatchError.code: + raise ExpectedTurnMismatchError( + details.get("expectedTurnId", "unknown"), + details.get("actualTurnId"), + ) from exc + for error_type in ( + NoActiveTurnError, + TurnAlreadyRunningError, + TurnNotSteerableError, + ): + if exc.stable_code == error_type.code: + raise error_type(str(exc), details=details) from exc + raise RemoteApplicationError(exc) from exc + + def reconnect(self, previous=None): + with self._connection_lock: + if self._closed: + raise RuntimeError("Service client is closed") + if previous is not None and self.client is not previous: + return + old = self.client + self.run(old.close()) + candidate = NativeRpcClient(self.files) + info = self.run( + candidate.connect( + { + "protocolVersion": "1.0", + "clientInfo": { + "name": "deepcode-cli", + "version": "1", + "surface": self.surface, + }, + } + ) + ) + self.client = candidate + self.info = info + self.generation += 1 + + def call(self, method: str, params: dict): + original = json.loads(json.dumps(params)) + policy = self.info.get("capabilities", {}).get("requestRetry", {}) + key = policy.get("keyedMethods", {}).get(method) + safe = method in policy.get("readMethods", []) or ( + key and isinstance(original.get(key), str) and bool(original[key].strip()) + ) + for attempt in range(3): + current = self.client + try: + if current.closed.is_set() and safe: + self.reconnect(current) + latest = self.info.get("capabilities", {}).get("requestRetry", {}) + if attempt and not ( + method in latest.get("readMethods", []) + or (key and latest.get("keyedMethods", {}).get(method) == key) + ): + raise RuntimeError( + "Service retry policy changed; inspect the original operation before retrying" + ) + return self.run( + self.client.request(method, original), + timeout=125 if method == "provider/test" else 40, + ) + except RemoteApplicationError as exc: + if ( + not safe + or exc.code + not in { + "CONNECTION_LOST", + "RESULT_UNKNOWN", + "NOT_CONNECTED", + "INPUT_DELIVERY_PENDING", + } + or attempt == 2 + ): + raise + if exc.code != "INPUT_DELIVERY_PENDING": + self.reconnect(current) + time.sleep(0.1 * (attempt + 1)) + + def close(self): + with self._connection_lock: + if self._closed: + return + try: + self.run(self.client.close()) + finally: + self._closed = True + self.loop.call_soon_threadsafe(self.loop.stop) + self.thread.join(timeout=2) + if not self.thread.is_alive(): + self.loop.close() diff --git a/app_server/browser_auth.py b/app_server/browser_auth.py new file mode 100644 index 000000000..e70389b54 --- /dev/null +++ b/app_server/browser_auth.py @@ -0,0 +1,72 @@ +"""Instance-local browser sessions; all access belongs to the HTTP event loop.""" + +from __future__ import annotations + +import secrets +import time +from collections import deque +from collections.abc import Callable + +from aiohttp import web + + +class BrowserAuth: + TICKET_TTL = 60 + SESSION_TTL = 12 * 60 * 60 + CAPACITY = 64 + EXCHANGES_PER_MINUTE = 60 + + def __init__( + self, instance_id: str, *, clock: Callable[[], float] = time.monotonic + ): + # Cookies are not scoped by port. Instance-specific names avoid collisions + # between separate local databases, and restart invalidates all sessions. + self.cookie_name = f"deepcode_session_{instance_id}" + self._clock = clock + self._tickets: dict[str, float] = {} + self._sessions: dict[str, float] = {} + self._attempts: deque[float] = deque() + + def issue(self) -> dict[str, str | int]: + self._prune(self._tickets) + if len(self._tickets) >= self.CAPACITY: + raise web.HTTPTooManyRequests(text="Too many pending browser links") + ticket = secrets.token_urlsafe(32) + self._tickets[ticket] = self._clock() + self.TICKET_TTL + return {"ticket": ticket, "expiresIn": self.TICKET_TTL} + + def exchange(self, ticket: object) -> str: + now = self._clock() + while self._attempts and self._attempts[0] <= now - 60: + self._attempts.popleft() + if len(self._attempts) >= self.EXCHANGES_PER_MINUTE: + raise web.HTTPTooManyRequests(text="Too many exchange attempts") + self._attempts.append(now) + self._prune(self._tickets) + self._prune(self._sessions) + if not isinstance(ticket, str) or ticket not in self._tickets: + raise web.HTTPUnauthorized(text="Invalid or expired browser link") + if len(self._sessions) >= self.CAPACITY: + raise web.HTTPTooManyRequests(text="Too many browser sessions") + del self._tickets[ticket] + session = secrets.token_urlsafe(32) + self._sessions[session] = now + self.SESSION_TTL + return session + + def remaining(self, session: str) -> float: + return max(0.0, self._sessions.get(session, 0.0) - self._clock()) + + def require(self, request: web.Request) -> str: + session = request.cookies.get(self.cookie_name, "") + if not self.remaining(session): + raise web.HTTPUnauthorized(text="Browser session expired; open a new link") + return session + + def revoke(self, session: str) -> None: + self._sessions.pop(session, None) + + def _prune(self, entries: dict[str, float]) -> None: + now = self._clock() + for key, expiry in tuple(entries.items()): + if expiry <= now: + del entries[key] diff --git a/app_server/config_watch.py b/app_server/config_watch.py index 6247ddfad..ffc93a205 100644 --- a/app_server/config_watch.py +++ b/app_server/config_watch.py @@ -1,4 +1,4 @@ -"""Notify a connection when the user configuration file changes on disk.""" +"""Notify the RPC host when the user configuration file changes on disk.""" from __future__ import annotations @@ -40,7 +40,11 @@ def start(self) -> None: name="deepcode-config-watch", daemon=True, ) - self._thread.start() + try: + self._thread.start() + except BaseException: + self._thread = None + raise def stop(self) -> None: self._stop.set() diff --git a/app_server/connection.py b/app_server/connection.py index bebaa2e7d..8e6aa8467 100644 --- a/app_server/connection.py +++ b/app_server/connection.py @@ -29,6 +29,9 @@ def initialize( self.initialized = True def close(self) -> None: + self.initialized = False + self.client_name = None + self.client_surface = ClientSurface.APP_SERVER if self.subscription_token is not None: self.broker.unsubscribe(self.subscription_token) self.subscription_token = None diff --git a/app_server/dispatcher.py b/app_server/dispatcher.py index fdd31e38c..93d49253a 100644 --- a/app_server/dispatcher.py +++ b/app_server/dispatcher.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from collections.abc import Callable from typing import Any @@ -15,6 +16,7 @@ from app_server.protocol import methods as rpc_methods from app_server.protocol.codec import DEFAULT_MAX_MESSAGE_BYTES from app_server.protocol.models import Request +from app_server.protocol.retry import retry_capabilities from core.agent_presets import METADATA_KEY as PRESET_METADATA_KEY from core.agent_presets import list_agent_presets from core.application.application import DeepCodeApplication @@ -207,10 +209,12 @@ def __init__( connection: ConnectionState, *, max_message_bytes: int = DEFAULT_MAX_MESSAGE_BYTES, + service_info: dict[str, Any] | None = None, ) -> None: self.application = application self.connection = connection self.max_message_bytes = max_message_bytes + self.service_info = service_info self._handlers: dict[str, Handler] = { rpc_methods.INITIALIZE: self._initialize, rpc_methods.SHUTDOWN: self._shutdown, @@ -225,6 +229,10 @@ def __init__( rpc_methods.PROVIDER_UPSERT: self._provider_upsert, rpc_methods.PROVIDER_REMOVE: self._provider_remove, rpc_methods.PROVIDER_TEST: self._provider_test, + rpc_methods.PROVIDER_LOGIN_START: self._provider_login_start, + rpc_methods.PROVIDER_LOGIN_POLL: self._provider_login_poll, + rpc_methods.PROVIDER_LOGIN_CANCEL: self._provider_login_cancel, + rpc_methods.PROVIDER_LOGOUT: self._provider_logout, rpc_methods.PROVIDER_DISCOVER: self._provider_discover, rpc_methods.MODEL_LIST: self._model_list, rpc_methods.PRESET_LIST: self._preset_list, @@ -262,6 +270,11 @@ def __init__( rpc_methods.THREAD_RESUME: self._thread_resume, rpc_methods.THREAD_LIST: self._thread_list, rpc_methods.THREAD_READ: self._thread_read, + rpc_methods.THREAD_EXECUTION_READ: self._thread_execution_read, + rpc_methods.THREAD_CONTEXT_CLEAR: self._thread_context_clear, + rpc_methods.THREAD_CONTEXT_COMPACT: self._thread_context_compact, + rpc_methods.TURN_LIST: self._turn_list, + rpc_methods.MODEL_REASONING: self._model_reasoning, rpc_methods.THREAD_RENAME: self._thread_rename, rpc_methods.THREAD_MODEL: self._thread_model, rpc_methods.THREAD_EXECUTION_UPDATE: self._thread_execution_update, @@ -279,6 +292,7 @@ def __init__( rpc_methods.TURN_ENQUEUE: self._turn_enqueue, rpc_methods.TURN_STEER: self._turn_steer, rpc_methods.TURN_READ: self._turn_read, + rpc_methods.TURN_INPUT_READ: self._turn_input_read, rpc_methods.TURN_INTERRUPT: self._turn_interrupt, rpc_methods.TURN_RETRY: self._turn_retry, rpc_methods.WORKFLOW_START: self._workflow_start, @@ -300,6 +314,8 @@ def __init__( rpc_methods.GIT_WORKTREE_CREATE: self._git_worktree_create, rpc_methods.GIT_WORKTREE_REMOVE: self._git_worktree_remove, rpc_methods.TERMINAL_CREATE: self._terminal_create, + rpc_methods.TERMINAL_LIST: self._terminal_list, + rpc_methods.TERMINAL_READ: self._terminal_read, rpc_methods.TERMINAL_WRITE: self._terminal_write, rpc_methods.TERMINAL_RESIZE: self._terminal_resize, rpc_methods.TERMINAL_CLOSE: self._terminal_close, @@ -348,6 +364,11 @@ def _initialize(self, params: Params) -> dict[str, Any]: self.connection.initialize(str(client_name), client_surface) return { "protocolVersion": PROTOCOL_VERSION, + **( + {"serviceInfo": self.service_info} + if self.service_info is not None + else {} + ), "serverInfo": {"name": "deepcode-app-server", "version": SERVER_VERSION}, "clientInfo": { "name": client_name, @@ -359,6 +380,11 @@ def _initialize(self, params: Params) -> dict[str, Any]: "eventReplay": True, "liveEvents": True, "maxMessageBytes": self.max_message_bytes, + **( + {"requestRetry": retry_capabilities()} + if self.service_info is not None + else {} + ), }, } @@ -466,22 +492,46 @@ def _provider_remove(self, params: Params) -> dict[str, Any]: expected_revision=params.string("expectedRevision", required=False), ) + def _provider_login_start(self, params: Params) -> dict: + params.only("connectionId", "openBrowser") + return self.application.llm.login_start( + str(params.string("connectionId")), + open_browser=params.boolean("openBrowser", default=False), + ) + + def _provider_login_poll(self, params: Params) -> dict: + params.only("flowId") + return self.application.llm.login_poll(str(params.string("flowId"))) + + def _provider_login_cancel(self, params: Params) -> dict: + params.only("flowId") + return self.application.llm.login_cancel(str(params.string("flowId"))) + + def _provider_logout(self, params: Params) -> dict: + params.only("connectionId") + return self.application.llm.logout(str(params.string("connectionId"))) + def _provider_test(self, params: Params) -> dict[str, Any]: - params.only("connectionId", "projectId", "model") + params.only("connectionId", "projectId", "model", "connection", "mode") return self.application.llm.test( str(params.string("connectionId")), project_id=params.string("projectId", required=False), model_id=params.string("model", required=False), + draft=params.object("connection", required=False), + mode=params.string("mode", required=False) or "quick", ) def _provider_discover(self, params: Params) -> dict[str, Any]: - params.only("connectionId", "template", "apiBase", "apiKey", "projectId") + params.only( + "connectionId", "template", "apiBase", "apiKey", "projectId", "connection" + ) return self.application.llm.discover_models( connection_id=params.string("connectionId", required=False), template=params.string("template", required=False), api_base=params.string("apiBase", required=False), api_key=params.string("apiKey", required=False), project_id=params.string("projectId", required=False), + draft=params.object("connection", required=False), ) def _model_list(self, params: Params) -> dict[str, Any]: @@ -927,6 +977,12 @@ def _thread_start(self, params: Params) -> dict[str, Any]: project_id, title=str(params.string("title")), mode=mode, + session_kind="tui" + if self.client_surface is ClientSurface.CLI + else "headless" + if self.client_surface is ClientSurface.HEADLESS + else "desktop", + inherit_default_preset=self.client_surface is not ClientSurface.HEADLESS, connection_id=connection_id, model=model, reasoning_effort=reasoning_effort, @@ -956,6 +1012,63 @@ def _thread_resume(self, params: Params) -> dict[str, Any]: ) return {"thread": thread_view(thread)} + def _thread_execution_read(self, params: Params) -> dict[str, Any]: + params.only("threadId") + thread = self.application.threads.read(str(params.string("threadId"))) + profile = self.application.llm.resolve( + thread.workspace_path, + ExecutionSelection( + connection_id=thread.connection_id, + model_id=thread.model, + reasoning_effort=thread.reasoning_effort, + context_window=thread.context_window, + ), + ) + security = self.application.turns.execution_security_policy.resolve(thread) + return { + "executionProfile": profile.to_dict(), + "securityProfile": security.to_dict(), + } + + def _thread_context_clear(self, params: Params) -> dict[str, Any]: + params.only("threadId") + self.application.turns.clear_live_context(str(params.string("threadId"))) + return {} + + def _thread_context_compact(self, params: Params) -> dict[str, Any]: + params.only("threadId") + return asyncio.run( + self.application.turns.compact_live_context(str(params.string("threadId"))) + ) + + def _turn_list(self, params: Params) -> dict[str, Any]: + params.only("threadId", "limit", "offset", "state") + thread_id = str(params.string("threadId")) + self.application.threads.read(thread_id) + limit = params.integer("limit", default=100, minimum=1, maximum=500) + offset = params.integer("offset", default=0, maximum=1_000_000) + state = params.string("state", required=False) or "all" + if state not in {"all", "active", "executing"}: + raise InvalidParams("state must be all, active, or executing") + turns = self.application.turns.list_for_thread( + thread_id, limit=limit + 1, offset=offset, state=state + ) + return { + "turns": [turn_view(turn) for turn in turns[:limit]], + "hasMore": len(turns) > limit, + } + + def _model_reasoning(self, params: Params) -> dict[str, Any]: + params.only("projectId", "connectionId", "model") + capabilities = self.application.llm.model_reasoning( + str(params.string("connectionId")), + str(params.string("model")), + project_id=params.string("projectId", required=False), + ) + return { + "reasoning": capabilities.to_dict() if capabilities is not None else None + } + def _thread_read(self, params: Params) -> dict[str, Any]: params.only("threadId") thread = self.application.threads.read(str(params.string("threadId"))) @@ -1098,7 +1211,12 @@ def _thread_goal_get(self, params: Params) -> dict[str, Any]: params.only("threadId") thread_id = str(params.string("threadId")) goal = self.application.goals.read(thread_id) - return self._goal_result(thread_id, goal) + return { + **self._goal_result(thread_id, goal), + "executionSettled": ( + goal is None or self.application.goals.execution_settled(goal) + ), + } def _thread_goal_set(self, params: Params) -> dict[str, Any]: params.only( @@ -1181,21 +1299,31 @@ def _thread_goal_pause(self, params: Params) -> dict[str, Any]: return self._goal_result(thread_id, goal) def _thread_goal_resume(self, params: Params) -> dict[str, Any]: - params.only("threadId", "expectedGoalId") + params.only( + "threadId", "expectedGoalId", "connectionId", "model", "reasoningEffort" + ) thread_id = str(params.string("threadId")) goal = self.application.goals.resume( thread_id, expected_goal_id=str(params.string("expectedGoalId")), client_surface=self.client_surface, + connection_id=params.string("connectionId", required=False), + model=params.string("model", required=False), + reasoning_effort=params.string("reasoningEffort", required=False), ) return self._goal_result(thread_id, goal) def _thread_goal_continue(self, params: Params) -> dict[str, Any]: - params.only("threadId", "expectedGoalId") + params.only( + "threadId", "expectedGoalId", "connectionId", "model", "reasoningEffort" + ) result = self.application.goals.continue_goal( str(params.string("threadId")), expected_goal_id=str(params.string("expectedGoalId")), client_surface=self.client_surface, + connection_id=params.string("connectionId", required=False), + model=params.string("model", required=False), + reasoning_effort=params.string("reasoningEffort", required=False), ) return { **self._goal_result(result.goal.thread_id, result.goal), @@ -1296,10 +1424,19 @@ def _turn_steer(self, params: Params) -> dict[str, Any]: return { "messageId": receipt.message_id, "delivery": receipt.delivery, + "deliveryState": "accepted", "duplicate": receipt.duplicate, "turn": turn_view(receipt.turn), } + def _turn_input_read(self, params: Params) -> dict[str, Any]: + params.only("threadId", "messageId") + item = self.application.turns.read_input( + str(params.string("threadId")), + str(params.string("messageId")), + ) + return {"item": item_view(item) if item is not None else None} + def _turn_read(self, params: Params) -> dict[str, Any]: params.only("turnId") return self._turn_snapshot( @@ -1416,18 +1553,20 @@ def _approval_respond(self, params: Params) -> dict[str, Any]: return {"approval": approval_view(approval)} def _event_replay(self, params: Params) -> dict[str, Any]: - params.only("threadId", "after", "limit") + params.only("threadId", "after", "limit", "through") thread_id = str(params.string("threadId")) self.application.threads.read(thread_id) page = self.application.events.replay_page( thread_id, after=params.integer("after", default=0, maximum=2**63 - 1), limit=params.integer("limit", default=500, minimum=1, maximum=1000), + through=params.optional_integer("through", maximum=2**63 - 1), ) return { "events": [event_view(event) for event in page.events], "nextAfter": page.next_after, "hasMore": page.has_more, + "headSequence": page.head_sequence, } def _file_list(self, params: Params) -> dict[str, Any]: @@ -1516,6 +1655,24 @@ def _terminal_create(self, params: Params) -> dict[str, Any]: ) return {"terminal": terminal_info_view(info)} + def _terminal_list(self, params: Params) -> dict[str, Any]: + params.only("threadId") + return { + "terminals": self.application.terminals.list(str(params.string("threadId"))) + } + + def _terminal_read(self, params: Params) -> dict[str, Any]: + params.only("threadId", "terminalId", "offset", "limit", "through") + return self.application.terminals.read( + str(params.string("threadId")), + str(params.string("terminalId")), + offset=params.integer("offset", default=0, maximum=2**53 - 1), + limit=params.integer( + "limit", default=16 * 1024, minimum=4, maximum=64 * 1024 + ), + through=params.optional_integer("through", maximum=2**53 - 1), + ) + def _terminal_write(self, params: Params) -> dict[str, Any]: params.only("threadId", "terminalId", "data") written = self.application.terminals.write( diff --git a/app_server/host.py b/app_server/host.py new file mode 100644 index 000000000..651ed7524 --- /dev/null +++ b/app_server/host.py @@ -0,0 +1,147 @@ +"""Application ownership and shared notifications for RPC transports.""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from contextlib import ExitStack +from typing import Any + +from app_server.config_watch import ConfigFileWatcher +from app_server.peer import RpcPeer +from app_server.protocol import notifications +from app_server.protocol.codec import DEFAULT_MAX_MESSAGE_BYTES +from core.application.application import DeepCodeApplication + + +class ServiceHost: + """Own one application until explicitly closed, even with no clients. + + Transports own their I/O and authentication. A peer's ``shutdown`` request + ends only that peer; the private stdio wrapper closes its host separately. + """ + + def __init__( + self, + application: DeepCodeApplication, + *, + max_message_bytes: int = DEFAULT_MAX_MESSAGE_BYTES, + notification_capacity: int = 256, + ) -> None: + self.application = application + self.max_message_bytes = max_message_bytes + self.notification_capacity = notification_capacity + self._lock = threading.RLock() + self._close_lock = threading.Lock() + self._peers: set[RpcPeer] = set() + self._resources = ExitStack() + self._resources.callback(self._close_application) + self._started = False + self._closed = False + self._application_closed = False + + def __enter__(self) -> ServiceHost: + self.start() + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + def start(self) -> None: + try: + with self._lock: + if self._closed: + raise RuntimeError("Service host is closed") + if self._started: + return + app = self.application + token = app.terminals.subscribe(self._publish) + self._resources.callback(app.terminals.unsubscribe, token) + token = app.skills.subscribe_changes( + lambda project_id: self._publish( + notifications.SKILLS_CHANGED, {"projectId": project_id} + ) + ) + self._resources.callback(app.skills.unsubscribe_changes, token) + token = app.plugins.subscribe_changes( + lambda _discovery: self._publish(notifications.PLUGINS_CHANGED, {}) + ) + self._resources.callback(app.plugins.unsubscribe_changes, token) + token = app.mcp.subscribe_changes( + lambda: self._publish(notifications.MCP_CHANGED, {}) + ) + self._resources.callback(app.mcp.unsubscribe_changes, token) + watcher = ConfigFileWatcher( + app.settings.store, + lambda revision: self._publish( + notifications.SETTINGS_CHANGED, {"configRevision": revision} + ), + ) + self._resources.callback(watcher.stop) + watcher.start() + self._started = True + except BaseException: + self.close() + raise + + def connect( + self, + send: Callable[[bytes], None], + *, + service_info: dict[str, Any] | None = None, + ) -> RpcPeer: + """Attach a transport whose writer accepts complete encoded frames.""" + self.start() + with self._lock: + if self._closed: + raise RuntimeError("Service host is closed") + peer = RpcPeer( + self.application, + send, + on_close=self._forget_peer, + max_message_bytes=self.max_message_bytes, + notification_capacity=self.notification_capacity, + service_info=service_info, + ) + self._peers.add(peer) + try: + peer.start() + except BaseException: + peer.close() + raise + return peer + + def close(self) -> None: + with self._close_lock: + with self._lock: + if self._closed: + peers = () + resources = None + else: + self._closed = True + peers = tuple(self._peers) + self._peers.clear() + resources = self._resources.pop_all() + if resources is None: + if not self._application_closed: + self._close_application() + return + try: + for peer in peers: + peer.close() + finally: + resources.close() + + def _close_application(self) -> None: + self.application.close() + self._application_closed = True + + def _forget_peer(self, peer: RpcPeer) -> None: + with self._lock: + self._peers.discard(peer) + + def _publish(self, method: str, payload: dict[str, Any]) -> None: + with self._lock: + peers = tuple(self._peers) + for peer in peers: + peer.notify(method, payload) diff --git a/app_server/launchd.py b/app_server/launchd.py new file mode 100644 index 000000000..1b183f9e3 --- /dev/null +++ b/app_server/launchd.py @@ -0,0 +1,227 @@ +"""macOS user LaunchAgents for the existing DeepCode service executable.""" + +from __future__ import annotations + +import hashlib +import os +import plistlib +import re +import subprocess +from pathlib import Path +from xml.parsers.expat import ExpatError + +from app_server.service_client import ServiceOperationError +from app_server.service_state import ( + ServiceFiles, + service_command, + service_command_port, + service_environment, + service_working_directory, + shell_only_variables, +) +from core.config import deepcode_home +from core.private_storage import open_existing_private_file, open_private_file + + +class LaunchAgent: + """OS adapter; the service CLI owns management locking and task draining.""" + + name = "LaunchAgent" + + def __init__(self, files: ServiceFiles, *, directory: Path | None = None) -> None: + self.files = files + identity = hashlib.sha256(str(files.database).encode()).hexdigest()[:16] + self.label = f"ai.deepcode.service.{identity}" + self.domain = f"gui/{os.getuid()}" + self.target = f"{self.domain}/{self.label}" + self.path = ( + directory or Path.home() / "Library" / "LaunchAgents" + ) / f"{self.label}.plist" + + @staticmethod + def _run(*args: str) -> subprocess.CompletedProcess: + try: + return subprocess.run( + ["/bin/launchctl", *args], capture_output=True, text=True, timeout=15 + ) + except subprocess.TimeoutExpired as exc: + raise ServiceOperationError( + "launchctl did not finish; inspect service doctor before retrying" + ) from exc + + def available(self) -> bool: + return self._run("print", self.domain).returncode == 0 + + def job(self) -> dict: + result = self._run("print", self.target) + if result.returncode: + if ( + "Could not find service" in result.stderr + or "Could not find domain" in result.stderr + ): + return {"loaded": False, "pid": None} + raise ServiceOperationError( + f"Cannot inspect LaunchAgent: {result.stderr.strip()}" + ) + pid = re.search(r"^\s*pid = (\d+)\s*$", result.stdout, re.MULTILINE) + return {"loaded": True, "pid": int(pid[1]) if pid else None} + + def read(self) -> dict | None: + try: + with os.fdopen(open_existing_private_file(self.path), "rb") as stream: + data = stream.read(65_537) + except FileNotFoundError: + return None + if len(data) > 65_536: + raise ServiceOperationError("LaunchAgent file is too large") + try: + value = plistlib.loads(data) + except (ValueError, plistlib.InvalidFileException, ExpatError) as exc: + raise ServiceOperationError("Invalid DeepCode LaunchAgent plist") from exc + if not isinstance(value, dict) or value.get("Label") != self.label: + raise ServiceOperationError( + "LaunchAgent identity does not match this database" + ) + arguments = value.get("ProgramArguments") + try: + service_command_port(arguments, self.files.database) + except ValueError as exc: + raise ServiceOperationError( + "LaunchAgent command has changed; stop and reinstall it" + ) from exc + if ( + not isinstance(value.get("WorkingDirectory"), str) + or not Path(value["WorkingDirectory"]).is_absolute() + or value.get("KeepAlive") != {"SuccessfulExit": False} + or value.get("RunAtLoad") is not True + or not isinstance(value.get("EnvironmentVariables"), dict) + ): + raise ServiceOperationError( + "LaunchAgent configuration has changed; stop and reinstall it" + ) + return value + + def install(self, *, port: int, path: str | None = None) -> dict: + if not 0 <= port <= 65535: + raise ValueError("port must be between 0 and 65535") + environment = service_environment(path) + command = service_command(self.files, port) + value = { + "Label": self.label, + "ProgramArguments": command, + "WorkingDirectory": str(service_working_directory(command)), + "EnvironmentVariables": environment, + "RunAtLoad": True, + "KeepAlive": {"SuccessfulExit": False}, + "ThrottleInterval": 10, + "ExitTimeOut": 30, + "ProcessType": "Background", + } + previous = self.read() + if previous != value and self.job()["loaded"]: + raise ServiceOperationError( + "Stop the loaded service before updating its LaunchAgent" + ) + if previous != value: + # LaunchAgents is a shared user directory; do not chmod it as though + # DeepCode owned every agent in it. Only our plist is private. + self.path.parent.mkdir(parents=True, exist_ok=True) + temporary = self.path.with_suffix(".plist.tmp") + try: + fd = open_private_file(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with os.fdopen(fd, "wb") as stream: + plistlib.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self.path) + finally: + temporary.unlink(missing_ok=True) + return {"installed": True, "atLogin": True, "path": str(self.path)} + + def start(self, *, port: int | None = None) -> None: + value = self.read() + if value is None: + raise ServiceOperationError("No LaunchAgent installed") + if port is not None and port != service_command_port( + value["ProgramArguments"], self.files.database + ): + raise ServiceOperationError( + "Requested port differs from the installed LaunchAgent; reinstall with the new port" + ) + if not self.available(): + raise ServiceOperationError( + "No macOS desktop login session is available; use deepcode serve --foreground" + ) + for key in ("ProgramArguments", "WorkingDirectory"): + target = value[key][0] if key == "ProgramArguments" else value[key] + if not Path(target).exists(): + raise ServiceOperationError( + f"Installed service path is missing: {target}; reinstall the LaunchAgent" + ) + if self.job()["loaded"]: + args = ("kickstart", self.target) + else: + args = ("bootstrap", self.domain, str(self.path)) + result = self._run(*args) + if result.returncode: + raise ServiceOperationError( + f"Cannot start LaunchAgent: {result.stderr.strip()}" + ) + + def unload(self) -> None: + if not self.job()["loaded"]: + return + result = self._run("bootout", self.target) + if result.returncode and self.job()["loaded"]: + raise ServiceOperationError( + f"Cannot unload LaunchAgent: {result.stderr.strip()}" + ) + + def uninstall(self) -> dict: + if self.job()["loaded"]: + raise ServiceOperationError("Stop the LaunchAgent before uninstalling it") + self.read() # Refuse to remove an unrelated/corrupt entry. + self.path.unlink(missing_ok=True) + return {"installed": False, "atLogin": False, "path": str(self.path)} + + def doctor(self) -> dict: + checks = [] + try: + value = self.read() + except ServiceOperationError as exc: + value = None + checks.append({"name": "configuration", "ok": False, "message": str(exc)}) + job = self.job() + if value: + executable = value["ProgramArguments"][0] + directory = value["WorkingDirectory"] + checks.extend( + [ + { + "name": "executable", + "ok": os.path.isfile(executable) + and os.access(executable, os.X_OK), + "path": executable, + }, + { + "name": "workingDirectory", + "ok": Path(directory).is_dir(), + "path": directory, + }, + { + "name": "runtimeHome", + "ok": value["EnvironmentVariables"].get("DEEPCODE_HOME") + == str(deepcode_home()), + }, + ] + ) + return { + "platform": "macos", + "installed": self.path.exists(), + "atLogin": value is not None, + "path": str(self.path), + "guiSessionAvailable": self.available(), + **job, + "checks": checks, + "shellOnlyVariables": shell_only_variables(), + } diff --git a/app_server/managed_entry.py b/app_server/managed_entry.py new file mode 100644 index 000000000..16c16d260 --- /dev/null +++ b/app_server/managed_entry.py @@ -0,0 +1,49 @@ +"""Read a private supervisor environment and enter the existing service host.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +from core.private_storage import open_existing_private_file + +ENVIRONMENT_KEYS = frozenset({"DEEPCODE_HOME", "DEEPCODE_SESSIONS_DIR", "PATH"}) + + +def read_configuration(path: Path) -> dict: + with os.fdopen(open_existing_private_file(path), "r", encoding="utf-8") as stream: + raw = stream.read(65537) + if len(raw) > 65536: + raise ValueError("Managed service configuration is too large") + value = json.loads(raw) + if ( + not isinstance(value, dict) + or value.get("schemaVersion") != 1 + or not isinstance(value.get("environment"), dict) + or set(value["environment"]) - ENVIRONMENT_KEYS + or any( + not isinstance(item, str) or "\x00" in item + for item in value["environment"].values() + ) + or not isinstance(value.get("database"), str) + or not Path(value["database"]).is_absolute() + or type(value.get("port")) is not int + or not 0 <= value["port"] <= 65535 + ): + raise ValueError("Invalid managed service configuration") + return value + + +def run(path: Path) -> int: + value = read_configuration(path) + from app_server.service_state import ServiceFiles + + if path.absolute().parent != ServiceFiles(Path(value["database"])).directory: + raise ValueError("Supervisor configuration does not belong to this database") + os.environ.update(value["environment"]) + from app_server.service import main + + return main( + ["--database", value["database"], "--port", str(value["port"]), "--log-file"] + ) diff --git a/app_server/native_client.py b/app_server/native_client.py new file mode 100644 index 000000000..244677b6b --- /dev/null +++ b/app_server/native_client.py @@ -0,0 +1,189 @@ +"""Authenticated service RPC for native clients; never owns an application.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Callable +from typing import Any + +import aiohttp + +from app_server.errors import RpcError +from app_server.protocol.codec import DEFAULT_MAX_MESSAGE_BYTES, encode_message +from app_server.service_client import ServiceClient, ServiceUnavailable +from app_server.service_state import ServiceFiles +from core.version import __version__ + + +class NativeRpcClient: + """One connection on one event loop; no implicit mutation replay or fallback.""" + + def __init__( + self, files: ServiceFiles, *, notify: Callable[[dict], None] | None = None + ): + self.files = files + self.notify = notify or (lambda _: None) + self.info: dict[str, Any] | None = None + self._session: aiohttp.ClientSession | None = None + self._socket: aiohttp.ClientWebSocketResponse | None = None + self._reader: asyncio.Task | None = None + self._pending: dict[int, asyncio.Future] = {} + self._serial = 0 + self.closed = asyncio.Event() + + async def connect( + self, initialize: dict, *, start: bool = False, port: int | None = None + ) -> dict: + if self._session is not None: + raise RuntimeError("connection already opened") + if start: + from cli.service_cli import start_service + + await asyncio.to_thread(start_service, self.files, port=port) + # Authenticate the discovered listener before releasing any native secret. + status = await asyncio.to_thread(ServiceClient(self.files).call, "status") + found = self.files.read() + if found is None or found[0].instance_id != status["instanceId"]: + raise ServiceUnavailable("Service changed while connecting; reconnect") + record, token = found + if record.version != __version__: + raise RpcError( + -32003, + "The running service and this client have different versions. " + "Finish active work and upgrade/restart the service explicitly.", + stable_code="SERVICE_VERSION_MISMATCH", + ) + self._session = aiohttp.ClientSession( + trust_env=False, + timeout=aiohttp.ClientTimeout(total=None, sock_connect=5), + ) + try: + self._socket = await self._session.ws_connect( + record.url + "/api/rpc", + headers={ + "Authorization": f"Bearer {token}", + "X-DeepCode-Instance": record.instance_id, + }, + max_msg_size=DEFAULT_MAX_MESSAGE_BYTES, + heartbeat=20, + ) + self._reader = asyncio.create_task(self._read()) + self.info = await self.request("initialize", initialize) + identity = self.info.get("serviceInfo", {}) + if ( + self.info.get("protocolVersion") != "1.0" + or identity.get("instanceId") != record.instance_id + ): + raise ServiceUnavailable( + "Service handshake does not match the authenticated instance" + ) + return self.info + except BaseException: + await self.close() + raise + + async def request( + self, method: str, params: dict, *, timeout: float | None = None + ) -> Any: + socket = self._socket + if socket is None or socket.closed or self.closed.is_set(): + raise RpcError( + -32000, + "Service disconnected; request was not sent", + stable_code="NOT_CONNECTED", + ) + if len(self._pending) >= 64: + raise RpcError( + -32000, "Too many pending service requests", stable_code="CLIENT_BUSY" + ) + self._serial += 1 + request_id = self._serial + encoded = encode_message( + {"jsonrpc": "2.0", "id": request_id, "method": method, "params": params} + ) + if len(encoded) > DEFAULT_MAX_MESSAGE_BYTES: + raise RpcError( + -32600, + "Request exceeds the service message limit", + stable_code="INVALID_REQUEST", + ) + future = asyncio.get_running_loop().create_future() + self._pending[request_id] = future + sent = False + try: + async with asyncio.timeout( + timeout + if timeout is not None + else (120 if method == "provider/test" else 25) + ): + # Once writing starts, failure cannot prove non-admission. + sent = True + await socket.send_str(encoded.decode()) + return await future + except (TimeoutError, aiohttp.ClientError, ConnectionError) as exc: + policy = (self.info or {}).get("capabilities", {}).get("requestRetry", {}) + read_only = ( + method in policy.get("readMethods", []) or method == "initialize" + ) + raise RpcError( + -32000, + "Service response was lost. Reconnect and inspect the current state before retrying.", + stable_code="CONNECTION_LOST" + if read_only or not sent + else "RESULT_UNKNOWN", + data={"retryable": read_only}, + ) from exc + finally: + self._pending.pop(request_id, None) + + async def _read(self) -> None: + try: + async for frame in self._socket: + if frame.type != aiohttp.WSMsgType.TEXT: + break + message = json.loads(frame.data) + if not isinstance(message, dict) or message.get("jsonrpc") != "2.0": + raise ValueError("Invalid service frame") + if "id" in message: + future = self._pending.get(message["id"]) + if future is None or future.done(): + continue + if "error" in message: + error = message["error"] + future.set_exception( + RpcError( + error["code"], + error["message"], + stable_code=error.get("data", {}).get( + "code", "RPC_ERROR" + ), + data=error.get("data"), + ) + ) + elif "result" in message: + future.set_result(message["result"]) + else: + raise ValueError("Invalid service response") + elif isinstance(message.get("method"), str): + self.notify(message) + else: + raise ValueError("Invalid service notification") + finally: + self.closed.set() + for future in self._pending.values(): + if not future.done(): + future.set_exception(ConnectionError("Service connection closed")) + + async def close(self) -> None: + self.closed.set() + if self._socket is not None: + try: + await asyncio.wait_for(self._socket.close(), 3) + except TimeoutError: + pass + if self._reader is not None: + self._reader.cancel() + await asyncio.gather(self._reader, return_exceptions=True) + if self._session is not None: + await self._session.close() diff --git a/app_server/peer.py b/app_server/peer.py new file mode 100644 index 000000000..2dbcf74c9 --- /dev/null +++ b/app_server/peer.py @@ -0,0 +1,311 @@ +"""One RPC client's requests and bounded notifications, independent of its host.""" + +from __future__ import annotations + +import logging +import threading +from collections import deque +from collections.abc import Callable +from typing import Any + +from app_server.connection import ConnectionState +from app_server.dispatcher import Dispatcher +from app_server.errors import RpcError, from_application_error +from app_server.protocol import methods as rpc_methods +from app_server.protocol import notifications as rpc_notifications +from app_server.protocol.codec import ( + DEFAULT_MAX_MESSAGE_BYTES, + decode_request, + encode_message, +) +from app_server.protocol.models import Request, Response, notification +from core.application.application import DeepCodeApplication +from core.application.errors import ApplicationError +from core.application.event_service import DeliveryBatch +from core.application.views import event_view + +logger = logging.getLogger(__name__) + + +class RpcPeer: + """Serialize one client's output; disconnect never closes the application. + + ``send`` belongs to the transport and must either complete or raise on + disconnection. Transport adapters own cancellation of blocked socket/pipe I/O. + Host notifications only enter a bounded queue, never write to a client. + """ + + def __init__( + self, + application: DeepCodeApplication, + send: Callable[[bytes], None], + *, + on_close: Callable[[RpcPeer], None], + max_message_bytes: int = DEFAULT_MAX_MESSAGE_BYTES, + notification_capacity: int = 256, + service_info: dict[str, Any] | None = None, + ) -> None: + if notification_capacity < 1: + raise ValueError("notification_capacity must be positive") + self.application = application + self.max_message_bytes = max_message_bytes + self.connection = ConnectionState(application.broker) + self.dispatcher = Dispatcher( + application, + self.connection, + max_message_bytes=max_message_bytes, + service_info=service_info, + ) + self._send = send + self._on_close = on_close + self._delivery_lock = threading.RLock() + self._queue_lock = threading.Lock() + self._pending: deque[dict[str, Any]] = deque(maxlen=notification_capacity) + self._dropped = 0 + self._stop = threading.Event() + self._pump = threading.Thread( + target=self._pump_events, name="deepcode-event-pump", daemon=True + ) + + @property + def closed(self) -> bool: + return self._stop.is_set() + + def start(self) -> None: + self._pump.start() + + def close(self) -> None: + with self._delivery_lock: + if self.closed: + return + self._stop.set() + with self._queue_lock: + self._pending.clear() + self.connection.close() + self._on_close(self) + if ( + self._pump.ident is not None + and threading.current_thread() is not self._pump + ): + self._pump.join(timeout=1.0) + + def notify(self, method: str, payload: dict[str, Any]) -> None: + with self._queue_lock: + if self.closed or not self.connection.initialized: + return + if len(self._pending) == self._pending.maxlen: + self._dropped += 1 + self._pending.append(notification(method, payload)) + + def receive( + self, raw: bytes, *, before_dispatch: Callable[[Request], None] | None = None + ) -> None: + """Dispatch one complete frame; responses precede its queued events.""" + try: + with self._delivery_lock: + if self.closed or self.connection.shutting_down: + return + request = None + try: + request = decode_request(raw, max_bytes=self.max_message_bytes) + if before_dispatch is not None: + before_dispatch(request) + result = self.dispatcher.dispatch(request) + except ApplicationError as exc: + if request is not None and request.has_id: + self._write_error(request.id, from_application_error(exc)) + except RpcError as exc: + if request is None or request.has_id: + self._write_error( + request.id if request is not None else None, exc + ) + except Exception: + logger.exception("Unhandled App Server request failure") + if request is not None and request.has_id: + self._write_error( + request.id, + RpcError( + -32603, "internal error", stable_code="INTERNAL_ERROR" + ), + ) + else: + if request.has_id: + self._write_response( + Response(id=request.id, result=result).to_dict(), + method=request.method, + ) + self._drain_events() + except (BrokenPipeError, OSError, ValueError): + self.close() + raise + if self.connection.shutting_down: + self.close() + + def reject_oversized_frame(self) -> None: + try: + with self._delivery_lock: + if not self.closed: + self._write_error( + None, + RpcError( + -32600, + "message exceeds the configured size limit", + stable_code="INVALID_REQUEST", + ), + ) + except (BrokenPipeError, OSError, ValueError): + self.close() + raise + + def _pump_events(self) -> None: + try: + while not self.closed: + token = self.connection.subscription_token + if token is None: + self._stop.wait(0.05) + else: + self.application.broker.wait_for_events(token, timeout=0.25) + with self._delivery_lock: + if self.closed: + return + self._drain_events() + except (BrokenPipeError, OSError, ValueError): + logger.debug("App Server client disconnected during event delivery") + self.close() + + def _drain_events(self) -> None: + with self._queue_lock: + pending = tuple(self._pending) + dropped = self._dropped + self._pending.clear() + self._dropped = 0 + if dropped: + self._write_notification( + notification( + rpc_notifications.SERVER_WARNING, + { + "code": "NOTIFICATION_QUEUE_OVERFLOW", + "dropped": dropped, + "replayRequired": True, + }, + ) + ) + for message in pending: + self._write_notification(message) + token = self.connection.subscription_token + if token is not None: + self._write_batch(self.application.broker.drain(token)) + + def _write_batch(self, batch: DeliveryBatch) -> None: + """Write one already-drained live batch while the caller owns the sink lock.""" + if batch.dropped: + self._write( + notification( + rpc_notifications.SERVER_WARNING, + { + "code": "EVENT_QUEUE_OVERFLOW", + "dropped": batch.dropped, + "replayRequired": True, + }, + ), + ) + for event in batch.events: + method = ( + rpc_notifications.THREAD_UPDATED + if event.type.startswith("thread.") + else event.type + ) + self._write_notification(notification(method, event_view(event))) + + def _write_error(self, request_id: Any, error: RpcError) -> None: + self._write( + Response(id=request_id, error=error.payload()).to_dict(), + ) + + def _write_response( + self, + message: dict[str, Any], + *, + method: str | None = None, + ) -> None: + encoded = encode_message(message) + if len(encoded) <= self.max_message_bytes: + self._send(encoded) + return + if method == rpc_methods.EVENT_REPLAY: + replay_page = self._fit_replay_response(message) + if replay_page is not None: + self._send(replay_page) + return + if method == rpc_methods.INITIALIZE: + # A rejected handshake must not leave a subscribed, initialized client. + self.connection.close() + request_id = message.get("id") + error = RpcError( + -32004, + "response exceeds the configured message limit", + stable_code="RESPONSE_TOO_LARGE", + data={"maxMessageBytes": self.max_message_bytes}, + ) + self._write( + Response(id=request_id, error=error.payload()).to_dict(), + ) + + def _fit_replay_response(self, message: dict[str, Any]) -> bytes | None: + """Return the largest replay prefix that fits one transport message.""" + + result = message.get("result") + if not isinstance(result, dict): + return None + events = result.get("events") + if not isinstance(events, list) or not events: + return None + + best: bytes | None = None + low = 1 + high = len(events) + while low <= high: + count = (low + high) // 2 + last_event = events[count - 1] + if not isinstance(last_event, dict): + return None + sequence = last_event.get("sequence") + if isinstance(sequence, bool) or not isinstance(sequence, int): + return None + candidate = { + **message, + "result": { + **result, + "events": events[:count], + "nextAfter": sequence, + "hasMore": True, + }, + } + encoded = encode_message(candidate) + if len(encoded) <= self.max_message_bytes: + best = encoded + low = count + 1 + else: + high = count - 1 + return best + + def _write_notification(self, message: dict[str, Any]) -> None: + encoded = encode_message(message) + if len(encoded) <= self.max_message_bytes: + self._send(encoded) + return + warning = notification( + rpc_notifications.SERVER_WARNING, + { + "code": "NOTIFICATION_TOO_LARGE", + "dropped": 1, + "replayRequired": True, + }, + ) + self._write(warning) + + def _write(self, message: dict[str, Any]) -> None: + encoded = encode_message(message) + if len(encoded) > self.max_message_bytes: + raise ValueError("outgoing message exceeds the configured size limit") + self._send(encoded) diff --git a/app_server/protocol/codec.py b/app_server/protocol/codec.py index 9a12464bb..b217e9b5c 100644 --- a/app_server/protocol/codec.py +++ b/app_server/protocol/codec.py @@ -23,7 +23,8 @@ def decode_request( raise ParseError("message is not valid UTF-8") from exc try: value = json.loads(text) - except json.JSONDecodeError as exc: + except (ValueError, RecursionError) as exc: + # Includes Python's integer-length guard as well as malformed JSON. raise ParseError("invalid JSON") from exc if not isinstance(value, dict) or value.get("jsonrpc") != "2.0": raise InvalidRequest("expected a JSON-RPC 2.0 object") diff --git a/app_server/protocol/methods.py b/app_server/protocol/methods.py index 4690ed586..0f073f961 100644 --- a/app_server/protocol/methods.py +++ b/app_server/protocol/methods.py @@ -12,6 +12,10 @@ PROVIDER_LIST = "provider/list" PROVIDER_UPSERT = "provider/upsert" PROVIDER_REMOVE = "provider/remove" +PROVIDER_LOGIN_START = "provider/login/start" +PROVIDER_LOGIN_POLL = "provider/login/poll" +PROVIDER_LOGIN_CANCEL = "provider/login/cancel" +PROVIDER_LOGOUT = "provider/logout" PROVIDER_TEST = "provider/test" PROVIDER_DISCOVER = "provider/discover" MODEL_LIST = "model/list" @@ -50,6 +54,11 @@ THREAD_RESUME = "thread/resume" THREAD_LIST = "thread/list" THREAD_READ = "thread/read" +THREAD_EXECUTION_READ = "thread/execution/read" +THREAD_CONTEXT_CLEAR = "thread/context/clear" +THREAD_CONTEXT_COMPACT = "thread/context/compact" +TURN_LIST = "turn/list" +MODEL_REASONING = "model/reasoning" THREAD_RENAME = "thread/rename" THREAD_MODEL = "thread/model" THREAD_EXECUTION_UPDATE = "thread/execution/update" @@ -67,6 +76,7 @@ TURN_ENQUEUE = "turn/enqueue" TURN_STEER = "turn/steer" TURN_READ = "turn/read" +TURN_INPUT_READ = "turn/input/read" TURN_INTERRUPT = "turn/interrupt" TURN_RETRY = "turn/retry" WORKFLOW_START = "workflow/start" @@ -88,6 +98,8 @@ GIT_WORKTREE_CREATE = "git/worktree/create" GIT_WORKTREE_REMOVE = "git/worktree/remove" TERMINAL_CREATE = "terminal/create" +TERMINAL_LIST = "terminal/list" +TERMINAL_READ = "terminal/read" TERMINAL_WRITE = "terminal/write" TERMINAL_RESIZE = "terminal/resize" TERMINAL_CLOSE = "terminal/close" diff --git a/app_server/protocol/retry.py b/app_server/protocol/retry.py new file mode 100644 index 000000000..bbb1a7893 --- /dev/null +++ b/app_server/protocol/retry.py @@ -0,0 +1,58 @@ +"""Explicit network retry contract; unlisted operations must not be replayed.""" + +READ_METHODS = frozenset( + { + "project/list", + "project/read", + "settings/read", + "provider/list", + "provider/login/poll", + "model/list", + "preset/list", + "preset/current", + "skills/list", + "skill/read", + "plugins/list", + "hooks/list", + "mcp/list", + "mcp/presets", + "diagnostics/read", + "automation/list", + "automation/runs", + "thread/list", + "thread/read", + "thread/execution/read", + "turn/list", + "model/reasoning", + "thread/goal/get", + "turn/read", + "turn/input/read", + "terminal/list", + "terminal/read", + "workflow/read", + "workflow/list", + "artifact/list", + "artifact/read", + "event/replay", + "file/list", + "file/read", + "git/status", + "git/diff", + "test/discover", + } +) + +KEYED_METHODS = { + "turn/start": "messageId", + "turn/enqueue": "messageId", + "turn/steer": "messageId", + "automation/run": "requestId", +} + + +def retry_capabilities() -> dict: + return { + "default": "never", + "readMethods": sorted(READ_METHODS), + "keyedMethods": dict(KEYED_METHODS), + } diff --git a/app_server/runtime_install.py b/app_server/runtime_install.py new file mode 100644 index 000000000..a40775259 --- /dev/null +++ b/app_server/runtime_install.py @@ -0,0 +1,49 @@ +"""Pin a bundled service outside an updater-owned Desktop installation.""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import sys +import tempfile +from pathlib import Path + +from core.config import deepcode_home +from core.file_lock import exclusive_file_lock +from core.private_storage import ensure_private_directory +from core.version import __version__ + + +def pinned_service_executable() -> Path: + """Publish a complete immutable onedir copy, without modifying older versions.""" + executable = Path(sys.executable).absolute() + source = executable.parent + if not (source / "_internal").is_dir(): + raise RuntimeError("The complete onedir App Server bundle is required") + # The executable includes the Python code; the manifest identifies separate + # frontend resources. Versioned copies survive Desktop updater replacement. + digest = hashlib.sha256(executable.read_bytes()) + manifest = source / "_internal" / "app_server" / "web_assets" / "web-build.json" + digest.update(manifest.read_bytes()) + root = ensure_private_directory(deepcode_home() / "runtimes") + destination = root / f"{__version__}-{digest.hexdigest()[:24]}" + target = destination / executable.name + if source.resolve() == destination.resolve(): + return executable + with exclusive_file_lock(root / "install.lock"): + if target.is_file() and (destination / ".complete").is_file(): + return target + if destination.exists(): + raise RuntimeError( + "Incomplete pinned service installation; inspect the runtime directory" + ) + staging = Path(tempfile.mkdtemp(prefix=".install-", dir=root)) + try: + shutil.copytree(source, staging, dirs_exist_ok=True, symlinks=True) + (staging / ".complete").write_text(digest.hexdigest(), encoding="ascii") + os.rename(staging, destination) + finally: + if staging.exists(): + shutil.rmtree(staging) + return target diff --git a/app_server/runtime_probe.py b/app_server/runtime_probe.py index 5023f5d96..25ced21ae 100644 --- a/app_server/runtime_probe.py +++ b/app_server/runtime_probe.py @@ -60,6 +60,8 @@ def verify_runtime() -> dict[str, Any]: importlib.import_module(module) bundled_skills = _verify_bundled_skills() bundled_mcp_presets = [preset.id for preset in McpPresetCatalog().list()] + from app_server.web_surface import ASSET_DIRECTORY, read_web_build + return { "ok": True, "modules": list(RUNTIME_MODULES), @@ -70,4 +72,7 @@ def verify_runtime() -> dict[str, Any]: "skillCreator": "skill-creator" in bundled_skills, "bundledMcpPresets": bundled_mcp_presets, "version": __version__, + "webAssets": bool( + read_web_build() and (ASSET_DIRECTORY / "index.html").is_file() + ), } diff --git a/app_server/server.py b/app_server/server.py index 67804f28f..42ccc83eb 100644 --- a/app_server/server.py +++ b/app_server/server.py @@ -1,32 +1,37 @@ -"""Single-client stdio JSON-RPC server.""" +"""Private stdio host and stream adapter for the shared RPC implementation.""" from __future__ import annotations -import logging -import threading -from typing import Any, BinaryIO +from typing import BinaryIO -from app_server.config_watch import ConfigFileWatcher -from app_server.connection import ConnectionState -from app_server.dispatcher import Dispatcher -from app_server.errors import RpcError, from_application_error -from app_server.protocol import methods as rpc_methods -from app_server.protocol import notifications as rpc_notifications -from app_server.protocol.codec import ( - DEFAULT_MAX_MESSAGE_BYTES, - decode_request, - encode_message, -) -from app_server.protocol.models import Response, notification +from app_server.host import ServiceHost +from app_server.peer import RpcPeer +from app_server.protocol.codec import DEFAULT_MAX_MESSAGE_BYTES from core.application.application import DeepCodeApplication -from core.application.errors import ApplicationError -from core.application.event_service import DeliveryBatch -from core.application.views import event_view -logger = logging.getLogger(__name__) + +def serve_stdio(peer: RpcPeer, source: BinaryIO) -> int: + """Read one connection until EOF/shutdown; release only that connection.""" + try: + while not peer.closed: + raw = source.readline(peer.max_message_bytes + 1) + if not raw: + break + if len(raw) > peer.max_message_bytes: + chunk = raw + while chunk and not chunk.endswith(b"\n"): + chunk = source.readline(64 * 1024) + peer.reject_oversized_frame() + continue + peer.receive(raw) + finally: + peer.close() + return 0 class AppServer: + """Preserve the dedicated stdio process's EOF/shutdown ownership contract.""" + def __init__( self, application: DeepCodeApplication, @@ -37,309 +42,11 @@ def __init__( self.max_message_bytes = max_message_bytes def serve(self, source: BinaryIO, sink: BinaryIO) -> int: - connection = ConnectionState(self.application.broker) - dispatcher = Dispatcher( - self.application, - connection, - max_message_bytes=self.max_message_bytes, - ) - delivery_lock = threading.RLock() - stop_pump = threading.Event() - - def deliver_terminal(method: str, payload: dict[str, Any]) -> None: - try: - with delivery_lock: - self._write_notification(sink, notification(method, payload)) - except (BrokenPipeError, OSError, ValueError): - stop_pump.set() - - terminal_token = self.application.terminals.subscribe(deliver_terminal) - - def deliver_skill_change(project_id: str) -> None: - try: - with delivery_lock: - self._write_notification( - sink, - notification( - rpc_notifications.SKILLS_CHANGED, - {"projectId": project_id}, - ), - ) - except (BrokenPipeError, OSError, ValueError): - stop_pump.set() - - skill_token = self.application.skills.subscribe_changes(deliver_skill_change) - - def deliver_plugin_change(_discovery) -> None: - try: - with delivery_lock: - self._write_notification( - sink, - notification(rpc_notifications.PLUGINS_CHANGED, {}), - ) - except (BrokenPipeError, OSError, ValueError): - stop_pump.set() - - plugin_token = self.application.plugins.subscribe_changes(deliver_plugin_change) - - def deliver_mcp_change() -> None: - try: - with delivery_lock: - self._write_notification( - sink, - notification(rpc_notifications.MCP_CHANGED, {}), - ) - except (BrokenPipeError, OSError, ValueError): - stop_pump.set() - - mcp_token = self.application.mcp.subscribe_changes(deliver_mcp_change) - - def deliver_settings_change(config_revision: str) -> None: - try: - with delivery_lock: - self._write_notification( - sink, - notification( - rpc_notifications.SETTINGS_CHANGED, - {"configRevision": config_revision}, - ), - ) - except (BrokenPipeError, OSError, ValueError): - stop_pump.set() - - settings_watcher = ConfigFileWatcher( - self.application.settings.store, - deliver_settings_change, - ) - settings_watcher.start() - pump = threading.Thread( - target=self._pump_events, - args=(sink, connection, delivery_lock, stop_pump), - name="deepcode-event-pump", - daemon=True, - ) - pump.start() - try: - while not connection.shutting_down: - request = None - raw = source.readline(self.max_message_bytes + 1) - if not raw: - break - if len(raw) > self.max_message_bytes: - self._discard_remainder(source, raw) - with delivery_lock: - self._write_error( - sink, - None, - RpcError( - -32600, - "message exceeds the configured size limit", - stable_code="INVALID_REQUEST", - ), - ) - continue - with delivery_lock: - try: - request = decode_request(raw, max_bytes=self.max_message_bytes) - result = dispatcher.dispatch(request) - except ApplicationError as exc: - if request is not None and request.has_id: - self._write_error( - sink, request.id, from_application_error(exc) - ) - continue - except RpcError as exc: - request_id = ( - request.id - if request is not None and request.has_id - else None - ) - if request is None or request.has_id: - self._write_error(sink, request_id, exc) - continue - except Exception: - logger.exception("Unhandled App Server request failure") - if request is not None and request.has_id: - self._write_error( - sink, - request.id, - RpcError( - -32603, - "internal error", - stable_code="INTERNAL_ERROR", - ), - ) - continue - - if request.has_id: - self._write_response( - sink, - Response(id=request.id, result=result).to_dict(), - method=request.method, - ) - self._drain_events(sink, connection) - finally: - stop_pump.set() - pump.join(timeout=1.0) - self.application.terminals.unsubscribe(terminal_token) - self.application.skills.unsubscribe_changes(skill_token) - self.application.plugins.unsubscribe_changes(plugin_token) - self.application.mcp.unsubscribe_changes(mcp_token) - settings_watcher.stop() - connection.close() - self.application.close() - return 0 - - def _pump_events( - self, - sink: BinaryIO, - connection: ConnectionState, - delivery_lock: threading.RLock, - stop: threading.Event, - ) -> None: - while not stop.is_set(): - token = connection.subscription_token - if token is None: - stop.wait(0.05) - continue - if not self.application.broker.wait_for_events(token, timeout=0.25): - continue - try: - with delivery_lock: - batch = self.application.broker.drain(token) - self._write_batch(sink, batch) - except (BrokenPipeError, OSError, ValueError): - logger.exception("App Server event delivery failed") - stop.set() - - def _drain_events(self, sink: BinaryIO, connection: ConnectionState) -> None: - token = connection.subscription_token - if token is None: - return - self._write_batch(sink, self.application.broker.drain(token)) - - def _write_batch(self, sink: BinaryIO, batch: DeliveryBatch) -> None: - """Write one already-drained live batch while the caller owns the sink lock.""" - if batch.dropped: - self._write( - sink, - notification( - rpc_notifications.SERVER_WARNING, - { - "code": "EVENT_QUEUE_OVERFLOW", - "dropped": batch.dropped, - "replayRequired": True, - }, - ), - ) - for event in batch.events: - method = ( - rpc_notifications.THREAD_UPDATED - if event.type.startswith("thread.") - else event.type - ) - self._write_notification(sink, notification(method, event_view(event))) - - @staticmethod - def _discard_remainder(source: BinaryIO, first_chunk: bytes) -> None: - chunk = first_chunk - while chunk and not chunk.endswith(b"\n"): - chunk = source.readline(64 * 1024) - - def _write_error(self, sink: BinaryIO, request_id: Any, error: RpcError) -> None: - self._write( - sink, - Response(id=request_id, error=error.payload()).to_dict(), - ) - - def _write_response( - self, - sink: BinaryIO, - message: dict[str, Any], - *, - method: str | None = None, - ) -> None: - encoded = encode_message(message) - if len(encoded) <= self.max_message_bytes: - self._write_encoded(sink, encoded) - return - if method == rpc_methods.EVENT_REPLAY: - replay_page = self._fit_replay_response(message) - if replay_page is not None: - self._write_encoded(sink, replay_page) - return - request_id = message.get("id") - error = RpcError( - -32004, - "response exceeds the configured message limit", - stable_code="RESPONSE_TOO_LARGE", - data={"maxMessageBytes": self.max_message_bytes}, - ) - self._write( - sink, - Response(id=request_id, error=error.payload()).to_dict(), - ) - - def _fit_replay_response(self, message: dict[str, Any]) -> bytes | None: - """Return the largest replay prefix that fits one transport message.""" - - result = message.get("result") - if not isinstance(result, dict): - return None - events = result.get("events") - if not isinstance(events, list) or not events: - return None - - best: bytes | None = None - low = 1 - high = len(events) - while low <= high: - count = (low + high) // 2 - last_event = events[count - 1] - if not isinstance(last_event, dict): - return None - sequence = last_event.get("sequence") - if isinstance(sequence, bool) or not isinstance(sequence, int): - return None - candidate = { - **message, - "result": { - **result, - "events": events[:count], - "nextAfter": sequence, - "hasMore": True, - }, - } - encoded = encode_message(candidate) - if len(encoded) <= self.max_message_bytes: - best = encoded - low = count + 1 - else: - high = count - 1 - return best - - def _write_notification(self, sink: BinaryIO, message: dict[str, Any]) -> None: - encoded = encode_message(message) - if len(encoded) <= self.max_message_bytes: - self._write_encoded(sink, encoded) - return - warning = notification( - rpc_notifications.SERVER_WARNING, - { - "code": "NOTIFICATION_TOO_LARGE", - "dropped": 1, - "replayRequired": True, - }, - ) - self._write(sink, warning) - - def _write(self, sink: BinaryIO, message: dict[str, Any]) -> None: - encoded = encode_message(message) - if len(encoded) > self.max_message_bytes: - raise ValueError("outgoing message exceeds the configured size limit") - self._write_encoded(sink, encoded) - - @staticmethod - def _write_encoded(sink: BinaryIO, encoded: bytes) -> None: - sink.write(encoded) - sink.flush() + def send(encoded: bytes) -> None: + sink.write(encoded) + sink.flush() + + with ServiceHost( + self.application, max_message_bytes=self.max_message_bytes + ) as host: + return serve_stdio(host.connect(send), source) diff --git a/app_server/service.py b/app_server/service.py new file mode 100644 index 000000000..f40300715 --- /dev/null +++ b/app_server/service.py @@ -0,0 +1,459 @@ +"""Loopback management and business listener for one DeepCode application.""" + +from __future__ import annotations + +import argparse +import asyncio +import hmac +import logging +from logging.handlers import RotatingFileHandler +import math +import os +from pathlib import Path +import secrets +import signal +import socket +import threading +import time + +from aiohttp import web + +from app_server.browser_auth import BrowserAuth +from app_server.host import ServiceHost +from app_server.errors import RpcError +from app_server.protocol.codec import decode_request +from app_server.service_client import ServiceClient, ServiceUnavailable +from app_server.service_state import ServiceFiles, ServiceRecord, identity_proof +from app_server.websocket import WebSocketTransport +from app_server.web_surface import WebSurface +from core.application.application import DeepCodeApplication +from core.application.service_lifecycle import ServiceLifecycle +from core.config import LoggerConfig +from core.observability import setup_logging +from core.persistence.database import default_database_path +from core.private_storage import ensure_private_directory, open_private_file + +logger = logging.getLogger(__name__) +MAX_CONTROL_BYTES = 16_384 + + +class _PrivateRotatingLog(RotatingFileHandler): + def _open(self): + descriptor = open_private_file( + Path(self.baseFilename), os.O_WRONLY | os.O_CREAT | os.O_APPEND + ) + return os.fdopen(descriptor, "a", encoding=self.encoding) + + +class ControlServer: + """Compose management and business transports with separate authorization.""" + + def __init__(self, host: ServiceHost, record: ServiceRecord, token: str) -> None: + self.host = host + self.record = record + self._token = token + self.lifecycle = ServiceLifecycle(host.application) + self.stopped = asyncio.Event() + self.interrupt = threading.Event() + self.phase = "ready" + self._operation = asyncio.Lock() + self.browser_auth = BrowserAuth(record.instance_id) + self.web_surface = WebSurface( + host.application, self.browser_auth, lambda: self.phase + ) + self.business = WebSocketTransport( + host, + self.browser_auth, + native_authenticated=self._authenticated, + phase=lambda: self.phase, + service_info={ + "instanceId": record.instance_id, + "schemaVersion": host.application.database.schema_version(), + "transport": "websocket", + "shutdownScope": "connection", + "frontendBuildId": (self.web_surface.build() or {}).get("buildId"), + }, + ) + + def application(self) -> web.Application: + app = web.Application( + client_max_size=MAX_CONTROL_BYTES, middlewares=[self._local_only] + ) + app.add_routes( + [ + web.get("/health/live", self._health), + web.get("/health/ready", self._health), + web.get("/control/identity", self._identity), + web.post("/control/rpc", self._rpc), + web.post("/auth/exchange", self._exchange), + web.post("/auth/logout", self._logout), + web.get("/api/rpc", self.business.handle), + ] + ) + app.add_routes(self.web_surface.routes()) + app.on_shutdown.append(self.business.shutdown) + app.on_cleanup.append(self.business.cleanup) + app.on_response_prepare.append(self._private_response) + return app + + @web.middleware + async def _local_only(self, request: web.Request, handler): + if request.headers.getall("Host", []) != [f"127.0.0.1:{self.record.port}"]: + raise web.HTTPForbidden(text="Local service management only") + origins = request.headers.getall("Origin", []) + browser_route = request.path.startswith( + ("/auth/", "/api/", "/assets/") + ) or request.path in {"/", "/index.html", "/web-build.json"} + if origins and (not browser_route or origins != [self.record.url]): + raise web.HTTPForbidden(text="Invalid browser origin") + if request.path.startswith("/auth/") and not origins: + raise web.HTTPForbidden(text="Browser origin required") + if ( + request.path.startswith("/api/") + and request.method not in {"GET", "HEAD"} + and origins != [self.record.url] + ): + raise web.HTTPForbidden(text="Browser origin required") + return await handler(request) + + async def _private_response(self, _request, response) -> None: + response.headers["Cache-Control"] = "no-store" + response.headers["Referrer-Policy"] = "no-referrer" + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["Content-Security-Policy"] = ( + f"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self' data:; worker-src 'self' blob:; connect-src 'self' ws://127.0.0.1:{self.record.port}; object-src 'none'; base-uri 'none'; frame-ancestors 'none'" + ) + + async def _exchange(self, request: web.Request) -> web.Response: + if self.phase != "ready": + raise web.HTTPServiceUnavailable(text="Service is not ready") + if request.content_type != "application/json": + raise web.HTTPUnsupportedMediaType(text="JSON body required") + try: + body = await request.json() + except (ValueError, RecursionError): + raise web.HTTPBadRequest(text="Invalid exchange body") from None + if not isinstance(body, dict) or set(body) != {"ticket"}: + raise web.HTTPBadRequest(text="Expected a browser ticket") + session = self.browser_auth.exchange(body["ticket"]) + response = web.json_response( + {"authenticated": True, "instanceId": self.record.instance_id} + ) + response.set_cookie( + self.browser_auth.cookie_name, + session, + httponly=True, + samesite="Strict", + max_age=self.browser_auth.SESSION_TTL, + path="/", + ) + return response + + async def _logout(self, request: web.Request) -> web.Response: + session = self.browser_auth.require(request) + await self.business.revoke(session) + response = web.json_response({"authenticated": False}) + response.del_cookie(self.browser_auth.cookie_name, path="/") + return response + + async def _health(self, request: web.Request) -> web.Response: + ready = self.phase == "ready" + status = 200 if request.path == "/health/live" or ready else 503 + return web.json_response({"status": self.phase}, status=status) + + async def _identity(self, request: web.Request) -> web.Response: + challenge = request.query.get("challenge", "") + if len(challenge) != 32 or any( + char not in "0123456789abcdef" for char in challenge + ): + raise web.HTTPBadRequest(text="Invalid identity challenge") + return web.json_response( + {"proof": identity_proof(self._token, self.record.instance_id, challenge)} + ) + + async def status(self) -> dict: + return { + "instanceId": self.record.instance_id, + "phase": self.phase, + "pid": self.record.pid, + "url": self.record.url, + "version": self.record.version, + "protocolVersion": self.record.protocol_version, + **await asyncio.to_thread(self.lifecycle.activity), + } + + def _authenticated(self, request: web.Request) -> bool: + return ( + hmac.compare_digest( + request.headers.get("Authorization", "").encode(), + f"Bearer {self._token}".encode(), + ) + and request.headers.get("X-DeepCode-Instance") == self.record.instance_id + ) + + async def _rpc(self, request: web.Request) -> web.Response: + if not self._authenticated(request): + raise web.HTTPUnauthorized(text="Invalid service credential") + try: + rpc = decode_request(await request.read(), max_bytes=MAX_CONTROL_BYTES) + except RpcError as exc: + return web.json_response( + {"jsonrpc": "2.0", "id": None, "error": exc.payload()}, status=400 + ) + if not rpc.has_id: + raise web.HTTPBadRequest(text="Service control requests require an id") + request_id = rpc.id + params = rpc.params + method = rpc.method + if method == "auth/issue" and not params: + if self.phase != "ready": + return self._error(request_id, "Service is not ready") + return web.json_response( + { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "instanceId": self.record.instance_id, + **self.browser_auth.issue(), + }, + } + ) + if method == "status" and not params: + return web.json_response( + {"jsonrpc": "2.0", "id": request_id, "result": await self.status()} + ) + if method == "resume" and not params: + if self.phase != "drained" or self._operation.locked(): + return self._error( + request_id, "Service is not awaiting supervisor stop" + ) + async with self._operation: + await asyncio.to_thread(self.lifecycle.resume) + self.phase = "ready" + return web.json_response( + { + "jsonrpc": "2.0", + "id": request_id, + "result": {"instanceId": self.record.instance_id, "accepted": True}, + } + ) + if method not in {"stop", "drain"}: + return self._error(request_id, "Unknown service operation") + timeout = params.get("timeout", 60.0) + cancel = params.get("cancelRunning", False) + if ( + set(params) - {"timeout", "cancelRunning"} + or type(cancel) is not bool + or (method == "drain" and cancel) + or type(timeout) not in (int, float) + or not 0 <= timeout <= 300 + or not math.isfinite(timeout) + ): + return self._error(request_id, "Invalid stop options") + if self._operation.locked() or self.phase not in {"ready", "drained"}: + return self._error( + request_id, "Another service stop is already in progress" + ) + async with self._operation: + previous_phase = self.phase + self.phase = "draining" + deadline = time.monotonic() + timeout + try: + if not await self.business.wait_idle(timeout): + return self._error( + request_id, "RPC drain timed out; service is still running" + ) + if not cancel and not await asyncio.to_thread( + self.lifecycle.drain, + max(0.0, deadline - time.monotonic()), + self.interrupt, + ): + return self._error( + request_id, + "Drain timed out; service is still running. Finish active work or explicitly cancel it.", + ) + self.phase = "drained" if method == "drain" else "stopping" + response = web.json_response( + { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "instanceId": self.record.instance_id, + "accepted": True, + }, + } + ) + if method == "drain": + return response + try: + await response.prepare(request) + await response.write_eof() + finally: + self.stopped.set() + return response + finally: + if self.phase == "draining": + self.phase = previous_phase + + @staticmethod + def _error(request_id: int | str | None, message: str) -> web.Response: + return web.json_response( + { + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": -32000, "message": message}, + } + ) + + +async def serve(files: ServiceFiles, port: int) -> None: + lease = files.acquire() + # A status probe briefly takes this lock too. Distinguish it from a real + # lifetime owner without waiting behind another running service forever. + for _ in range(10): + if lease is not None: + break + await asyncio.sleep(0.02) + lease = files.acquire() + if lease is None: + # Crash records survive their owner. A successful exit suppresses native + # supervisor recovery, so only an authenticated live owner permits it. + try: + await asyncio.to_thread(ServiceClient(files).call, "status", timeout=1) + except ServiceUnavailable as exc: + raise ServiceUnavailable( + "Service ownership is busy but no live owner could be verified" + ) from exc + logger.info("A DeepCode service already owns this database") + return + host = None + runner = None + control = None + installed_signals = [] + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + loop = asyncio.get_running_loop() + try: + files.clear() + if os.name != "nt": + # Rebind after our old TCP connections enter TIME_WAIT. Do not use + # SO_REUSEPORT or Windows address sharing with another live listener. + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + listener.bind(("127.0.0.1", port)) + except OSError as exc: + logger.error("Cannot listen on 127.0.0.1:%s: %s", port, exc.strerror) + raise + listener.listen(socket.SOMAXCONN) + listener.setblocking(False) + record = ServiceRecord( + secrets.token_hex(16), + str(files.database), + os.getpid(), + listener.getsockname()[1], + ) + token = secrets.token_hex(32) + application = await asyncio.to_thread( + DeepCodeApplication.open, + files.database, + host_surface="service", + run_automation_scheduler=True, + ) + host = ServiceHost(application) + from core.private_storage import atomic_write_private_json + + atomic_write_private_json( + files.directory / "state-layout.json", + { + "schemaVersion": 1, + "database": str(files.database), + "sessions": str(application.session_store.root), + "config": str(application.llm.config_store.path), + "credentials": str(application.credentials.path), + "revisions": str( + application.credentials.path.parent / "provider_revisions" + ), + }, + ) + await asyncio.to_thread(host.start) + control = ControlServer(host, record, token) + runner = web.AppRunner( + control.application(), access_log=None, shutdown_timeout=10 + ) + await runner.setup() + await web.SockSite(runner, listener).start() + + def stop() -> None: + control.interrupt.set() + control.stopped.set() + + for sig in (signal.SIGINT, signal.SIGTERM): + try: + loop.add_signal_handler(sig, stop) + except NotImplementedError: + # Windows asyncio uses the process's ordinary signal path. + continue + installed_signals.append(sig) + files.publish(record, token) + logger.info("Service ready at %s (pid %d)", record.url, record.pid) + await control.stopped.wait() + finally: + if control is not None: + control.interrupt.set() + control.phase = "stopping" + try: + if runner is not None: + await runner.cleanup() + finally: + try: + if host is not None: + await asyncio.to_thread(host.close) + files.clear() + lease.close() + finally: + listener.close() + for sig in installed_signals: + loop.remove_signal_handler(sig) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Run the DeepCode service in the foreground" + ) + parser.add_argument("--database", type=Path, default=None) + parser.add_argument("--port", type=int, default=3081) + parser.add_argument( + "--foreground", + action="store_true", + help="Keep the service attached to this terminal (default)", + ) + parser.add_argument("--log-file", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args(argv) + if not 0 <= args.port <= 65535: + parser.error("port must be between 0 and 65535") + files = ServiceFiles(args.database or default_database_path()) + if args.log_file: + ensure_private_directory(files.directory) + handler = _PrivateRotatingLog( + files.log, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8" + ) + else: + handler = None + setup_logging( + LoggerConfig( + level=os.environ.get("DEEPCODE_LOG_LEVEL", "INFO"), transports=["console"] + ), + force=True, + console_sink=handler, + ) + try: + asyncio.run(serve(files, args.port)) + except KeyboardInterrupt: + return 130 + except Exception as exc: + logger.exception("DeepCode service failed (%s): %s", type(exc).__name__, exc) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app_server/service_client.py b/app_server/service_client.py new file mode 100644 index 000000000..064f83c58 --- /dev/null +++ b/app_server/service_client.py @@ -0,0 +1,100 @@ +"""Authenticated local management client; never owns an application runtime.""" + +from __future__ import annotations + +import hmac +import secrets +from typing import Any + +import httpx + +from app_server.service_state import ServiceFiles, identity_proof + + +class ServiceUnavailable(RuntimeError): + pass + + +class ServiceOperationError(RuntimeError): + pass + + +class ServiceClient: + def __init__(self, files: ServiceFiles) -> None: + self.files = files + + def call( + self, + method: str, + params: dict[str, Any] | None = None, + *, + timeout: float = 5.0, + instance_id: str | None = None, + ) -> dict[str, Any]: + if not self.files.running(): + raise ServiceUnavailable("DeepCode service is not running") + discovered = self.files.read() + if discovered is None: + raise ServiceUnavailable("DeepCode service is starting or stopping") + record, token = discovered + if instance_id is not None and record.instance_id != instance_id: + raise ServiceUnavailable("Service instance changed before the operation") + challenge = secrets.token_hex(16) + try: + # The lock and challenge prevent sending a stale token to another + # process that has taken the old port. Ignore ambient HTTP proxies. + with httpx.Client( + base_url=record.url, trust_env=False, timeout=timeout + ) as client: + response = client.get( + "/control/identity", params={"challenge": challenge} + ) + response.raise_for_status() + identity = response.json() + proof = identity.get("proof") if isinstance(identity, dict) else None + if not isinstance(proof, str) or not hmac.compare_digest( + proof.encode(), + identity_proof(token, record.instance_id, challenge).encode(), + ): + raise ServiceUnavailable( + "Service identity does not match its private record" + ) + response = client.post( + "/control/rpc", + headers={ + "Authorization": f"Bearer {token}", + "X-DeepCode-Instance": record.instance_id, + }, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": method, + "params": params or {}, + }, + ) + response.raise_for_status() + value = response.json() + except (httpx.HTTPError, ValueError) as exc: + # Keep diagnostics actionable without echoing request headers/tokens. + detail = ( + f"HTTP {exc.response.status_code}" + if isinstance(exc, httpx.HTTPStatusError) + else type(exc).__name__ + ) + raise ServiceUnavailable( + f"Cannot communicate with the local DeepCode service ({detail}); check service logs" + ) from exc + if not isinstance(value, dict) or value.get("id") != 1: + raise ServiceUnavailable("Invalid service response") + if "error" in value: + error = value["error"] + if not isinstance(error, dict) or not isinstance(error.get("message"), str): + raise ServiceUnavailable("Invalid service error response") + raise ServiceOperationError(error["message"]) + result = value.get("result") + if ( + not isinstance(result, dict) + or result.get("instanceId") != record.instance_id + ): + raise ServiceUnavailable("Service instance changed during the request") + return result diff --git a/app_server/service_state.py b/app_server/service_state.py new file mode 100644 index 000000000..ba6458c51 --- /dev/null +++ b/app_server/service_state.py @@ -0,0 +1,205 @@ +"""Private discovery records and the managed service's OS-backed lifetime lease.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import secrets +import sys +from dataclasses import asdict, dataclass +from pathlib import Path + +from core.file_lock import FileLease, exclusive_file_lock +from core.private_storage import open_existing_private_file, open_private_file +from core.version import __version__ + +SERVICE_PROTOCOL_VERSION = 1 +SERVICE_WORKING_DIRECTORY = Path(__file__).resolve().parents[1] + + +def service_command(files: ServiceFiles, port: int) -> list[str]: + # Keep the venv executable path; resolving its symlink loses the venv. + if getattr(sys, "frozen", False): + from app_server.runtime_install import pinned_service_executable + + launcher = [str(pinned_service_executable()), "--serve"] + else: + launcher = [os.path.abspath(sys.executable), "-m", "app_server.service"] + return [ + *launcher, + "--database", + str(files.database), + "--port", + str(port), + "--log-file", + ] + + +def identity_proof(token: str, instance_id: str, challenge: str) -> str: + return hmac.new( + token.encode(), f"{instance_id}:{challenge}".encode(), hashlib.sha256 + ).hexdigest() + + +@dataclass(frozen=True) +class ServiceRecord: + instance_id: str + database: str + pid: int + port: int + version: str = __version__ + protocol_version: int = SERVICE_PROTOCOL_VERSION + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + +class ServiceFiles: + def __init__(self, database: Path) -> None: + self.database = database.expanduser().resolve() + self.directory = self.database.with_name(self.database.name + ".service") + self.lock = self.directory / "instance.lock" + self._discovery_lock = self.directory / "discovery.lock" + self.record = self.directory / "instance.json" + self.token = self.directory / "token" + self.log = self.directory / "service.log" + + def acquire(self) -> FileLease | None: + return FileLease.acquire(self.lock, shared=False, blocking=False) + + def running(self) -> bool: + lease = self.acquire() + if lease is None: + return True + lease.close() + return False + + def read(self) -> tuple[ServiceRecord, str] | None: + # The lifetime lease identifies the owner; it does not exclude readers. + # Keep the record/token pair consistent and prevent Windows readers from + # opening files while the owner deletes or replaces them. + with exclusive_file_lock(self._discovery_lock): + try: + value = json.loads(_read(self.record)) + token = _read(self.token).strip() + except FileNotFoundError: + return None + if not isinstance(value, dict): + raise ValueError("Invalid service discovery record") + try: + record = ServiceRecord(**value) + except TypeError as exc: + raise ValueError("Invalid service discovery record") from exc + if ( + record.database != str(self.database) + or type(record.pid) is not int + or record.pid < 1 + or type(record.port) is not int + or not 1 <= record.port <= 65535 + or type(record.protocol_version) is not int + or record.protocol_version != SERVICE_PROTOCOL_VERSION + or not isinstance(record.version, str) + or not isinstance(record.instance_id, str) + or len(record.instance_id) != 32 + or any(char not in "0123456789abcdef" for char in record.instance_id) + or len(token) != 64 + or any(char not in "0123456789abcdef" for char in token) + ): + raise ValueError("Invalid or incompatible service discovery record") + return record, token + + def publish(self, record: ServiceRecord, token: str) -> None: + with exclusive_file_lock(self._discovery_lock): + _write(self.token, token) + _write(self.record, json.dumps(asdict(record))) + + def clear(self) -> None: + """Only the exclusive lease holder may remove these records.""" + with exclusive_file_lock(self._discovery_lock): + self.record.unlink(missing_ok=True) + self.token.unlink(missing_ok=True) + + +def _read(path: Path) -> str: + with os.fdopen(open_existing_private_file(path), "r", encoding="utf-8") as stream: + data = stream.read(16_385) + if len(data) > 16_384: + raise ValueError("Service discovery record is too large") + return data + + +def _write(path: Path, content: str) -> None: + temporary = path.with_name(f".{path.name}.{secrets.token_hex(8)}") + try: + fd = open_private_file(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def service_command_port(arguments: list[str], database: Path) -> int: + """Validate either source or frozen launch syntax without executing it.""" + if not isinstance(arguments, list) or not all( + isinstance(value, str) for value in arguments + ): + raise ValueError("Invalid service arguments") + if arguments[1:3] == ["-m", "app_server.service"]: + tail = arguments[3:] + elif arguments[1:2] == ["--serve"]: + tail = arguments[2:] + else: + raise ValueError("Unsupported service launcher") + if ( + len(tail) != 5 + or tail[:2] != ["--database", str(database)] + or tail[2] != "--port" + or tail[4] != "--log-file" + or not tail[3].isdigit() + or not 0 <= int(tail[3]) <= 65535 + or not Path(arguments[0]).is_absolute() + ): + raise ValueError("Invalid service launch identity") + return int(tail[3]) + + +def service_working_directory(arguments: list[str]) -> Path: + return ( + Path(arguments[0]).parent + if arguments[1:2] == ["--serve"] + else SERVICE_WORKING_DIRECTORY + ) + + +def service_environment(path: str | None = None) -> dict[str, str]: + """Capture only the deliberate launch environment, never shell credentials.""" + from core.config import deepcode_home + + environment = { + "DEEPCODE_HOME": str(deepcode_home()), + "PATH": path if path is not None else os.environ.get("PATH", os.defpath), + } + if os.environ.get("DEEPCODE_SESSIONS_DIR"): + environment["DEEPCODE_SESSIONS_DIR"] = str( + Path(os.environ["DEEPCODE_SESSIONS_DIR"]).expanduser().resolve() + ) + return environment + + +def shell_only_variables() -> list[str]: + from core.providers.registry import PROVIDERS + + names = {provider.env_key for provider in PROVIDERS if provider.env_key} + names.update( + key + for key in os.environ + if key.endswith("_API_KEY") + or key.lower() in {"http_proxy", "https_proxy", "all_proxy"} + ) + return sorted(name for name in names if os.environ.get(name)) diff --git a/app_server/state_backup.py b/app_server/state_backup.py new file mode 100644 index 000000000..e267637e1 --- /dev/null +++ b/app_server/state_backup.py @@ -0,0 +1,446 @@ +"""Offline, verifiable snapshots of the database and canonical runtime state. + +Locks reuse existing application, Session and credential mutation boundaries. +A restore journal blocks application startup until an interrupted restore is +resumed. Project working trees and installed executables are not restored. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import sqlite3 +import tempfile +from contextlib import ExitStack, closing, contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +from app_server.service_state import ServiceFiles +from core.config import deepcode_home, home_config_path +from core.file_lock import FileLease +from core.persistence.database import Database +from core.private_storage import ( + atomic_write_private_json, + ensure_private_directory, + open_existing_private_file, + open_private_file, +) +from core.version import __version__ + +SESSION_INTERNAL = {".locks", ".running", ".activity", ".store.lock"} + + +@dataclass(frozen=True) +class StatePaths: + database: Path + sessions: Path + config: Path + credentials: Path + revisions: Path + + @classmethod + def current(cls, files: ServiceFiles, *, sessions: Path | None = None): + layout = files.directory / "state-layout.json" + if layout.exists(): + with os.fdopen( + open_existing_private_file(layout), "r", encoding="utf-8" + ) as stream: + value = json.loads(stream.read(16385)) + if ( + not isinstance(value, dict) + or value.get("schemaVersion") != 1 + or value.get("database") != str(files.database) + ): + raise ValueError("Invalid saved service data layout") + fields = { + name: Path(value[name]) + for name in ( + "database", + "sessions", + "config", + "credentials", + "revisions", + ) + } + if not all(path.is_absolute() for path in fields.values()): + raise ValueError("Service data paths must be absolute") + if ( + sessions is not None + and sessions.expanduser().resolve() != fields["sessions"] + ): + raise ValueError( + "The requested Session directory differs from the recorded service layout" + ) + return cls(**fields) + home = deepcode_home().resolve() + return cls( + files.database, + ( + sessions + or Path( + os.environ.get("DEEPCODE_SESSIONS_DIR") or str(home / "sessions") + ) + ) + .expanduser() + .resolve(), + home_config_path().resolve(), + home / "credentials.json", + home / "provider_revisions", + ) + + def targets(self) -> dict[str, Path]: + return { + "database.sqlite3": self.database, + "sessions": self.sessions, + "config.json": self.config, + "credentials.json": self.credentials, + "revisions": self.revisions, + "mcp-credentials.json": self.credentials.parent / "auth" / "mcp.json", + } + + +def _excluded(relative: Path) -> bool: + parts = relative.parts + if parts[:1] == ("sessions",) and len(parts) > 1: + return ( + parts[1] in SESSION_INTERNAL + or parts[1] == "index.db" + or parts[1].startswith("index.db-") + ) + return parts == ("revisions", "write.lock") + + +def _files(root: Path, *, prefix=Path()): + if root.is_symlink(): + raise ValueError("State snapshots do not follow symlinks") + if not root.exists(): + return + if root.is_file(): + yield prefix, root + return + if not root.is_dir(): + raise ValueError("State snapshots require regular files and directories") + for child in sorted(root.iterdir()): + relative = prefix / child.name + if not _excluded(relative): + yield from _files(child, prefix=relative) + + +def _digest(path: Path) -> dict: + digest = hashlib.sha256() + size = 0 + with os.fdopen(open_existing_private_file(path), "rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + return {"bytes": size, "sha256": digest.hexdigest()} + + +def _copy_file(source: Path, target: Path) -> None: + with os.fdopen(open_existing_private_file(source), "rb") as reader: + with os.fdopen( + open_private_file(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL), "wb" + ) as writer: + shutil.copyfileobj(reader, writer, length=1024 * 1024) + writer.flush() + os.fsync(writer.fileno()) + + +def _sync_directory(directory: Path): + if os.name != "nt": + fd = os.open(directory, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _sync_tree(directory: Path): + for child in directory.iterdir(): + if child.is_dir() and not child.is_symlink(): + _sync_tree(child) + _sync_directory(directory) + + +@contextmanager +def _offline(paths: StatePaths, *, extra_sessions=()): + files = ServiceFiles(paths.database) + with ExitStack() as stack: + + def lock(path): + lease = FileLease.acquire(path, shared=False, blocking=False) + if lease is None: + raise ValueError( + "State is in use. Stop DeepCode services, CLI/TUI and other writers before backing up or restoring." + ) + stack.enter_context(lease) + + lock(files.directory / "management.lock") + lock(files.lock) + lock(paths.database.with_name(paths.database.name + ".application.lock")) + lock(paths.database.with_name(paths.database.name + ".migration.lock")) + lock(paths.config.with_suffix(paths.config.suffix + ".lock")) + lock(paths.credentials.with_suffix(paths.credentials.suffix + ".lock")) + mcp_credentials = paths.targets()["mcp-credentials.json"] + lock(mcp_credentials.with_suffix(mcp_credentials.suffix + ".lock")) + lock(paths.revisions / "write.lock") + lock(paths.sessions / ".store.lock") + ids = set(extra_sessions) | { + item.name + for item in paths.sessions.iterdir() + if item.is_dir() and not item.name.startswith(".") + } + for identity in sorted(ids): + if Path(identity).name != identity or identity in {"", ".", ".."}: + raise ValueError("Invalid Session identity in snapshot") + for directory in (".activity", ".running", ".locks"): + lock(paths.sessions / directory / f"{identity}.lock") + yield + + +def _snapshot( + paths: StatePaths, destination: Path, *, require_idle: bool = True +) -> dict: + if destination.exists(): + raise ValueError("Snapshot destination already exists") + for name, source in paths.targets().items(): + if destination == source or ( + name in {"sessions", "revisions"} and destination.is_relative_to(source) + ): + raise ValueError("Snapshot destination overlaps runtime data") + ensure_private_directory(destination.parent) + staging = Path(tempfile.mkdtemp(prefix=".snapshot-", dir=destination.parent)) + try: + present = [] + for name, source in paths.targets().items(): + if not source.exists(): + continue + present.append(name) + if name == "database.sqlite3": + target = staging / name + os.close( + open_private_file(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + ) + with ( + closing( + sqlite3.connect(source.as_uri() + "?mode=ro", uri=True) + ) as reader, + closing(sqlite3.connect(target)) as writer, + ): + if require_idle: + _require_idle_database(reader) + reader.backup(writer) + writer.execute("PRAGMA journal_mode=DELETE") + if writer.execute("PRAGMA quick_check").fetchone()[0] != "ok": + raise ValueError("Snapshot database integrity check failed") + with target.open("rb+") as stream: + os.fsync(stream.fileno()) + else: + for relative, original in _files(source, prefix=Path(name)): + _copy_file(original, staging / relative) + inventory = {str(relative): _digest(path) for relative, path in _files(staging)} + manifest = { + "schemaVersion": 1, + "runtimeVersion": __version__, + "createdAt": datetime.now(UTC).isoformat(), + "paths": {name: str(path) for name, path in paths.targets().items()}, + "present": present, + "files": inventory, + } + atomic_write_private_json(staging / "manifest.json", manifest) + _sync_tree(staging) + os.rename(staging, destination) + _sync_directory(destination.parent) + return { + "snapshot": str(destination), + "fileCount": len(inventory), + "runtimeVersion": __version__, + "paths": manifest["paths"], + } + finally: + if staging.exists(): + shutil.rmtree(staging) + + +def create_snapshot(paths: StatePaths, destination: Path) -> dict: + with _offline(paths): + if Database(paths.database).restore_marker.exists(): + raise ValueError( + "Resume the pending restore before making another snapshot" + ) + return _snapshot(paths, destination.expanduser().absolute()) + + +def _require_idle_database(connection): + if ( + connection.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='turns'" + ).fetchone() + and connection.execute( + "SELECT 1 FROM turns WHERE status IN ('queued', 'running', 'waiting_approval') LIMIT 1" + ).fetchone() + ): + raise ValueError( + "Runtime snapshots require all Turns to be settled. Drain or explicitly cancel pending work first." + ) + + +def _manifest(snapshot: Path, paths: StatePaths) -> dict: + with os.fdopen( + open_existing_private_file(snapshot / "manifest.json"), "r", encoding="utf-8" + ) as stream: + raw = stream.read(8 * 1024 * 1024 + 1) + if len(raw) > 8 * 1024 * 1024: + raise ValueError("Snapshot manifest exceeds 8 MiB") + value = json.loads(raw) + if ( + not isinstance(value, dict) + or value.get("schemaVersion") != 1 + or value.get("paths") + != {name: str(path) for name, path in paths.targets().items()} + ): + raise ValueError( + "Snapshot format or original data locations do not match this service" + ) + expected = value.get("files") + if ( + not isinstance(expected, dict) + or not isinstance(value.get("present"), list) + or set(value["present"]) - paths.targets().keys() + ): + raise ValueError("Invalid snapshot manifest") + actual = { + str(relative): _digest(path) + for relative, path in _files(snapshot) + if str(relative) != "manifest.json" + } + if actual != expected: + raise ValueError("Snapshot contents failed checksum verification") + for relative in expected: + path = Path(relative) + if ( + path.is_absolute() + or ".." in path.parts + or not path.parts + or path.parts[0] not in paths.targets() + ): + raise ValueError("Snapshot contains an invalid destination") + if path.parts[0] not in {"sessions", "revisions"} and len(path.parts) != 1: + raise ValueError("Snapshot file destination is invalid") + if "database.sqlite3" in expected: + with closing( + sqlite3.connect( + (snapshot / "database.sqlite3").as_uri() + "?mode=ro&immutable=1", + uri=True, + ) + ) as connection: + _require_idle_database(connection) + return value + + +def _install_file(source: Path, target: Path): + temporary = target.with_name("." + target.name + ".restore-new") + # A failed earlier attempt may have left only its staging file. + temporary.unlink(missing_ok=True) + try: + _copy_file(source, temporary) + os.replace(temporary, target) + _sync_directory(target.parent) + finally: + temporary.unlink(missing_ok=True) + + +def restore_snapshot(paths: StatePaths, snapshot: Path, *, replace_data: bool) -> dict: + if not replace_data: + raise ValueError( + "Restoring replaces runtime data after the snapshot. Pass --replace-data explicitly." + ) + snapshot = snapshot.expanduser().absolute() + if any(snapshot.is_relative_to(root) for root in (paths.sessions, paths.revisions)): + raise ValueError( + "The snapshot must be outside the runtime directories being restored" + ) + manifest = _manifest(snapshot, paths) + extra = { + Path(name).parts[1] + for name in manifest["files"] + if name.startswith("sessions/") + and len(Path(name).parts) > 2 + and not Path(name).parts[1].startswith(".") + } + marker = Database(paths.database).restore_marker + with _offline(paths, extra_sessions=extra): + # Recheck after locking; nothing has been modified yet. + manifest = _manifest(snapshot, paths) + identity = _digest(snapshot / "manifest.json")["sha256"] + if marker.exists(): + with os.fdopen( + open_existing_private_file(marker), "r", encoding="utf-8" + ) as stream: + journal = json.load(stream) + if ( + journal.get("snapshot") != str(snapshot) + or journal.get("manifestHash") != identity + ): + raise ValueError( + "A different restore is pending. Resume its original snapshot first." + ) + else: + before = ( + paths.database.parent + / "backups" + / ("before-restore-" + datetime.now(UTC).strftime("%Y%m%dT%H%M%S%f")) + ) + _snapshot(paths, before, require_idle=False) + journal = { + "schemaVersion": 1, + "snapshot": str(snapshot), + "manifestHash": identity, + "beforeRestore": str(before), + } + atomic_write_private_json(marker, journal) + wanted = set(manifest["files"]) + for name, target in paths.targets().items(): + if name in {"sessions", "revisions"}: + # Preserve lock inodes, preventing a competing process from + # acquiring a newly created replacement lock during restore. + for relative, current in list(_files(target, prefix=Path(name))): + if str(relative) not in wanted: + current.unlink() + elif name not in manifest["present"]: + target.unlink(missing_ok=True) + for relative in sorted(wanted): + parts = Path(relative).parts + target = paths.targets()[parts[0]].joinpath(*parts[1:]) + _install_file(snapshot / relative, target) + # Disposable indexes/WAL must not replay changes from the replaced DB. + for path in (paths.database, paths.sessions / "index.db"): + for suffix in ("-wal", "-shm", "-journal"): + Path(str(path) + suffix).unlink(missing_ok=True) + (paths.sessions / "index.db").unlink(missing_ok=True) + for root in (paths.sessions, paths.revisions): + _sync_tree(root) + # Pending external login flows must never become valid again merely + # because a backup restored their old generation values. + if paths.credentials.exists(): + data = json.loads(paths.credentials.read_text()) + import secrets + + data["loginGenerations"] = { + key: secrets.token_hex(24) for key in data.get("loginGenerations", {}) + } + atomic_write_private_json(paths.credentials, data) + atomic_write_private_json( + Database(paths.database).restore_recovery_marker, + {"schemaVersion": 1, "snapshot": str(snapshot)}, + ) + marker.unlink() + _sync_directory(marker.parent) + return { + "restored": str(snapshot), + "beforeRestore": journal["beforeRestore"], + "phase": "stopped", + } diff --git a/app_server/stdio_relay.py b/app_server/stdio_relay.py new file mode 100644 index 000000000..4e6b853ca --- /dev/null +++ b/app_server/stdio_relay.py @@ -0,0 +1,218 @@ +"""Native stdio attachment to a shared service; EOF releases only this client.""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import queue +import os +import select +import threading +from typing import BinaryIO + +from app_server.errors import NotInitialized, RpcError +from app_server.native_client import NativeRpcClient +from app_server.protocol.codec import ( + DEFAULT_MAX_MESSAGE_BYTES, + decode_request, + encode_message, +) +from app_server.protocol.models import Response +from app_server.service_state import ServiceFiles + + +def serve_relay(files: ServiceFiles, source: BinaryIO, sink: BinaryIO) -> int: + """Bounded pipes keep slow/closed Desktop clients from retaining the service.""" + incoming: queue.Queue[bytes | None] = queue.Queue(maxsize=32) + outgoing: queue.Queue[bytes | None] = queue.Queue(maxsize=256) + output_lock = threading.Lock() + output_bytes = 0 + stopped = threading.Event() + failed = threading.Event() + loop = asyncio.new_event_loop() + thread = threading.Thread( + target=loop.run_forever, name="deepcode-native-rpc", daemon=True + ) + thread.start() + client = NativeRpcClient(files, notify=lambda message: emit(message)) + + def emit(message: dict) -> None: + nonlocal output_bytes + try: + frame = encode_message(message) + if len(frame) > DEFAULT_MAX_MESSAGE_BYTES: + raise ValueError("Response exceeds the message limit") + with output_lock: + if output_bytes + len(frame) > 8 * 1024 * 1024: + raise queue.Full + outgoing.put_nowait(frame) + output_bytes += len(frame) + except (queue.Full, ValueError): + failed.set() + + def read() -> None: + try: + for raw in _input_frames(source, stopped): + if len(raw) > DEFAULT_MAX_MESSAGE_BYTES: + failed.set() + return + while not stopped.is_set(): + try: + incoming.put(raw or None, timeout=0.1) + break + except queue.Full: + continue + if not raw: + return + except (OSError, ValueError): + failed.set() + + def write() -> None: + nonlocal output_bytes + try: + while True: + frame = outgoing.get() + if frame is None: + return + sink.write(frame) + sink.flush() + with output_lock: + output_bytes -= len(frame) + except (OSError, ValueError): + failed.set() + + reader = threading.Thread(target=read, name="deepcode-native-stdin", daemon=True) + writer = threading.Thread(target=write, name="deepcode-native-stdout", daemon=True) + reader.start() + writer.start() + + def run(coroutine): + future = asyncio.run_coroutine_threadsafe(coroutine, loop) + try: + return future.result(timeout=40) + except concurrent.futures.TimeoutError: + future.cancel() + raise RpcError( + -32000, "Service connection timed out", stable_code="RESULT_UNKNOWN" + ) from None + + try: + initialized = False + while not failed.is_set(): + if initialized and client.closed.is_set(): + # EOF tells the native bridge to offer Reconnect. Never start a + # replacement application after a network failure. + break + try: + raw = incoming.get(timeout=0.1) + except queue.Empty: + continue + if raw is None: + break + request = None + try: + request = decode_request(raw) + if not initialized: + if request.method != "initialize": + raise NotInitialized() + result = run(client.connect(request.params, start=True)) + initialized = True + elif request.method == "shutdown": + result = {"accepted": True} + elif request.method == "service/status": + from app_server.service_client import ServiceClient + + if request.params: + raise RpcError( + -32602, + "Unexpected parameters", + stable_code="INVALID_REQUEST", + ) + result = ServiceClient(files).call("status") + elif request.method == "service/stop": + from cli.service_cli import stop_service + + if request.params: + raise RpcError( + -32602, + "Unexpected parameters", + stable_code="INVALID_REQUEST", + ) + # Native management only, bounded drain; no implicit cancel. + result = stop_service(files, timeout=10, cancel_running=False) + else: + result = run(client.request(request.method, request.params)) + if request.has_id: + emit(Response(request.id, result=result).to_dict()) + if request.method == "shutdown": + break + except RpcError as exc: + if request is None or request.has_id: + emit( + Response( + request.id if request else None, error=exc.payload() + ).to_dict() + ) + except (OSError, RuntimeError, ValueError) as exc: + error = RpcError(-32000, str(exc), stable_code="SERVICE_UNAVAILABLE") + emit( + Response( + request.id if request else None, error=error.payload() + ).to_dict() + ) + break + return 1 if failed.is_set() else 0 + finally: + stopped.set() + try: + asyncio.run_coroutine_threadsafe(client.close(), loop).result(timeout=5) + except (concurrent.futures.TimeoutError, RuntimeError): + pass + loop.call_soon_threadsafe(loop.stop) + thread.join(timeout=1) + if not thread.is_alive(): + loop.close() + try: + outgoing.put(None, timeout=1) + except queue.Full: + pass + writer.join(timeout=1) + reader.join(timeout=1) + + +def _input_frames(source: BinaryIO, stopped: threading.Event): + """Frame raw pipe reads without holding Python's buffered-stdin lock. + + A shutdown request may arrive while the parent keeps stdin open. A daemon + blocked in BufferedReader.readline would abort Python during finalization. + POSIX readiness polling also lets the reader exit without waiting for EOF. + Windows raw ReadFile may remain blocked until process exit, but owns no + Python buffered stream lock and does not prevent the relay from exiting. + """ + try: + descriptor = source.fileno() + except (AttributeError, OSError, ValueError): + while not stopped.is_set(): + frame = source.readline(DEFAULT_MAX_MESSAGE_BYTES + 1) + yield frame + if not frame: + return + return + pending = bytearray() + while not stopped.is_set(): + if os.name != "nt" and not select.select([descriptor], [], [], 0.1)[0]: + continue + chunk = os.read( + descriptor, min(65536, DEFAULT_MAX_MESSAGE_BYTES + 1 - len(pending)) + ) + if not chunk: + if pending: + yield bytes(pending) + yield b"" + return + pending.extend(chunk) + while (end := pending.find(b"\n")) >= 0: + yield bytes(pending[: end + 1]) + del pending[: end + 1] + if len(pending) > DEFAULT_MAX_MESSAGE_BYTES: + raise ValueError("Input frame exceeds the message limit") diff --git a/app_server/systemd_user.py b/app_server/systemd_user.py new file mode 100644 index 000000000..c0b66d95f --- /dev/null +++ b/app_server/systemd_user.py @@ -0,0 +1,280 @@ +"""Linux user-systemd supervision for the existing service process.""" + +from __future__ import annotations + +import configparser +import hashlib +import os +import shlex +import subprocess +from pathlib import Path + +from app_server.service_client import ServiceOperationError +from app_server.service_state import ( + ServiceFiles, + service_command, + service_command_port, + service_environment, + service_working_directory, + shell_only_variables, +) +from core.config import deepcode_home +from core.private_storage import open_existing_private_file, open_private_file + + +def _specifier(value: str) -> str: + if any(ord(char) < 32 for char in value): + raise ValueError( + "Service paths and environment cannot contain control characters" + ) + return value.replace("%", "%%") + + +def _quote(value: str) -> str: + return '"' + _specifier(value).replace("\\", "\\\\").replace('"', '\\"') + '"' + + +class SystemdUserService: + name = "systemd user service" + + def __init__(self, files: ServiceFiles, *, directory: Path | None = None): + self.files = files + identity = hashlib.sha256(str(files.database).encode()).hexdigest()[:16] + self.label = f"deepcode-{identity}.service" + config = Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config"))) + if not config.is_absolute(): + raise ValueError("XDG_CONFIG_HOME must be an absolute path") + self.path = (directory or config / "systemd/user") / self.label + + @staticmethod + def _run(*args, timeout=15): + try: + return subprocess.run( + ["systemctl", "--user", *args], + check=False, + capture_output=True, + text=True, + timeout=timeout, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ServiceOperationError( + "Cannot communicate with the user systemd manager; inspect service doctor" + ) from exc + + def available(self): + try: + return self._run("show", "--property=Version", "--value").returncode == 0 + except ServiceOperationError: + return False + + def job(self): + if not self.path.exists() or not self.available(): + return {"loaded": False, "pid": None} + result = self._run( + "show", self.label, "--property=ActiveState,SubState,MainPID", "--no-pager" + ) + if result.returncode: + raise ServiceOperationError("Cannot inspect the installed systemd service") + values = dict( + line.split("=", 1) for line in result.stdout.splitlines() if "=" in line + ) + active = values.get("ActiveState") in { + "active", + "activating", + "reloading", + "deactivating", + } + pid = int(values.get("MainPID", "0")) + return {"loaded": active, "pid": pid or None} + + def read(self): + try: + with os.fdopen( + open_existing_private_file(self.path), "r", encoding="utf-8" + ) as stream: + text = stream.read(65537) + except FileNotFoundError: + return None + if len(text) > 65536: + raise ServiceOperationError("Systemd service file is too large") + parser = configparser.ConfigParser(interpolation=None, strict=True) + parser.optionxform = str + try: + parser.read_string(text) + command = parser["Service"]["ExecStart"] + if not command.startswith(":"): + raise ValueError("Environment substitution must be disabled") + arguments = [value.replace("%%", "%") for value in shlex.split(command[1:])] + port = service_command_port(arguments, self.files.database) + directory = parser["Service"]["WorkingDirectory"].replace("%%", "%") + environment = dict( + value.replace("%%", "%").split("=", 1) + for value in shlex.split(parser["Service"]["Environment"]) + ) + if ( + not Path(directory).is_absolute() + or parser["Service"]["Restart"] != "on-failure" + or parser["Service"]["KillMode"] != "mixed" + or set(environment) - {"DEEPCODE_HOME", "DEEPCODE_SESSIONS_DIR", "PATH"} + ): + raise ValueError("Unexpected service configuration") + value = { + "command": arguments, + "directory": directory, + "environment": environment, + "port": port, + } + if text != self._render(value): + raise ValueError( + "Service definition differs from its managed configuration" + ) + return value + except (configparser.Error, KeyError, ValueError, IndexError) as exc: + raise ServiceOperationError( + "Systemd service configuration changed; stop and reinstall it" + ) from exc + + @staticmethod + def _render(value): + return "\n".join( + [ + "[Unit]", + "Description=DeepCode local service", + "StartLimitIntervalSec=120", + "StartLimitBurst=5", + "", + "[Service]", + "Type=exec", + "ExecStart=:" + " ".join(_quote(arg) for arg in value["command"]), + "WorkingDirectory=" + _specifier(value["directory"]), + "Environment=" + + " ".join( + _quote(key + "=" + item) + for key, item in value["environment"].items() + ), + "Restart=on-failure", + "RestartSec=10", + "TimeoutStopSec=35", + "KillMode=mixed", + "", + "[Install]", + "WantedBy=default.target", + "", + ] + ) + + def install(self, *, port, path=None): + command = service_command(self.files, port) + service_command_port(command, self.files.database) + environment = service_environment(path) + value = { + "command": command, + "directory": str(service_working_directory(command)), + "environment": environment, + "port": port, + } + previous = self.read() + if previous != value and self.job()["loaded"]: + raise ServiceOperationError( + "Stop the loaded service before changing its systemd unit" + ) + if previous != value: + self.path.parent.mkdir(parents=True, exist_ok=True) + temporary = self.path.with_suffix(".tmp") + try: + fd = open_private_file(temporary, os.O_CREAT | os.O_WRONLY | os.O_TRUNC) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(self._render(value)) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self.path) + finally: + temporary.unlink(missing_ok=True) + result = self._run("--no-reload", "enable", str(self.path)) + if result.returncode: + raise ServiceOperationError( + "Could not enable the user service; inspect service doctor" + ) + if self.available() and self._run("daemon-reload").returncode: + raise ServiceOperationError("Systemd could not reload the installed unit") + return {"installed": True, "atLogin": True, "path": str(self.path)} + + def start(self, *, port=None): + value = self.read() + if value is None: + raise ServiceOperationError("No user service is installed") + if port is not None and port != value["port"]: + raise ServiceOperationError( + "Requested port differs from the installed service; reinstall it" + ) + if not self.available(): + raise ServiceOperationError( + "No user systemd session is available; use deepcode serve --foreground" + ) + if ( + not Path(value["command"][0]).is_file() + or not Path(value["directory"]).is_dir() + ): + raise ServiceOperationError( + "Installed runtime path is missing; reinstall the service" + ) + self._run("reset-failed", self.label) + result = self._run("start", self.label) + if result.returncode: + raise ServiceOperationError( + "Systemd could not start the service; inspect service doctor and journalctl --user" + ) + + def unload(self): + if ( + self.job()["loaded"] + and self._run("stop", self.label, timeout=50).returncode + ): + raise ServiceOperationError("Systemd could not stop the service") + + def uninstall(self): + if self.job()["loaded"]: + raise ServiceOperationError("Stop the user service before uninstalling it") + value = self.read() + if value is not None: + if self._run("--no-reload", "disable", self.label).returncode: + raise ServiceOperationError("Could not disable the user service") + self.path.unlink() + if self.available(): + self._run("daemon-reload") + return {"installed": False, "atLogin": False, "path": str(self.path)} + + def doctor(self): + available = self.available() + checks = [{"name": "userManager", "ok": available}] + try: + value = self.read() + checks.append({"name": "configuration", "ok": value is not None}) + if value: + checks.extend( + [ + { + "name": "executable", + "ok": Path(value["command"][0]).is_file(), + }, + { + "name": "workingDirectory", + "ok": Path(value["directory"]).is_dir(), + }, + { + "name": "runtimeHome", + "ok": value["environment"].get("DEEPCODE_HOME") + == str(deepcode_home()), + }, + ] + ) + except ServiceOperationError as exc: + checks.append({"name": "configuration", "ok": False, "message": str(exc)}) + return { + "installed": self.path.exists(), + "path": str(self.path), + "sessionAvailable": available, + **self.job(), + "checks": checks, + "shellOnlyVariables": shell_only_variables(), + } diff --git a/app_server/web_surface.py b/app_server/web_surface.py new file mode 100644 index 000000000..78a2ba56b --- /dev/null +++ b/app_server/web_surface.py @@ -0,0 +1,279 @@ +"""Same-origin browser assets and bounded, authenticated workspace transfers.""" + +from __future__ import annotations + +import asyncio +import itertools +import json +import os +import re +import secrets +import stat +from pathlib import Path +from urllib.parse import quote + +from aiohttp import web + +from app_server.browser_auth import BrowserAuth +from core.application.errors import ApplicationError +from core.private_storage import ensure_private_file +from core.version import __version__ + +ASSET_DIRECTORY = Path(__file__).with_name("web_assets") +MAX_UPLOAD = 10 * 1024 * 1024 +MAX_DOWNLOAD = 32 * 1024 * 1024 + + +def read_web_build(assets: Path = ASSET_DIRECTORY) -> dict | None: + try: + value = json.loads((assets / "web-build.json").read_text()) + if value.get("version") != __version__ or not isinstance( + value.get("buildId"), str + ): + return None + return value + except (OSError, ValueError, AttributeError): + return None + + +async def _file_io(function, *args): + """Finish the current descriptor operation before cancellation closes it.""" + task = asyncio.create_task(asyncio.to_thread(function, *args)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + await task + raise + + +class WebSurface: + def __init__( + self, application, auth: BrowserAuth, phase, *, assets: Path = ASSET_DIRECTORY + ): + self.application = application + self.auth = auth + self.phase = phase + self.assets = assets.resolve() + self._uploads = asyncio.Semaphore(4) + self._upload_lock = asyncio.Lock() + + def build(self) -> dict | None: + return read_web_build(self.assets) + + def routes(self): + return [ + web.get("/", self.index), + web.get("/index.html", self.index), + web.get("/web-build.json", self.manifest), + web.get("/assets/{path:.*}", self.asset), + web.get("/api/session", self.session), + web.post("/api/uploads", self.upload), + web.get("/api/download", self.download), + ] + + async def index(self, request): + if self.build() is None or not (self.assets / "index.html").is_file(): + return web.Response( + status=503, + text="DeepCode web assets are missing or incompatible. Reinstall a complete release, or run npm ci and npm run build:web in desktop/ before starting this service.", + ) + return web.FileResponse(self.assets / "index.html") + + async def manifest(self, request): + build = self.build() + if build is None: + raise web.HTTPServiceUnavailable(text="Web assets are unavailable") + return web.json_response(build) + + async def asset(self, request): + relative = Path(request.match_info["path"]) + path = (self.assets / "assets" / relative).resolve() + if ( + relative.is_absolute() + or ".." in relative.parts + or not path.is_relative_to(self.assets / "assets") + or not path.is_file() + ): + raise web.HTTPNotFound() + return web.FileResponse(path) + + async def session(self, request): + self.auth.require(request) + return web.json_response( + { + "authenticated": True, + "phase": self.phase(), + "version": __version__, + "webBuild": self.build(), + } + ) + + async def upload(self, request): + self.auth.require(request) + if self.phase() != "ready": + raise web.HTTPServiceUnavailable(text="Service is not accepting uploads") + if request.content_type != "application/octet-stream": + raise web.HTTPUnsupportedMediaType(text="Binary upload body required") + if request.content_length is not None and request.content_length > MAX_UPLOAD: + raise web.HTTPRequestEntityTooLarge( + max_size=MAX_UPLOAD, actual_size=request.content_length + ) + try: + context = await asyncio.to_thread( + self.application.workspaces.resolve, + request.query.get("threadId", ""), + require_trusted=True, + ) + except ApplicationError as exc: + raise web.HTTPForbidden(text=exc.user_message) from exc + # A new root-level file avoids following user-controlled subdirectories + # or accepting a browser-local path as a server filesystem path. + name = ( + re.sub( + r"[^\w.\-]", "_", Path(request.query.get("name", "attachment")).name + )[:100] + or "attachment" + ) + filename = f"deepcode-upload-{secrets.token_hex(12)}-{name}" + async with self._uploads, self._upload_lock: + directory = ( + os.open(context.root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + if os.open in os.supports_dir_fd + else None + ) + staging_name = f".{filename}.part" + target = ( + staging_name if directory is not None else context.root / staging_name + ) + destination = filename if directory is not None else context.root / filename + descriptor = None + completed = False + try: + existing = await asyncio.to_thread( + lambda: sum( + path.stat().st_size + for path in itertools.chain( + context.root.glob("deepcode-upload-*"), + context.root.glob(".deepcode-upload-*.part"), + ) + if path.is_file() and not path.is_symlink() + ) + ) + if existing >= 64 * 1024 * 1024: + raise web.HTTPConflict( + text="Remove unused uploaded files before adding more (64 MiB workspace upload budget)" + ) + descriptor = os.open( + target, + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_BINARY", 0), + 0o600, + dir_fd=directory, + ) + if os.name == "nt": + await _file_io(ensure_private_file, context.root / staging_name) + count = 0 + async for chunk in request.content.iter_chunked(64 * 1024): + count += len(chunk) + if count > MAX_UPLOAD or existing + count > 64 * 1024 * 1024: + raise web.HTTPRequestEntityTooLarge( + max_size=MAX_UPLOAD, actual_size=count + ) + + # os.write may be partial even for a regular file. + def write_all(data=chunk): + view = memoryview(data) + while view: + written = os.write(descriptor, view) + if not written: + raise OSError("upload write made no progress") + view = view[written:] + + await _file_io(write_all) + await _file_io(os.fsync, descriptor) + os.close(descriptor) + descriptor = None + if directory is None: + # Windows rename fails if the destination already exists. + os.rename(target, destination) + else: + os.link( + target, + destination, + src_dir_fd=directory, + dst_dir_fd=directory, + follow_symlinks=False, + ) + os.unlink(target, dir_fd=directory) + await _file_io(os.fsync, directory) + completed = True + return web.json_response( + {"path": str(context.root / filename), "name": name, "size": count} + ) + finally: + if descriptor is not None: + os.close(descriptor) + if not completed: + try: + os.unlink(target, dir_fd=directory) + except FileNotFoundError: + pass + if directory is not None: + os.close(directory) + + async def download(self, request): + self.auth.require(request) + try: + context = await asyncio.to_thread( + self.application.workspaces.resolve, request.query.get("threadId", "") + ) + path = self.application.workspaces.path( + context, request.query.get("path", "") + ) + except ApplicationError as exc: + raise web.HTTPForbidden(text=exc.user_message) from exc + # Open once; the response uses this descriptor's bytes, not a later path + # lookup which could follow a swapped symlink. + async with self._uploads: + before = path.lstat() + if not stat.S_ISREG(before.st_mode): + raise web.HTTPBadRequest(text="Download requires a regular file") + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_BINARY", 0), + ) + try: + info = os.fstat(descriptor) + if not os.path.samestat(before, info): + raise web.HTTPConflict(text="File changed while opening download") + if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_DOWNLOAD: + raise web.HTTPBadRequest( + text="Download requires a regular file up to 32 MiB" + ) + response = web.StreamResponse( + headers={ + "Content-Type": "application/octet-stream", + "Content-Length": str(info.st_size), + "Content-Disposition": f"attachment; filename*=UTF-8''{quote(path.name, safe='')}", + } + ) + await response.prepare(request) + remaining = info.st_size + while remaining: + data = await _file_io( + os.read, descriptor, min(64 * 1024, remaining) + ) + if not data: + raise ConnectionError("File changed during download") + await response.write(data) + remaining -= len(data) + await response.write_eof() + return response + finally: + os.close(descriptor) diff --git a/app_server/websocket.py b/app_server/websocket.py new file mode 100644 index 000000000..66d4daaf9 --- /dev/null +++ b/app_server/websocket.py @@ -0,0 +1,329 @@ +"""Bounded aiohttp transport over the same RPC peer used by stdio.""" + +from __future__ import annotations + +import asyncio +from collections import deque +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from functools import partial +import threading + +from aiohttp import WSMsgType, web + +from app_server.browser_auth import BrowserAuth +from app_server.errors import RpcError +from app_server.host import ServiceHost +from app_server.peer import RpcPeer +from app_server.protocol.models import Request + + +async def _close_socket( + ws: web.WebSocketResponse, *, code: int = 1000, message: bytes = b"" +) -> None: + # aiohttp's close timeout covers reading the peer's reply, not all writes. + # Bound the whole close so a stalled client cannot hold logout or shutdown. + try: + await asyncio.wait_for(ws.close(code=code, message=message, drain=False), 3) + except TimeoutError: + pass + + +class FrameQueue: + """A bounded frame queue, including its scheduled wakeups. + + Peer writers never wait for a socket. Overflow disconnects that client so + replies cannot be silently dropped; durable events remain replayable. + """ + + def __init__(self, *, max_frames: int = 1024, max_bytes: int = 8 * 1024 * 1024): + self._loop = asyncio.get_running_loop() + self._wake = asyncio.Event() + self._lock = threading.Lock() + self._frames: deque[bytes] = deque() + self._size = 0 + self._closed = False + self._finished = False + self._max_frames = max_frames + self._max_bytes = max_bytes + self.overflowed = False + + def send(self, frame: bytes) -> None: + with self._lock: + if self._closed or self._finished: + raise BrokenPipeError("WebSocket disconnected") + if ( + len(self._frames) >= self._max_frames + or self._size + len(frame) > self._max_bytes + ): + self.overflowed = True + self._close_locked() + raise BrokenPipeError("WebSocket frame capacity exceeded") + wake = not self._frames + self._frames.append(frame) + self._size += len(frame) + if wake: + self._loop.call_soon_threadsafe(self._wake.set) + + def close(self) -> None: + with self._lock: + self._close_locked() + + def finish(self) -> None: + """Close after the accepted frames have been written.""" + with self._lock: + self._finished = True + self._loop.call_soon_threadsafe(self._wake.set) + + def _close_locked(self) -> None: + if not self._closed: + self._closed = True + self._frames.clear() + self._size = 0 + self._loop.call_soon_threadsafe(self._wake.set) + + async def receive(self) -> bytes | None: + while True: + await self._wake.wait() + with self._lock: + if self._closed: + return None + if self._frames: + frame = self._frames.popleft() + self._size -= len(frame) + if not self._frames and not self._finished: + self._wake.clear() + return frame + if self._finished: + return None + self._wake.clear() + + +class WebSocketTransport: + MAX_CONNECTIONS = 32 + # Business calls have their own bounded pool; management stays responsive. + MAX_REQUESTS = 8 + DRAIN_METHODS = frozenset( + { + "initialize", + "shutdown", + "event/replay", + "thread/read", + "turn/read", + "turn/input/read", + "approval/respond", + "turn/interrupt", + "workflow/respond", + "workflow/interrupt", + "terminal/close", + "terminal/list", + "terminal/read", + } + ) + + def __init__( + self, + host: ServiceHost, + auth: BrowserAuth, + *, + native_authenticated: Callable[[web.Request], bool], + phase: Callable[[], str], + service_info: dict, + ) -> None: + self.host = host + self.auth = auth + self._native_authenticated = native_authenticated + self._phase = phase + self._service_info = service_info + self._connections: dict[web.WebSocketResponse, str | None] = {} + self._slots = asyncio.Semaphore(self.MAX_REQUESTS) + self._executor = ThreadPoolExecutor( + max_workers=self.MAX_REQUESTS, thread_name_prefix="deepcode-rpc" + ) + self._inflight: set[asyncio.Future] = set() + self._idle = asyncio.Event() + self._idle.set() + self._closing = False + + async def wait_idle(self, timeout: float) -> bool: + if self._idle.is_set(): + return True + try: + await asyncio.wait_for(self._idle.wait(), timeout) + except TimeoutError: + return False + return True + + def _guard(self, request: Request) -> None: + phase = self._phase() + if ( + self._closing + or phase == "stopping" + or (phase != "ready" and request.method not in self.DRAIN_METHODS) + ): + raise RpcError( + -32000, + "Service is draining; request was not dispatched", + stable_code="SERVICE_DRAINING", + data={"retryable": True}, + ) + + async def handle(self, request: web.Request) -> web.WebSocketResponse: + if "Origin" in request.headers: + session = self.auth.require(request) + elif self._native_authenticated(request): + session = None + else: + raise web.HTTPUnauthorized(text="Business RPC requires authentication") + if self._closing or self._phase() != "ready": + raise web.HTTPServiceUnavailable( + text="Service is not accepting connections" + ) + if len(self._connections) >= self.MAX_CONNECTIONS: + raise web.HTTPServiceUnavailable(text="Too many RPC connections") + ws = web.WebSocketResponse( + heartbeat=30, + timeout=2, + max_msg_size=self.host.max_message_bytes, + compress=False, + ) + # Reserve before the first await, including handshakes in progress. + self._connections[ws] = session + outbox = FrameQueue() + incoming = FrameQueue(max_frames=32, max_bytes=4 * 1024 * 1024) + peer = None + writer = expiry = dispatcher = None + try: + await ws.prepare(request) + if self._closing or ( + session is not None and not self.auth.remaining(session) + ): + await _close_socket( + ws, code=1008, message=b"Connection authorization ended" + ) + return ws + # The host is already started; registration only takes short locks + # and starts the peer pump. Keep ownership transfer uncancellable. + peer = self.host.connect(outbox.send, service_info=self._service_info) + writer = asyncio.create_task(self._write(ws, outbox)) + dispatcher = asyncio.create_task( + self._dispatch(ws, peer, incoming, outbox, session) + ) + if session is not None: + expiry = asyncio.create_task(self._expire(ws, session)) + async for message in ws: + if message.type != WSMsgType.TEXT: + if message.type == WSMsgType.BINARY: + await _close_socket( + ws, code=1003, message=b"JSON text frames required" + ) + break + try: + incoming.send(message.data.encode("utf-8")) + except BrokenPipeError: + await _close_socket( + ws, code=1013, message=b"Request capacity exceeded" + ) + break + except (ConnectionError, OSError): + pass + finally: + incoming.close() + outbox.close() + if dispatcher is not None: + await asyncio.gather(dispatcher, return_exceptions=True) + if peer is not None: + await asyncio.to_thread(peer.close) + for task in (writer, expiry): + if task is not None: + task.cancel() + await asyncio.gather( + *(t for t in (writer, expiry) if t is not None), return_exceptions=True + ) + if ws.prepared: + await _close_socket(ws) + self._connections.pop(ws, None) + return ws + + async def _dispatch( + self, + ws: web.WebSocketResponse, + peer: RpcPeer, + incoming: FrameQueue, + outgoing: FrameQueue, + session: str | None, + ) -> None: + """Keep request order without blocking the socket's ping/pong reader.""" + work = None + try: + while (raw := await incoming.receive()) is not None: + async with self._slots: + if ws.closed or self._closing: + break + if session is not None and not self.auth.remaining(session): + await _close_socket( + ws, code=1008, message=b"Browser session expired" + ) + break + work = asyncio.get_running_loop().run_in_executor( + self._executor, + partial(peer.receive, raw, before_dispatch=self._guard), + ) + self._inflight.add(work) + self._idle.clear() + work.add_done_callback(self._finished) + # Disconnect/cancellation must not cancel an admitted mutation. + await asyncio.shield(work) + if peer.closed: + break + except (ConnectionError, OSError): + pass + finally: + if work is not None: + await asyncio.gather(work, return_exceptions=True) + # A shutdown reply precedes the WebSocket close frame. + outgoing.finish() + + def _finished(self, task: asyncio.Future) -> None: + self._inflight.discard(task) + if not self._inflight: + self._idle.set() + + async def _write(self, ws: web.WebSocketResponse, outbox: FrameQueue) -> None: + try: + while (frame := await outbox.receive()) is not None: + await asyncio.wait_for( + ws.send_str(frame.decode("utf-8").rstrip("\n")), 10 + ) + except (TimeoutError, ConnectionError, OSError): + outbox.close() + finally: + await _close_socket(ws, code=1013 if outbox.overflowed else 1000) + + async def _expire(self, ws: web.WebSocketResponse, session: str) -> None: + await asyncio.sleep(self.auth.remaining(session)) + await _close_socket(ws, code=1008, message=b"Browser session expired") + + async def revoke(self, session: str) -> None: + self.auth.revoke(session) + await asyncio.gather( + *( + _close_socket(ws, code=1008, message=b"Browser session revoked") + for ws, owner in tuple(self._connections.items()) + if owner == session and ws.prepared + ) + ) + + async def shutdown(self, _app: web.Application) -> None: + self._closing = True + await asyncio.gather( + *( + _close_socket(ws, code=1001, message=b"Service stopping") + for ws in tuple(self._connections) + if ws.prepared + ) + ) + + async def cleanup(self, _app: web.Application) -> None: + await asyncio.gather(*tuple(self._inflight), return_exceptions=True) + await asyncio.to_thread(self._executor.shutdown, wait=True) diff --git a/app_server/windows_task.py b/app_server/windows_task.py new file mode 100644 index 000000000..5d0233b50 --- /dev/null +++ b/app_server/windows_task.py @@ -0,0 +1,391 @@ +"""User Task Scheduler adapter; no administrator service or shell wrapper.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import subprocess +import time +import xml.etree.ElementTree as ET +from pathlib import Path + +from app_server.managed_entry import read_configuration +from app_server.service_client import ServiceClient, ServiceOperationError +from app_server.service_state import ( + ServiceFiles, + service_command, + service_command_port, + service_environment, + service_working_directory, + shell_only_variables, +) +from core.private_storage import open_private_file + +TASK_NS = "http://schemas.microsoft.com/windows/2004/02/mit/task" +ET.register_namespace("", TASK_NS) + + +def task_definition(config: dict, sid: str, label: str, configuration: Path) -> str: + def child(parent, name, text=None, **attributes): + element = ET.SubElement(parent, "{" + TASK_NS + "}" + name, attributes) + element.text = text + return element + + root = ET.Element("{" + TASK_NS + "}Task", {"version": "1.4"}) + registration = child(root, "RegistrationInfo") + child(registration, "Description", "DeepCode local service " + label) + child(registration, "URI", "\\DeepCode\\" + label) + trigger = child(child(root, "Triggers"), "LogonTrigger") + child(trigger, "Enabled", "true") + child(trigger, "UserId", sid) + principal = child(child(root, "Principals"), "Principal", id="Author") + child(principal, "UserId", sid) + child(principal, "LogonType", "InteractiveToken") + child(principal, "RunLevel", "LeastPrivilege") + settings = child(root, "Settings") + for name, value in { + "MultipleInstancesPolicy": "IgnoreNew", + "DisallowStartIfOnBatteries": "false", + "StopIfGoingOnBatteries": "false", + "AllowHardTerminate": "true", + "StartWhenAvailable": "true", + "RunOnlyIfNetworkAvailable": "false", + "AllowStartOnDemand": "true", + "Enabled": "true", + "Hidden": "true", + "ExecutionTimeLimit": "PT0S", + }.items(): + child(settings, name, value) + restart = child(settings, "RestartOnFailure") + child(restart, "Interval", "PT1M") + child(restart, "Count", "3") + action = child(child(root, "Actions", Context="Author"), "Exec") + arguments = config["command"] + frozen = arguments[1:2] == ["--serve"] + program = Path(arguments[0]) + if not frozen and program.with_name("pythonw.exe").is_file(): + program = program.with_name("pythonw.exe") + child(action, "Command", str(program)) + child( + action, + "Arguments", + subprocess.list2cmdline( + ([] if frozen else ["-m", "app_server"]) + + ["--managed-config", str(configuration)] + ), + ) + child(action, "WorkingDirectory", config["directory"]) + return ET.tostring(root, encoding="unicode") + + +class WindowsUserTask: + name = "Windows user task" + + def __init__(self, files: ServiceFiles): + self.files = files + identity = hashlib.sha256(str(files.database).encode()).hexdigest()[:16] + self.label = f"DeepCode-{identity}" + self.path = files.directory / "task-configuration.json" + + def _run(self, script, *, timeout=20, extra=None): + prelude = "$ErrorActionPreference='Stop'; [Console]::OutputEncoding=[Text.UTF8Encoding]::new($false); " + encoded = base64.b64encode((prelude + script).encode("utf-16-le")).decode( + "ascii" + ) + environment = {**os.environ, "DEEPCODE_TASK_LABEL": self.label, **(extra or {})} + try: + result = subprocess.run( + [ + "powershell.exe", + "-NoProfile", + "-NonInteractive", + "-EncodedCommand", + encoded, + ], + check=False, + capture_output=True, + encoding="utf-8", + timeout=timeout, + env=environment, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ServiceOperationError( + "Windows Task Scheduler operation did not finish; inspect service doctor" + ) from exc + if result.returncode: + raise ServiceOperationError( + "Windows Task Scheduler rejected the operation: " + + result.stderr.strip()[:600] + ) + return result.stdout.strip() + + @staticmethod + def _connect(): + return ( + "$scheduler=New-Object -ComObject Schedule.Service; $scheduler.Connect(); " + ) + + def _sid(self): + return self._run( + "[Security.Principal.WindowsIdentity]::GetCurrent().User.Value" + ) + + def available(self): + try: + self._run(self._connect()) + return True + except ServiceOperationError: + return False + + def _task(self): + script = ( + self._connect() + + """ +try { $folder=$scheduler.GetFolder('\\DeepCode'); $task=$folder.GetTask($env:DEEPCODE_TASK_LABEL) } +catch { + $failure=$_.Exception + while ($null -ne $failure) { + if ($failure.HResult -in @(-2147024894,-2147024893)) { @{registered=$false} | ConvertTo-Json -Compress; exit 0 } + $failure=$failure.InnerException + } + throw +} +@{registered=$true; state=[int]$task.State; xml=$task.Xml} | ConvertTo-Json -Compress +""" + ) + value = json.loads(self._run(script)) + if value.get("registered"): + root = ET.fromstring(value["xml"]) + ns = {"t": TASK_NS} + if ( + root.findtext("t:RegistrationInfo/t:URI", namespaces=ns) + != "\\DeepCode\\" + self.label + ): + raise ServiceOperationError( + "Task identity does not match this DeepCode service" + ) + if ( + root.findtext("t:Principals/t:Principal/t:UserId", namespaces=ns) + != self._sid() + ): + raise ServiceOperationError("Task belongs to another Windows user") + return value + + def _verify_definition(self, task, value): + if not task.get("registered"): + return + actual = ET.fromstring(task["xml"]) + expected = ET.fromstring( + task_definition(value, self._sid(), self.label, self.path) + ) + ns = {"t": TASK_NS} + for container in ("Actions", "Triggers", "Principals"): + if len(actual.findall(f"t:{container}/*", ns)) != 1: + raise ServiceOperationError( + "Task definition has unexpected actions or principals" + ) + for path in ( + "Actions/Exec/Command", + "Actions/Exec/Arguments", + "Actions/Exec/WorkingDirectory", + "Principals/Principal/LogonType", + "Principals/Principal/RunLevel", + "Triggers/LogonTrigger/UserId", + "Settings/ExecutionTimeLimit", + "Settings/MultipleInstancesPolicy", + "Settings/RestartOnFailure/Count", + ): + query = "/".join("t:" + part for part in path.split("/")) + if actual.findtext(query, namespaces=ns) != expected.findtext( + query, namespaces=ns + ): + raise ServiceOperationError( + "Task definition changed; stop and reinstall it" + ) + + def job(self): + if not self.path.exists(): + return {"loaded": False, "pid": None} + task = self._task() + return {"loaded": task.get("state") in {2, 4}, "pid": None} + + def read(self): + try: + value = read_configuration(self.path) + except FileNotFoundError: + return None + try: + if value["database"] != str(self.files.database): + raise ValueError("Wrong service database") + if ( + service_command_port(value["command"], self.files.database) + != value["port"] + ): + raise ValueError("Wrong service port") + if not Path(value["directory"]).is_absolute(): + raise ValueError("Invalid working directory") + except (KeyError, ValueError) as exc: + raise ServiceOperationError( + "Task configuration changed; stop and reinstall it" + ) from exc + return value + + def install(self, *, port, path=None): + command = service_command(self.files, port) + service_command_port(command, self.files.database) + environment = service_environment(path) + value = { + "schemaVersion": 1, + "database": str(self.files.database), + "port": port, + "command": command, + "directory": str(service_working_directory(command)), + "environment": environment, + } + previous, task = self.read(), self._task() + if previous is not None: + self._verify_definition(task, previous) + if task.get("registered") and previous is None: + raise ServiceOperationError( + "An existing task has no matching DeepCode configuration" + ) + if previous != value and task.get("state") in {2, 4}: + raise ServiceOperationError( + "Stop the active task before updating its configuration" + ) + self.path.parent.mkdir(parents=True, exist_ok=True) + temporary = self.path.with_suffix(".tmp") + xml_path = self.path.with_suffix(".xml.tmp") + try: + with os.fdopen( + open_private_file(temporary, os.O_CREAT | os.O_TRUNC | os.O_WRONLY), + "w", + encoding="utf-8", + ) as stream: + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + definition = task_definition(value, self._sid(), self.label, self.path) + with os.fdopen( + open_private_file(xml_path, os.O_CREAT | os.O_TRUNC | os.O_WRONLY), + "w", + encoding="utf-8", + ) as stream: + stream.write(definition) + # Register only a logon trigger: registration does not launch work. + script = ( + self._connect() + + """ +try { $folder=$scheduler.GetFolder('\\DeepCode') } catch { $folder=$scheduler.GetFolder('\\').CreateFolder('DeepCode') } +$definition=$scheduler.NewTask(0); $definition.XmlText=[IO.File]::ReadAllText($env:DEEPCODE_TASK_XML) +$sid=[Security.Principal.WindowsIdentity]::GetCurrent().User.Value +$null=$folder.RegisterTaskDefinition($env:DEEPCODE_TASK_LABEL,$definition,6,$sid,$null,3) +""" + ) + os.replace(temporary, self.path) + try: + self._run(script, extra={"DEEPCODE_TASK_XML": str(xml_path)}) + except BaseException: + if previous is None: + self.path.unlink(missing_ok=True) + else: + with os.fdopen( + open_private_file( + temporary, os.O_CREAT | os.O_TRUNC | os.O_WRONLY + ), + "w", + encoding="utf-8", + ) as stream: + json.dump(previous, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self.path) + raise + finally: + temporary.unlink(missing_ok=True) + xml_path.unlink(missing_ok=True) + return {"installed": True, "atLogin": True, "path": str(self.path)} + + def start(self, *, port=None): + value = self.read() + task = self._task() + if value is None or not task.get("registered"): + raise ServiceOperationError("No Windows user task is installed") + self._verify_definition(task, value) + if port is not None and port != value["port"]: + raise ServiceOperationError( + "Requested port differs from the installed task" + ) + self._run( + self._connect() + + "$task=$scheduler.GetFolder('\\DeepCode').GetTask($env:DEEPCODE_TASK_LABEL); $task.Enabled=$true; $null=$task.Run($null)" + ) + + def unload(self): + if not self.job()["loaded"]: + return + task = "$task=$scheduler.GetFolder('\\DeepCode').GetTask($env:DEEPCODE_TASK_LABEL); " + self._run(self._connect() + task + "$task.Enabled=$false") + try: + if self.files.running(): + stopped = ServiceClient(self.files).call( + "stop", {"timeout": 10, "cancelRunning": True}, timeout=25 + ) + deadline = time.monotonic() + 35 + while self.files.running(): + current = self.files.read() + if ( + current is not None + and current[0].instance_id != stopped["instanceId"] + ): + raise ServiceOperationError( + "Service was replaced during stop; the replacement was not stopped" + ) + if time.monotonic() >= deadline: + raise ServiceOperationError( + "Service cleanup has not finished; inspect service logs" + ) + time.sleep(0.05) + self._run(self._connect() + task + "$task.Stop(0)") + finally: + # Ending while disabled suppresses failure restarts; restore only + # the next-login setting, without issuing Run again. + self._run(self._connect() + task + "$task.Enabled=$true") + + def uninstall(self): + if self.job()["loaded"]: + raise ServiceOperationError( + "Stop the Windows user task before uninstalling it" + ) + self.read() + if self._task().get("registered"): + self._run( + self._connect() + + "$scheduler.GetFolder('\\DeepCode').DeleteTask($env:DEEPCODE_TASK_LABEL,0)" + ) + self.path.unlink(missing_ok=True) + return {"installed": False, "atLogin": False, "path": str(self.path)} + + def doctor(self): + available = self.available() + checks = [{"name": "taskScheduler", "ok": available}] + try: + value = self.read() + checks.append({"name": "configuration", "ok": value is not None}) + if value: + checks.append( + {"name": "executable", "ok": Path(value["command"][0]).is_file()} + ) + except (OSError, ValueError, ServiceOperationError) as exc: + checks.append({"name": "configuration", "ok": False, "message": str(exc)}) + job = self.job() if available else {"loaded": False, "pid": None} + return { + "installed": self.path.exists(), + "path": str(self.path), + "sessionAvailable": available, + **job, + "checks": checks, + "shellOnlyVariables": shell_only_variables(), + } diff --git a/cli/__init__.py b/cli/__init__.py index e6a611fb3..025d441fc 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,17 +1,5 @@ -"""DeepCode CLI entries (P2, L5). +"""DeepCode clients for the shared local service. -Two frontends, both pure consumers of the SQ/EQ event stream — neither -touches the kernel directly (DEEPCODE_V2_MASTER_PLAN.md §3 event-sourcing -first): - -- ``python -m cli.tui`` — the interactive terminal UI: free-form multi-turn - conversation, streaming output, tool progress, slash commands, session - resume. The Claude Code / Codex CLI analogue. -- ``python -m cli.exec_cli`` — headless one-shot: run a task, stream NDJSON - events, exit. The CI / harness / scripting entry. - -CLI agent assembly lives in :mod:`core.agent_setup`; the TUI persists through -the canonical :mod:`core.sessions` JSONL store. Desktop uses the same agent -kernel and SessionStore through its own stdio App Server adapter, without -routing CLI command lifecycles through Desktop application services. +TUI, headless, Goal and MCP task entrypoints consume the service's commands and +persistent events. The service owns Agent execution and canonical Session data. """ diff --git a/cli/automation_cli.py b/cli/automation_cli.py index f91d39149..ef16c2117 100644 --- a/cli/automation_cli.py +++ b/cli/automation_cli.py @@ -336,8 +336,8 @@ def _print_result(command: str, result: dict[str, Any], *, as_json: bool) -> Non f"{latest_status:<12} {automation['name']} · {automation['id']}" ) print( - "Interval schedules run only while a scheduler-enabled DeepCode " - "Desktop or App Server is active." + "Interval schedules run while the DeepCode background service is running." + " Closing a client does not stop the service." ) _print_pagination_hint(result, "Automations") return @@ -352,8 +352,8 @@ def _print_result(command: str, result: dict[str, Any], *, as_json: bool) -> Non print(f"thread: {automation['threadId']}") if command == "create" and automation["scheduleKind"] == "interval": print( - "Interval schedules run only while a scheduler-enabled DeepCode " - "Desktop or App Server is active." + "Interval schedules run while the DeepCode background service is running." + " Closing a client does not stop the service." ) return @@ -446,8 +446,17 @@ def run( factory = application_factory or _default_application_factory application: DeepCodeApplication | None = None try: - application = factory() - result = _dispatch(application, args) + if application_factory is None and args.command == "run": + from cli.service_automation import run_automation_service + + result = run_automation_service( + args.automation_id, + request_id=args.request_id, + interactive=bool(not args.json and sys.stdin.isatty()), + ) + else: + application = factory() + result = _dispatch(application, args) except (ApplicationError, ValueError) as exc: _print_error(exc, as_json=args.json) return 1 diff --git a/cli/desktop_cli.py b/cli/desktop_cli.py new file mode 100644 index 000000000..d856d8f00 --- /dev/null +++ b/cli/desktop_cli.py @@ -0,0 +1,171 @@ +"""Launch the native Desktop from an installed app or its source checkout.""" + +from __future__ import annotations + +import argparse +from importlib.metadata import PackageNotFoundError, distribution +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +from urllib.parse import urlsplit +from urllib.request import url2pathname + + +def _is_checkout(root: Path) -> bool: + return (root / "desktop/package.json").is_file() and ( + root / "desktop/src-tauri/tauri.conf.json" + ).is_file() + + +def _source_checkout() -> Path | None: + root = Path(__file__).resolve().parents[1] + if _is_checkout(root): + return root + # A normal uv tool install from a local directory records its source. This + # works without editable .pth hooks and without trusting the caller's cwd. + try: + data = json.loads( + distribution("deepcode-hku").read_text("direct_url.json") or "{}" + ) + url = urlsplit(data.get("url", "")) + if url.scheme == "file" and url.netloc in ("", "localhost"): + root = Path(url2pathname(url.path)) + if _is_checkout(root): + return root + except (PackageNotFoundError, ValueError, TypeError, OSError): + pass + return None + + +def _run_source(root: Path, *, setup: bool, options: list[str]) -> int: + npm = shutil.which("npm") + if npm is None: + raise ValueError("Desktop source development requires Node.js 22+ and npm.") + if shutil.which("cargo") is None: + raise ValueError( + "Desktop source development requires Rust and the platform Tauri prerequisites." + ) + desktop = root / "desktop" + python = ( + root / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + ) + environment = { + **os.environ, + "DEEPCODE_PYTHON": str(python if python.is_file() else sys.executable), + } + if ( + setup + or not ( + desktop + / "node_modules/.bin" + / ("tauri.cmd" if os.name == "nt" else "tauri") + ).is_file() + ): + subprocess.run([npm, "ci"], cwd=desktop, env=environment, check=True) + binary = ( + desktop + / "build/sidecar/dist/deepcode-app-server" + / ("deepcode-app-server.exe" if os.name == "nt" else "deepcode-app-server") + ) + if setup or not binary.is_file(): + for task in ("setup:sidecar", "build:sidecar"): + subprocess.run([npm, "run", task], cwd=desktop, env=environment, check=True) + print("Opening DeepCode Desktop. Keep this development terminal open.", flush=True) + return subprocess.call( + [npm, "run", "tauri", "--", "dev", *options], cwd=desktop, env=environment + ) + + +def _installed_app() -> Path | None: + if sys.platform == "darwin": + candidates = [ + Path.home() / "Applications/DeepCode.app", + Path("/Applications/DeepCode.app"), + ] + elif os.name == "nt": + candidates = [ + Path(base) / "DeepCode/deepcode-desktop.exe" + for key in ("LOCALAPPDATA", "ProgramFiles", "ProgramFiles(x86)") + if (base := os.environ.get(key)) + ] + else: + candidates = [ + Path(directory) / "deepcode-desktop" for directory in os.get_exec_path() + ] + for candidate in candidates: + if sys.platform == "darwin" and candidate.is_dir(): + return candidate + if candidate.is_file(): + # Do not recurse through the historical shell alias of our command. + with candidate.open("rb") as stream: + signature = stream.read(4) + if signature.startswith(b"MZ") or signature == b"\x7fELF": + return candidate + return None + + +def _open_app(app: Path) -> int: + app = app.expanduser().resolve() + if sys.platform == "darwin" and app.suffix == ".app" and app.is_dir(): + return subprocess.call(["open", "-a", str(app)]) + if not app.is_file(): + raise ValueError(f"Desktop application was not found: {app}") + subprocess.Popen( + [str(app)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=os.name != "nt", + ) + return 0 + + +def run(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="deepcode desktop", description="Open the native DeepCode Desktop client." + ) + location = parser.add_mutually_exclusive_group() + location.add_argument( + "--app", + type=Path, + help="Installed application or AppImage at a custom location.", + ) + location.add_argument( + "--source", type=Path, help="Use a specific trusted source checkout." + ) + parser.add_argument( + "--setup", + action="store_true", + help="Rebuild source dependencies and Desktop resources before opening.", + ) + parser.add_argument( + "options", + nargs=argparse.REMAINDER, + help="Optional Tauri development arguments after --.", + ) + args = parser.parse_args(argv) + options = args.options[1:] if args.options[:1] == ["--"] else args.options + try: + source = args.source.expanduser().resolve() if args.source else None + if source is not None and not _is_checkout(source): + raise ValueError("--source must point to a DeepCode source checkout.") + if args.app is None: + source = source or _source_checkout() + if source is not None: + return _run_source(source, setup=args.setup, options=options) + if args.setup or options: + raise ValueError("--setup and Tauri arguments require a source checkout.") + app = args.app or _installed_app() + if app is None: + raise ValueError( + "Desktop is not installed. Install the Desktop app, or use deepcode desktop --source /path/to/DeepCode." + ) + return _open_app(app) + except (ValueError, OSError, subprocess.CalledProcessError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + return 130 diff --git a/cli/exec_cli.py b/cli/exec_cli.py index 83df49432..6735ed12b 100644 --- a/cli/exec_cli.py +++ b/cli/exec_cli.py @@ -14,7 +14,8 @@ add_workspace_trust_argument, parse_access_preset, ) -from cli.headless_turn import HeadlessTurnOptions, run_headless_turn, succeeded +from cli.thread_client import HeadlessTurnOptions +from core.domain.turn import TurnStatus from cli.transcript import TranscriptMode from core.application.errors import ApplicationError from core.config import ConfigError @@ -105,7 +106,7 @@ def _approval_decider(approval: Approval) -> ApprovalStatus: return ApprovalStatus.DENIED -def _run(args: argparse.Namespace) -> int: +def _run(args: argparse.Namespace, *, shared_service: bool = True) -> int: transcript_mode = TranscriptMode.parse( "verbose" if args.verbose else args.transcript ) @@ -125,7 +126,17 @@ def on_event(event) -> None: _emit_human(event, transcript_mode) try: - result = run_headless_turn( + runner_options = {} + if shared_service: + from cli.service_turn import run_service_turn + + runner = run_service_turn + runner_options["detach"] = args.detach + else: + from cli.headless_turn import run_headless_turn + + runner = run_headless_turn + result = runner( HeadlessTurnOptions( prompt=args.prompt, workspace=workspace, @@ -141,6 +152,7 @@ def on_event(event) -> None: ), on_event=on_event, decide_approval=_approval_decider, + **runner_options, ) except ConfigError as exc: print(format_config_error(exc), file=sys.stderr, flush=True) @@ -161,14 +173,32 @@ def on_event(event) -> None: file=sys.stderr, flush=True, ) - return 0 if succeeded(result) else 1 + if args.detach: + print( + json.dumps( + { + "threadId": result.session_id, + "turnId": result.turn.id, + "status": result.turn.status.value, + "detached": True, + } + ), + flush=True, + ) + return 0 + return 0 if result.turn.status is TurnStatus.COMPLETED else 1 -def main(argv: list[str] | None = None) -> int: +def main(argv: list[str] | None = None, *, shared_service: bool = True) -> int: parser = argparse.ArgumentParser( prog="deepcode exec", description="Run one durable coding Turn headlessly.", ) + parser.add_argument( + "--detach", + action="store_true", + help="Submit to the service and return its task identity without waiting", + ) parser.add_argument("prompt", help="The coding task to perform.") parser.add_argument( "--workspace", @@ -227,16 +257,17 @@ def main(argv: list[str] | None = None) -> int: metavar="ID_OR_NAME", help="Select a Skill for this Turn (repeatable, maximum 8).", ) - parser.add_argument( - "--max-iterations", - type=int, - default=None, - help="Optional model-sampling limit for diagnostics (unlimited by default).", - ) + parser.set_defaults(max_iterations=None) + if not shared_service: + parser.add_argument( + "--max-iterations", type=int, help="Optional model-sampling limit." + ) args = parser.parse_args(argv) + if args.detach and not shared_service: + parser.error("--detach requires the shared service") if len(args.skill) > MAX_SELECTED_SKILLS: parser.error(f"--skill may be specified at most {MAX_SELECTED_SKILLS} times") - return _run(args) + return _run(args, shared_service=shared_service) if __name__ == "__main__": diff --git a/cli/goal_runner.py b/cli/goal_runner.py index 5ff4989a7..eeea30ef8 100644 --- a/cli/goal_runner.py +++ b/cli/goal_runner.py @@ -7,23 +7,15 @@ from dataclasses import dataclass from pathlib import Path -from cli.project_trust import ( - open_workspace_project, - require_project_trusted, - set_project_trusted, -) -from core.application.agent_adapter import ConfiguredAgentSessionFactory -from core.application.application import DeepCodeApplication -from core.application.errors import ( - GoalNotFoundError, - InvalidArgumentError, - TurnNotFoundError, -) +import threading +import time +from cli.rpc_models import from_view +from cli.service_thread_client import ServiceThreadClient +from core.application.errors import GoalNotFoundError, InvalidArgumentError from core.domain.execution_security import ExecutionAccessPreset from core.domain.message_provenance import ClientSurface from core.domain.thread_goal import GoalOutcome, ThreadGoal, ThreadGoalStatus from core.events import Event -from core.harness.permissions import PermissionMode @dataclass(frozen=True, slots=True) @@ -37,7 +29,6 @@ class GoalRunOptions: skill_identifiers: tuple[str, ...] = () completion_evidence_command: str = "" token_budget: int | None = None - max_iterations: int | None = None trust_workspace: bool = False access_preset: ExecutionAccessPreset | None = None @@ -50,7 +41,6 @@ class GoalResumeOptions: model: str | None = None reasoning_effort: str | None = None token_budget: int | None = None - max_iterations: int | None = None trust_workspace: bool = False access_preset: ExecutionAccessPreset | None = None @@ -68,187 +58,143 @@ class GoalRunResult: async def run_goal( - options: GoalRunOptions, - *, - on_progress: ProgressHook | None = None, - on_event: EventHook | None = None, + options: GoalRunOptions, *, on_progress=None, on_event=None ) -> GoalRunResult: - """Create a canonical Session and wait on its ordinary-Turn Goal lifecycle.""" + return await _run_attached(options, on_progress=on_progress, on_event=on_event) - workspace = Path(options.workspace).expanduser().resolve() - workspace.mkdir(parents=True, exist_ok=True) - objective = _objective_with_completion_evidence( - options.objective, - options.completion_evidence_command, - ) - factory = ConfiguredAgentSessionFactory( - default_permission_mode=PermissionMode.DEFAULT, - streaming=False, - max_iterations=options.max_iterations, - ) - application = DeepCodeApplication.open( - session_factory=factory, - host_surface="headless", - run_automation_scheduler=False, - ) - event_token: str | None = None + +async def resume_goal( + options: GoalResumeOptions, *, on_progress=None, on_event=None +) -> GoalRunResult: + if not options.session_id.strip(): + raise InvalidArgumentError("Session ID must not be empty") + return await _run_attached(options, on_progress=on_progress, on_event=on_event) + + +async def _run_attached(options, *, on_progress, on_event): + detached = threading.Event() try: - project = open_workspace_project( - application, - str(workspace), - grant_trust=options.trust_workspace, - ) - require_project_trusted(project) - if options.skill_ids and options.skill_identifiers: - raise InvalidArgumentError("pass Skill IDs or Skill identifiers, not both") - skill_ids = options.skill_ids or tuple( - application.skills.select(project.id, identifier).id - for identifier in options.skill_identifiers - ) - thread = application.threads.start( - project.id, - title=objective.splitlines()[0][:60], - # Automated goal runs choose their own composition; an - # interactive default must not narrow them silently. - inherit_default_preset=False, - connection_id=options.connection_id, - model=options.model, - reasoning_effort=options.reasoning_effort, - access_preset_override=options.access_preset, - ) - if on_event is not None: - event_token = application.turns.subscribe_thread_events( - thread.id, - on_event, - ) - goal = application.goals.create( - thread.id, - objective=objective, - token_budget=options.token_budget, - skill_ids=skill_ids, - start=True, - client_surface=ClientSurface.HEADLESS, - ) - return await _wait_for_goal( - application, - thread_id=thread.id, - workspace=thread.workspace_path, - initial=goal, - on_progress=on_progress, + return await asyncio.to_thread( + _execute, options, on_progress, on_event, detached ) finally: - if event_token is not None: - application.turns.unsubscribe_thread_events(event_token) - application.close() + detached.set() -async def resume_goal( - options: GoalResumeOptions, - *, - on_progress: ProgressHook | None = None, - on_event: EventHook | None = None, -) -> GoalRunResult: - """Resume the existing Goal without replacing Session identity or history.""" - - session_id = options.session_id.strip() - if not session_id: - raise InvalidArgumentError("Session ID must not be empty") - workspace_override = ( - str(Path(options.workspace).expanduser().resolve()) - if options.workspace is not None - else None +def _execute(options, on_progress, on_event, detached) -> GoalRunResult: + resuming = isinstance(options, GoalResumeOptions) + objective = ( + None + if resuming + else _objective_with_completion_evidence( + options.objective, options.completion_evidence_command + ) ) - factory = ConfiguredAgentSessionFactory( - default_permission_mode=PermissionMode.DEFAULT, + if not resuming and options.skill_ids and options.skill_identifiers: + raise InvalidArgumentError("pass Skill IDs or Skill identifiers, not both") + if options.workspace is not None: + Path(options.workspace).expanduser().mkdir(parents=True, exist_ok=True) + client = ServiceThreadClient( + workspace=options.workspace, + model=options.model, + connection_id=options.connection_id, + reasoning_effort=options.reasoning_effort, + max_iterations=None, streaming=False, - max_iterations=options.max_iterations, + trust_workspace=options.trust_workspace, + resume_id=options.session_id if resuming else None, + event_sink=on_event, + surface="headless", ) - application = DeepCodeApplication.open( - session_factory=factory, - host_surface="headless", - run_automation_scheduler=False, - ) - event_token: str | None = None try: - thread = application.threads.resume( - session_id, - workspace_path=workspace_override, - ) - project = application.projects.read(thread.project_id) - if options.trust_workspace: - project = set_project_trusted(application, project) - require_project_trusted(project) if options.access_preset is not None: - thread = application.threads.set_access_preset( - thread.id, - options.access_preset, + client.set_access_preset(options.access_preset) + if resuming: + goal = client.goals.read(client.thread.id) + if goal is None: + raise GoalNotFoundError( + f"no Goal is attached to Session {client.thread.id}" + ) + if goal.status is ThreadGoalStatus.COMPLETE: + return GoalRunResult( + goal, + client.thread.id, + client.workspace, + client.goals.read_outcome(client.thread.id), + ) + goal = _apply_budget_override( + client.goals, goal=goal, token_budget=options.token_budget ) - goal = application.goals.read(thread.id) - if goal is None: - raise GoalNotFoundError(f"no Goal is attached to Session {thread.id}") - if goal.status is ThreadGoalStatus.COMPLETE: - return _result( - application, - goal=goal, - workspace=thread.workspace_path, + execution = dict( + connection_id=options.connection_id, + model=options.model, + reasoning_effort=options.reasoning_effort, ) - if on_event is not None: - event_token = application.turns.subscribe_thread_events( - thread.id, - on_event, + if goal.status is ThreadGoalStatus.ACTIVE: + goal = client.goals.continue_goal( + client.thread.id, expected_goal_id=goal.id, **execution + ).goal + elif goal.status in { + ThreadGoalStatus.PAUSED, + ThreadGoalStatus.BLOCKED, + ThreadGoalStatus.BUDGET_LIMITED, + }: + goal = client.goals.resume( + client.thread.id, expected_goal_id=goal.id, **execution + ) + else: + raise InvalidArgumentError( + f"Goal status cannot be resumed: {goal.status.value}" + ) + else: + skills = options.skill_ids or tuple( + client.skills.select(client.project.id, value).id + for value in options.skill_identifiers ) - - goal = _apply_budget_override( - application, - goal=goal, - token_budget=options.token_budget, - ) - execution_options = { - "client_surface": ClientSurface.HEADLESS, - "connection_id": options.connection_id, - "model": options.model, - "reasoning_effort": options.reasoning_effort, - } - if goal.status is ThreadGoalStatus.ACTIVE: - continued = application.goals.continue_goal( - thread.id, - expected_goal_id=goal.id, - **execution_options, - ) - goal = continued.goal - elif goal.status in { - ThreadGoalStatus.PAUSED, - ThreadGoalStatus.BLOCKED, - ThreadGoalStatus.BUDGET_LIMITED, - }: - goal = application.goals.resume( - thread.id, - expected_goal_id=goal.id, - **execution_options, + client.rename_thread(objective.splitlines()[0][:60]) + goal = client.goals.create( + client.thread.id, + objective=objective, + token_budget=options.token_budget, + skill_ids=skills, ) - else: # pragma: no cover - exhaustive guard for future statuses - raise InvalidArgumentError( - f"Goal status cannot be resumed: {goal.status.value}" - ) - - return await _wait_for_goal( - application, - thread_id=thread.id, - workspace=thread.workspace_path, - initial=goal, - on_progress=on_progress, - ) + last_snapshot = None + while not detached.is_set(): + state = client.rpc.call("thread/goal/get", {"threadId": client.thread.id}) + if "executionSettled" not in state: + raise InvalidArgumentError( + "Restart the service with the updated installation to inspect Goal completion" + ) + current = from_view(ThreadGoal, state["goal"]) if state["goal"] else None + if current is None or current.id != goal.id: + raise InvalidArgumentError("The attached Goal was cleared or replaced") + goal = current + client.drain_events() + settled = state["executionSettled"] + if ( + on_progress + and goal != last_snapshot + and (goal.status is ThreadGoalStatus.ACTIVE or settled) + ): + on_progress(goal) + last_snapshot = goal + if settled: + return GoalRunResult( + goal, + client.thread.id, + client.workspace, + from_view(GoalOutcome, state["outcome"]) + if state["outcome"] + else None, + ) + time.sleep(0.05) + raise InterruptedError("Client detached; the Goal continues in the service") finally: - if event_token is not None: - application.turns.unsubscribe_thread_events(event_token) - application.close() + asyncio.run(client.close()) def _apply_budget_override( - application: DeepCodeApplication, - *, - goal: ThreadGoal, - token_budget: int | None, + goals, *, goal: ThreadGoal, token_budget: int | None ) -> ThreadGoal: if token_budget is None: if ( @@ -256,16 +202,14 @@ def _apply_budget_override( and goal.token_budget is not None ): raise InvalidArgumentError( - "the Goal exhausted its token budget; provide a larger " - "--token-budget to resume" + "the Goal exhausted its token budget; provide a larger --token-budget to resume" ) return goal if token_budget <= goal.tokens_used: raise InvalidArgumentError( - "the resumed token budget must be greater than tokens already used " - f"({goal.tokens_used})" + f"the resumed token budget must be greater than tokens already used ({goal.tokens_used})" ) - return application.goals.edit( + return goals.edit( goal.thread_id, expected_goal_id=goal.id, objective=goal.objective, @@ -276,67 +220,6 @@ def _apply_budget_override( ) -async def _wait_for_goal( - application: DeepCodeApplication, - *, - thread_id: str, - workspace: str, - initial: ThreadGoal, - on_progress: ProgressHook | None, -) -> GoalRunResult: - goal = initial - last_snapshot: ThreadGoal | None = None - while True: - settled = _goal_execution_settled(application, goal) - if ( - on_progress is not None - and goal != last_snapshot - and (goal.status is ThreadGoalStatus.ACTIVE or settled) - ): - on_progress(goal) - last_snapshot = goal - if settled: - break - await asyncio.sleep(0.05) - goal = application.goals.read(thread_id) or goal - return _result(application, goal=goal, workspace=workspace) - - -def _goal_execution_settled( - application: DeepCodeApplication, - goal: ThreadGoal, -) -> bool: - if goal.status is ThreadGoalStatus.ACTIVE: - return False - outcome = application.goals.read_outcome(goal.thread_id) - deciding_turn_id = outcome.decided_by_turn_id if outcome is not None else None - if deciding_turn_id is None: - return True - try: - deciding_turn = application.turns.read(deciding_turn_id).turn - except TurnNotFoundError: - return True - return deciding_turn.status.is_terminal and application.goals.is_turn_accounted( - goal.thread_id, - goal_id=goal.id, - turn_id=deciding_turn.id, - ) - - -def _result( - application: DeepCodeApplication, - *, - goal: ThreadGoal, - workspace: str, -) -> GoalRunResult: - return GoalRunResult( - goal=goal, - session_id=goal.thread_id, - workspace=workspace, - outcome=application.goals.read_outcome(goal.thread_id), - ) - - def _objective_with_completion_evidence(objective: str, command: str) -> str: clean = objective.strip() if not clean: diff --git a/cli/headless_turn.py b/cli/headless_turn.py index d72400b66..205bb3155 100644 --- a/cli/headless_turn.py +++ b/cli/headless_turn.py @@ -4,9 +4,9 @@ import threading from collections.abc import Callable -from dataclasses import dataclass from pathlib import Path +from cli.thread_client import HeadlessTurnOptions, HeadlessTurnResult from cli.project_trust import ( open_workspace_project, require_project_trusted, @@ -17,35 +17,11 @@ from core.domain.approval import Approval, ApprovalStatus from core.domain.common import new_id from core.domain.execution_profile import ExecutionSelection -from core.domain.execution_security import ExecutionAccessPreset from core.domain.message_provenance import ClientSurface -from core.domain.turn import Turn, TurnStatus from core.events import Event from core.harness.permissions import PermissionMode -@dataclass(frozen=True, slots=True) -class HeadlessTurnOptions: - prompt: str - workspace: str | None = None - resume_id: str | None = None - connection_id: str | None = None - model: str | None = None - reasoning_effort: str | None = None - skill_identifiers: tuple[str, ...] = () - max_iterations: int | None = None - trust_workspace: bool = False - access_preset: ExecutionAccessPreset | None = None - agent_preset: str | None = None - - -@dataclass(frozen=True, slots=True) -class HeadlessTurnResult: - turn: Turn - session_id: str - workspace: str - - EventHook = Callable[[Event], None] ApprovalDecider = Callable[[Approval], ApprovalStatus] @@ -210,15 +186,4 @@ def observe(event: Event) -> None: application.close() -def succeeded(result: HeadlessTurnResult) -> bool: - return result.turn.status is TurnStatus.COMPLETED - - -__all__ = [ - "ApprovalDecider", - "EventHook", - "HeadlessTurnOptions", - "HeadlessTurnResult", - "run_headless_turn", - "succeeded", -] +__all__ = ["run_headless_turn"] diff --git a/cli/loop_cli.py b/cli/loop_cli.py index 5d76b62d5..5f1e1008e 100644 --- a/cli/loop_cli.py +++ b/cli/loop_cli.py @@ -97,7 +97,6 @@ def on_progress(goal) -> None: connection_id=args.connection, reasoning_effort=args.reasoning_effort, token_budget=args.token_budget, - max_iterations=args.max_iterations, trust_workspace=args.trust, access_preset=parse_access_preset(args.access), ), @@ -118,7 +117,6 @@ def on_progress(goal) -> None: reasoning_effort=args.reasoning_effort, skill_identifiers=tuple(args.skill), token_budget=args.token_budget, - max_iterations=args.max_iterations, trust_workspace=args.trust, access_preset=parse_access_preset(args.access), ), @@ -217,12 +215,6 @@ def main(argv: list[str] | None = None) -> int: default=None, help="Total budget for a new Goal, or a larger budget when resuming.", ) - parser.add_argument( - "--max-iterations", - type=int, - default=None, - help="Optional model-sampling limit for diagnostics (unlimited by default).", - ) args = parser.parse_args(argv) if (args.goal is None) == (args.resume is None): parser.error("provide exactly one of GOAL or --resume SESSION_ID") diff --git a/cli/mcp_server.py b/cli/mcp_server.py index 550b9cf69..43c94769c 100644 --- a/cli/mcp_server.py +++ b/cli/mcp_server.py @@ -4,17 +4,11 @@ servers for tools, this lets any MCP client (another agent, an IDE, a second DeepCode) drive DeepCode itself as a coding sub-agent over stdio. -Ported from the reference agent's ``mcp-server/`` (``codex_tool_config`` + -``codex_tool_runner`` + ``message_processor``), adapted to DeepCode's ``mcp`` -SDK and :func:`core.agent_setup.build_agent_session`. Two tools are exposed: - -- ``deepcode`` — run a coding task on a prompt (starts a session) -- ``deepcode-reply`` — continue a prior session by id (multi-turn) - -Every call runs through ``build_agent_session``, so it inherits the full agent -— native tools, hooks, summarization compaction, sandbox, spawn_agent -delegation, skills, and memory. The transport is stdio; the JSON-RPC channel is -stdout, so all logging must stay on stderr (configured below). +The stdio server submits durable Turns to the shared local service. The +``deepcode`` tool starts a Session; ``deepcode-reply`` resumes its canonical id. +Workspace trust and tool approval remain enforced by the service. Closing MCP +stdio detaches the client; it does not close the Agent or erase Session history. +Logs stay on stderr so stdout remains the MCP protocol channel. """ from __future__ import annotations @@ -22,17 +16,12 @@ import asyncio import os import sys -import uuid from typing import Any import mcp.types as types from mcp.server.lowlevel import Server from mcp.server.stdio import stdio_server -# Sessions kept alive for ``deepcode-reply`` follow-ups, keyed by the id we -# return from a ``deepcode`` call (mirrors the reference's thread_id map). -_SESSIONS: dict[str, Any] = {} - _DEEPCODE_TOOL = types.Tool( name="deepcode", title="DeepCode", @@ -69,7 +58,7 @@ title="DeepCode reply", description=( "Continue an existing DeepCode session (started by a prior deepcode " - "call) with a follow-up prompt. The session keeps its full history." + "call) with a follow-up prompt. The shared service keeps its full history across MCP connections." ), inputSchema={ "type": "object", @@ -89,52 +78,62 @@ ) -async def _run_turn(session: Any, prompt: str) -> tuple[str, str]: - """Run one turn on ``session`` and return (final_text, stop_reason).""" - from core.events import UserInput - - final, stop_reason = "", "completed" - async for event in session.run_stream(UserInput(text=prompt)): - if event.msg.type == "task_complete": - final = event.msg.final_text or "" - stop_reason = event.msg.stop_reason or "completed" - return (final.strip() or "(the agent produced no summary)"), stop_reason - - def _reply( text: str, structured: dict[str, Any] ) -> tuple[list[types.TextContent], dict[str, Any]]: return [types.TextContent(type="text", text=text)], structured -async def _handle_deepcode(arguments: dict[str, Any]): - from core.agent_setup import build_agent_session +async def _run_task(prompt: str, *, workspace=None, model=None, session_id=None): + from cli.thread_client import HeadlessTurnOptions + from cli.service_turn import run_service_turn_async + from core.application.errors import ApplicationError + + summary = "" + def on_event(event): + nonlocal summary + if event.msg.type == "agent_message" and event.msg.phase.value == "final_answer": + summary = event.msg.text + elif event.msg.type == "task_complete": + summary = event.msg.final_text or summary + + try: + result = await run_service_turn_async( + HeadlessTurnOptions( + prompt=prompt, workspace=workspace, model=model, resume_id=session_id + ), + on_event=on_event, + ) + except (ApplicationError, OSError, ValueError) as exc: + return _reply(f"Error: {exc}", {"error": getattr(exc, "code", "task failed")}) + return _reply( + summary.strip() or "(the agent produced no summary)", + { + "session_id": result.session_id, + "stop_reason": result.turn.stop_reason or result.turn.status.value, + }, + ) + + +async def _handle_deepcode(arguments: dict[str, Any]): prompt = str(arguments.get("prompt") or "").strip() if not prompt: return _reply("Error: 'prompt' is required.", {"error": "missing prompt"}) workspace = os.path.abspath(str(arguments.get("workspace") or os.getcwd())) - model = arguments.get("model") or None - session, _model, _engine = build_agent_session(workspace=workspace, model=model) - session_id = uuid.uuid4().hex - _SESSIONS[session_id] = session - final, stop_reason = await _run_turn(session, prompt) - return _reply(final, {"session_id": session_id, "stop_reason": stop_reason}) + return await _run_task( + prompt, workspace=workspace, model=arguments.get("model") or None + ) async def _handle_reply(arguments: dict[str, Any]): - session_id = str(arguments.get("session_id") or "") - session = _SESSIONS.get(session_id) - if session is None: - return _reply( - f"Error: no such session {session_id!r}. Start one with the deepcode tool first.", - {"error": "unknown session"}, - ) + session_id = str(arguments.get("session_id") or "").strip() + if not session_id: + return _reply("Error: 'session_id' is required.", {"error": "missing session"}) prompt = str(arguments.get("prompt") or "").strip() if not prompt: return _reply("Error: 'prompt' is required.", {"error": "missing prompt"}) - final, stop_reason = await _run_turn(session, prompt) - return _reply(final, {"session_id": session_id, "stop_reason": stop_reason}) + return await _run_task(prompt, session_id=session_id) def build_server() -> Server: diff --git a/cli/provider_cli.py b/cli/provider_cli.py index 37af30bff..a09641224 100644 --- a/cli/provider_cli.py +++ b/cli/provider_cli.py @@ -6,6 +6,7 @@ import getpass import json import sys +import time from core.application.errors import ApplicationError from core.application.llm_configuration_service import LLMConfigurationService @@ -29,6 +30,21 @@ def _parser() -> argparse.ArgumentParser: add.add_argument("--template") add.add_argument("--label") add.add_argument("--adapter", choices=("openai_compat", "anthropic")) + add.add_argument( + "--protocol", + choices=("auto", "openai_chat", "openai_responses", "anthropic_messages"), + ) + add.add_argument("--auth", choices=("api_key", "none", "oauth")) + add.add_argument( + "--compat", + type=json.loads, + help="Typed compatibility object (JSON); unknown fields are rejected.", + ) + add.add_argument( + "--model-declarations", + type=json.loads, + help="Model declaration array (JSON), including capacities and capabilities.", + ) add.add_argument("--api-base") add.add_argument("--api-key-env") add.add_argument( @@ -57,11 +73,30 @@ def _parser() -> argparse.ArgumentParser: ) command_parsers.append(test) test.add_argument("id") + test.add_argument( + "--agent", + action="store_true", + help="Verify streaming and a local tool round trip: up to 3 model requests, 90 seconds; no shell or file tools.", + ) test.add_argument( "--model", help="Send a minimal real inference request to this model.", ) + login = commands.add_parser( + "login", + help="Sign in to a saved OpenRouter OAuth connection (up to 5 minutes).", + ) + command_parsers.append(login) + login.add_argument("id") + login.add_argument("--no-browser", action="store_true") + logout = commands.add_parser( + "logout", + help="Disconnect a Provider account locally; remote key revocation stays in the Provider settings.", + ) + command_parsers.append(logout) + logout.add_argument("id") + models = commands.add_parser("models", help="List models for a connection.") command_parsers.append(models) models.add_argument("id") @@ -91,6 +126,9 @@ def run(argv: list[str] | None = None) -> int: ("template", "template"), ("label", "label"), ("adapter", "adapter"), + ("protocol", "protocol"), + ("auth", "auth"), + ("compat", "compat"), ("api_base", "apiBase"), ("api_key_env", "apiKeyEnv"), ("catalog", "modelCatalog"), @@ -100,6 +138,10 @@ def run(argv: list[str] | None = None) -> int: candidate = getattr(args, argument) if candidate is not None: value[field] = candidate + if args.model_declarations is not None: + if args.model: + raise ValueError("Use either --model or --model-declarations") + value["manualModels"] = args.model_declarations if args.clear_api_key: value["clearApiKey"] = True if args.api_key: @@ -107,8 +149,31 @@ def run(argv: list[str] | None = None) -> int: result = service.upsert(value) elif args.command == "remove": result = service.remove(args.id) + elif args.command == "login": + flow = service.login_start(args.id, open_browser=not args.no_browser) + try: + if flow["authorizationUrl"]: + print( + flow["authorizationUrl"], + file=sys.stderr if args.json else sys.stdout, + ) + while flow["status"] in {"starting", "pending", "exchanging"}: + time.sleep(0.5) + flow = service.login_poll(flow["flowId"]) + result = flow + except KeyboardInterrupt: + service.login_cancel(flow["flowId"]) + return 130 + finally: + service.close() + elif args.command == "logout": + result = service.logout(args.id) elif args.command == "test": - result = service.test(args.id, model_id=args.model) + result = service.test( + args.id, + model_id=args.model, + **({"mode": "agent"} if args.agent else {}), + ) else: result = service.list_models(args.id, refresh=args.refresh) except (ApplicationError, ValueError) as exc: @@ -117,6 +182,13 @@ def run(argv: list[str] | None = None) -> int: if args.json: print(json.dumps(result, ensure_ascii=False, indent=2)) + elif args.command in {"login", "logout"}: + if args.command == "login": + print( + f"{result['status']}: {result.get('accountId') or result.get('error') or args.id}" + ) + else: + print("Disconnected locally. Remote key settings: " + result["manageUrl"]) elif args.command == "list" or args.command in {"set", "remove"}: for connection in result["connections"]: state = "ready" if connection["configured"] else "credential needed" @@ -150,7 +222,7 @@ def run(argv: list[str] | None = None) -> int: ) if result.get("error"): print(f"warning: {result['error']}", file=sys.stderr) - return 0 + return 1 if args.command == "login" and result["status"] != "authenticated" else 0 if __name__ == "__main__": diff --git a/cli/rpc_models.py b/cli/rpc_models.py new file mode 100644 index 000000000..78500dee0 --- /dev/null +++ b/cli/rpc_models.py @@ -0,0 +1,47 @@ +"""Decode the existing domain wire views without rewriting arbitrary JSON keys.""" + +from __future__ import annotations + +from dataclasses import fields, is_dataclass +from functools import lru_cache +from typing import get_args, get_origin, get_type_hints + +from pydantic import TypeAdapter +from pydantic.alias_generators import to_camel + + +@lru_cache(maxsize=32) +def _adapter(model): + return TypeAdapter(model) + + +@lru_cache(maxsize=32) +def _fields(model): + hints = get_type_hints(model) + return [ + (field.name, to_camel(field.name), hints[field.name]) for field in fields(model) + ] + + +def _input(model, value): + if value is None: + return None + if is_dataclass(model) and isinstance(value, dict): + return { + name: _input(hint, value[wire]) + for name, wire, hint in _fields(model) + if wire in value + } + origin, args = get_origin(model), get_args(model) + if origin in (list, tuple) and isinstance(value, (list, tuple)) and args: + return [_input(args[0], item) for item in value] + # Optional dataclasses need field aliases too. Dict/JSON payloads remain + # untouched: keys such as tool arguments are application data, not fields. + for candidate in args: + if is_dataclass(candidate): + return _input(candidate, value) + return value + + +def from_view(model, value): + return _adapter(model).validate_python(_input(model, value)) diff --git a/cli/schedule_cli.py b/cli/schedule_cli.py index bd5831fc2..176224df0 100644 --- a/cli/schedule_cli.py +++ b/cli/schedule_cli.py @@ -43,12 +43,45 @@ def _autodream_task( connection_id: str | None, reasoning_effort: str | None, ): + async def prompt_runner(prompt): + from cli.thread_client import HeadlessTurnOptions + from cli.service_turn import run_service_turn_async + + summary = "" + + def on_event(event): + nonlocal summary + if ( + event.msg.type == "agent_message" + and event.msg.phase.value == "final_answer" + ): + summary = event.msg.text + elif event.msg.type == "task_complete": + summary = event.msg.final_text or summary + + result = await run_service_turn_async( + HeadlessTurnOptions( + prompt=prompt, + workspace=workspace, + model=model, + connection_id=connection_id, + reasoning_effort=reasoning_effort, + ), + on_event=on_event, + ) + if result.turn.status.value != "completed": + raise RuntimeError( + result.turn.error_message or "Memory maintenance did not complete" + ) + return result.turn.stop_reason or "completed", summary + async def task(run_index: int) -> RunOutcome: result = await consolidate_memory( workspace, model=model, connection_id=connection_id, reasoning_effort=reasoning_effort, + prompt_runner=prompt_runner, ) detail = ( f"{result.notes_before}->{result.notes_after} notes" @@ -75,7 +108,6 @@ async def task(run_index: int) -> RunOutcome: connection_id=args.connection, reasoning_effort=args.reasoning_effort, token_budget=args.token_budget, - max_iterations=args.max_iterations, ) ) return RunOutcome( @@ -149,12 +181,6 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument("--max-runs", type=int, default=5) parser.add_argument("--token-budget", type=int, default=None) - parser.add_argument( - "--max-iterations", - type=int, - default=None, - help="Optional model-sampling limit for diagnostics (unlimited by default).", - ) parser.add_argument("--once", action="store_true", help="Run a single pass.") args = parser.parse_args(argv) if args.job == "loop" and not args.goal: diff --git a/cli/service_automation.py b/cli/service_automation.py new file mode 100644 index 000000000..ce714464a --- /dev/null +++ b/cli/service_automation.py @@ -0,0 +1,120 @@ +"""Monitor a manual Automation Run owned by the shared service.""" + +from __future__ import annotations + +import time + +from app_server.blocking_client import BlockingServiceClient +from app_server.service_state import ServiceFiles +from cli.automation_foreground import ( + ForegroundApprovalRequiredError, + _foreground_settled, +) +from cli.exec_cli import _approval_decider +from cli.rpc_models import from_view +from core.domain.approval import Approval +from core.domain.common import new_id +from core.persistence.database import default_database_path + + +def run_automation_service(automation_id, *, request_id, interactive): + rpc = BlockingServiceClient( + ServiceFiles(default_database_path()), surface="headless" + ) + try: + result = rpc.call( + "automation/run", + { + "automationId": automation_id, + "requestId": request_id or new_id("manual"), + }, + ) + run = result["run"] + thread_id, run_id = run["threadId"], run["id"] + sequence = rpc.call("event/replay", {"threadId": thread_id, "limit": 1})[ + "headSequence" + ] + # The Run may settle between admission and reading the event head. + # Read its current projection once before relying on incremental replay. + offset = 0 + while True: + page = rpc.call( + "automation/runs", + {"automationId": automation_id, "offset": offset, "limit": 100}, + ) + current = next( + (candidate for candidate in page["runs"] if candidate["id"] == run_id), + None, + ) + if current is not None: + run = current + break + if not page["hasMore"]: + raise RuntimeError("The admitted Automation Run is no longer available") + offset = page["nextOffset"] + while True: + # Listing reconciles crash-interrupted Goal/Run projections in the + # existing service; the CLI never performs that recovery itself. + rpc.call("automation/runs", {"automationId": automation_id, "limit": 1}) + through = None + while True: + page = rpc.call( + "event/replay", + { + "threadId": thread_id, + "after": sequence, + **({"through": through} if through is not None else {}), + }, + ) + through = page["headSequence"] if through is None else through + for event in page["events"]: + sequence = event["sequence"] + candidate = event["payload"].get("run") + if ( + event["type"] == "automation.updated" + and candidate + and candidate["id"] == run_id + ): + run = candidate + if not page["hasMore"]: + break + if _foreground_settled(run): + turn = ( + rpc.call("turn/read", {"turnId": run["turnId"]})["turn"] + if run.get("turnId") + else None + ) + return {"run": run, "turn": turn} + active = rpc.call( + "turn/list", {"threadId": thread_id, "state": "active", "limit": 1} + )["turns"] + if active and active[0].get("goalId") == run.get("goalId"): + snapshot = rpc.call("turn/read", {"turnId": active[0]["id"]}) + for value in snapshot["approvals"]: + if value["status"] != "pending": + continue + approval = from_view(Approval, value) + if not interactive: + raise ForegroundApprovalRequiredError( + "Automation requires human approval in an interactive DeepCode client", + details={ + "automationId": automation_id, + "runId": run_id, + "threadId": thread_id, + "turnId": approval.turn_id, + "approvalId": approval.id, + "cleanup": "service_keeps_waiting", + }, + ) + decision = _approval_decider(approval) + try: + rpc.call( + "approval/respond", + {"approvalId": approval.id, "decision": decision.value}, + ) + except Exception as exc: + if getattr(exc, "code", None) != "APPROVAL_ALREADY_RESOLVED": + raise + time.sleep(0.25) + finally: + rpc.close() diff --git a/cli/service_catalogs.py b/cli/service_catalogs.py new file mode 100644 index 000000000..9e5b0a68f --- /dev/null +++ b/cli/service_catalogs.py @@ -0,0 +1,249 @@ +"""Typed RPC adapters for the TUI's existing catalog and Goal commands.""" + +from __future__ import annotations + +from cli.rpc_models import from_view +from core.application.errors import InvalidArgumentError +from core.application.goal_extension import GoalContinueResult +from core.application.mcp_service import McpInventory, McpPresetInventory +from core.application.plugin_service import PluginDiscovery +from core.application.skill_service import SkillDiscovery, SkillInfo +from core.domain.thread_goal import GoalOutcome, ThreadGoal +from core.mcp.oauth import McpOAuthFlowInfo +from core.mcp.probe import McpProbeResult +from core.providers.reasoning import ModelReasoningCapabilities + + +class ServiceLLM: + def __init__(self, rpc): + self.rpc = rpc + + def list_connections(self, project_id=None): + return self.rpc.call("provider/list", {"projectId": project_id}) + + def list_models(self, connection_id, *, project_id=None): + return self.rpc.call( + "model/list", {"projectId": project_id, "connectionId": connection_id} + ) + + def model_reasoning(self, connection_id, model, *, project_id=None): + value = self.rpc.call( + "model/reasoning", + {"projectId": project_id, "connectionId": connection_id, "model": model}, + )["reasoning"] + return ModelReasoningCapabilities.from_dict(value) + + +class ServiceSkills: + def __init__(self, rpc): + self.rpc = rpc + + def list(self, project_id): + return from_view( + SkillDiscovery, self.rpc.call("skills/list", {"projectId": project_id}) + ) + + def select(self, project_id, identifier): + value = self.rpc.call( + "skill/read", {"projectId": project_id, "name": identifier} + )["skill"] + skill = from_view(SkillInfo, value) + if not skill.selectable: + raise InvalidArgumentError(f"Skill {skill.name} is not selectable") + return skill + + +class ServicePlugins: + def __init__(self, rpc): + self.rpc = rpc + + def list(self): + return from_view(PluginDiscovery, self.rpc.call("plugins/list", {})) + + +class ServiceMCP: + def __init__(self, rpc): + self.rpc = rpc + + def list(self, project_id=None): + return from_view( + McpInventory, self.rpc.call("mcp/list", {"projectId": project_id}) + ) + + def list_presets(self, project_id=None): + return from_view( + McpPresetInventory, self.rpc.call("mcp/presets", {"projectId": project_id}) + ) + + def add_preset(self, preset_id, *, project_id=None): + return self.rpc.call( + "mcp/preset/add", {"projectId": project_id, "presetId": preset_id} + ) + + def probe(self, name, *, project_id=None): + return from_view( + McpProbeResult, + self.rpc.call("mcp/probe", {"projectId": project_id, "name": name}), + ) + + def oauth_start(self, name, *, project_id=None, open_browser=True): + return from_view( + McpOAuthFlowInfo, + self.rpc.call( + "mcp/oauth/start", + {"projectId": project_id, "name": name, "openBrowser": open_browser}, + ), + ) + + def oauth_logout(self, name, *, project_id=None): + return self.rpc.call( + "mcp/oauth/logout", {"projectId": project_id, "name": name} + )["removed"] + + def oauth_cancel(self, name, *, project_id=None): + return self.rpc.call( + "mcp/oauth/cancel", {"projectId": project_id, "name": name} + )["cancelled"] + + def set_enabled(self, name, *, enabled, project_id=None): + return self.rpc.call( + "mcp/set-enabled", + {"projectId": project_id, "name": name, "enabled": enabled}, + ) + + def remove(self, *, name, scope, project_id=None): + return self.rpc.call( + "mcp/remove", {"projectId": project_id, "name": name, "scope": scope} + ) + + +class ServiceGoals: + def __init__(self, rpc): + self.rpc = rpc + + def read(self, thread_id): + value = self.rpc.call("thread/goal/get", {"threadId": thread_id})["goal"] + return from_view(ThreadGoal, value) if value else None + + def read_outcome(self, thread_id): + value = self.rpc.call("thread/goal/get", {"threadId": thread_id})["outcome"] + return from_view(GoalOutcome, value) if value else None + + def pause(self, thread_id, *, expected_goal_id): + value = self.rpc.call( + "thread/goal/pause", + {"threadId": thread_id, "expectedGoalId": expected_goal_id}, + ) + return from_view(ThreadGoal, value["goal"]) + + def resume( + self, + thread_id, + *, + expected_goal_id, + client_surface=None, + connection_id=None, + model=None, + reasoning_effort=None, + ): + value = self.rpc.call( + "thread/goal/resume", + { + "threadId": thread_id, + "expectedGoalId": expected_goal_id, + **( + {"connectionId": connection_id} if connection_id is not None else {} + ), + **({"model": model} if model is not None else {}), + **( + {"reasoningEffort": reasoning_effort} + if reasoning_effort is not None + else {} + ), + }, + ) + return from_view(ThreadGoal, value["goal"]) + + def continue_goal( + self, + thread_id, + *, + expected_goal_id, + client_surface=None, + connection_id=None, + model=None, + reasoning_effort=None, + ): + return from_view( + GoalContinueResult, + self.rpc.call( + "thread/goal/continue", + { + "threadId": thread_id, + "expectedGoalId": expected_goal_id, + **( + {"connectionId": connection_id} + if connection_id is not None + else {} + ), + **({"model": model} if model is not None else {}), + **( + {"reasoningEffort": reasoning_effort} + if reasoning_effort is not None + else {} + ), + }, + ), + ) + + def clear(self, thread_id, *, expected_goal_id): + self.rpc.call( + "thread/goal/clear", + {"threadId": thread_id, "expectedGoalId": expected_goal_id}, + ) + + def create( + self, + thread_id, + *, + objective, + skill_ids=(), + start=True, + client_surface=None, + token_budget=None, + ): + value = self.rpc.call( + "thread/goal/set", + { + "threadId": thread_id, + "objective": objective, + "tokenBudget": token_budget, + "skills": list(skill_ids), + "start": start, + }, + ) + return from_view(ThreadGoal, value["goal"]) + + def edit( + self, + thread_id, + *, + expected_goal_id, + objective, + token_budget, + skill_ids, + continue_work, + client_surface=None, + ): + value = self.rpc.call( + "thread/goal/set", + { + "threadId": thread_id, + "expectedGoalId": expected_goal_id, + "objective": objective, + "tokenBudget": token_budget, + "skills": list(skill_ids), + "start": continue_work, + }, + ) + return from_view(ThreadGoal, value["goal"]) diff --git a/cli/service_cli.py b/cli/service_cli.py new file mode 100644 index 000000000..f5d92d218 --- /dev/null +++ b/cli/service_cli.py @@ -0,0 +1,465 @@ +"""Manage the local service without opening another execution runtime.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +from app_server.service_client import ( + ServiceClient, + ServiceOperationError, + ServiceUnavailable, +) +from app_server.service_state import ( + ServiceFiles, + service_command, + service_working_directory, +) +from core.file_lock import exclusive_file_lock +from core.persistence.database import default_database_path +from core.private_storage import open_existing_private_file + + +def _service_manager(files: ServiceFiles): + if sys.platform == "darwin": + from app_server.launchd import LaunchAgent + + return LaunchAgent(files) + if sys.platform == "linux": + from app_server.systemd_user import SystemdUserService + + return SystemdUserService(files) + if sys.platform == "win32": + from app_server.windows_task import WindowsUserTask + + return WindowsUserTask(files) + return None + + +def start_service( + files: ServiceFiles, *, port: int | None = None, timeout: float = 30.0 +) -> dict: + with exclusive_file_lock(files.directory / "management.lock"): + return _start_service(files, port=port, timeout=timeout) + + +def _start_service(files: ServiceFiles, *, port: int | None, timeout: float) -> dict: + if files.running(): + status = _wait_ready( + ServiceClient(files), timeout, port=port, existing_only=True + ) + if status is not None: + return status + agent = _service_manager(files) + if agent is not None and agent.path.exists(): + agent.start(port=port) + try: + return _wait_ready(ServiceClient(files), timeout, port=port) + except BaseException: + agent.unload() + raise + return _start_detached(files, port=port, timeout=timeout) + + +def stop_service(files: ServiceFiles, *, timeout: float, cancel_running: bool) -> dict: + with exclusive_file_lock(files.directory / "management.lock"): + return _stop_service(files, timeout=timeout, cancel_running=cancel_running) + + +def _stop_service(files: ServiceFiles, *, timeout: float, cancel_running: bool) -> dict: + agent = _service_manager(files) + job = agent.job() if agent is not None else {"loaded": False} + if job["loaded"]: + discovered = files.read() if files.running() else None + if files.running() and discovered is None: + raise ServiceUnavailable("Service is starting; retry stop when it is ready") + record = discovered[0] if discovered is not None else None + client = ServiceClient(files) + if record is not None: + if job["pid"] not in (None, record.pid): + raise ServiceOperationError( + "Service manager PID does not match the running service; inspect service doctor" + ) + if not cancel_running: + client.call( + "drain", + {"timeout": timeout}, + timeout=timeout + 10, + instance_id=record.instance_id, + ) + try: + agent.unload() + except BaseException: + if record is not None and not cancel_running: + client.call("resume", instance_id=record.instance_id) + raise + if record is not None: + if job["pid"] is None: + # An idle job may coexist with a manual service. Drain that + # service before unloading too: launchd can start between queries. + try: + client.call( + "stop", + {"timeout": timeout, "cancelRunning": cancel_running}, + timeout=timeout + 10, + instance_id=record.instance_id, + ) + except (ServiceUnavailable, ServiceOperationError): + # A launchd child may already be shutting down after bootout. + # The bounded identity-aware wait below still verifies exit. + pass + return _wait_stopped(files, record.instance_id, timeout=35) + return _stop_detached(files, timeout=timeout, cancel_running=cancel_running) + + +def _start_detached( + files: ServiceFiles, *, port: int | None = None, timeout: float = 30.0 +) -> dict: + client = ServiceClient(files) + command = service_command(files, 3081 if port is None else port) + process = subprocess.Popen( + command, + cwd=service_working_directory(command), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=os.name != "nt", + creationflags=( + subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP + ) + if os.name == "nt" + else 0, + ) + try: + result = _wait_ready(client, timeout, process=process, port=port) + if result["pid"] != process.pid: + # Another concurrent launcher won. Reap our losing child before + # returning, so it cannot become a replacement after the winner stops. + _stop_startup_child(process) + return result + except BaseException: + _stop_startup_child(process) + raise + + +def _stop_startup_child(process: subprocess.Popen) -> None: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def _wait_ready( + client: ServiceClient, + timeout: float, + *, + process=None, + port=None, + existing_only=False, +) -> dict | None: + deadline = time.monotonic() + timeout + last_failure = "No service probe completed" + while True: + try: + status = client.call("status", timeout=1) + if port not in (None, 0) and status["url"] != f"http://127.0.0.1:{port}": + raise ServiceOperationError( + f"Service already runs at {status['url']}; stop it before changing ports" + ) + if status["phase"] == "drained": + client.call("resume", instance_id=status["instanceId"]) + continue + if status["phase"] == "ready": + return status + except ServiceUnavailable as exc: + last_failure = str(exc) + # A status probe can hold the lifetime lock briefly. If that apparent + # owner exits, the caller must launch instead of waiting for a phantom. + if existing_only and not client.files.running(): + return None + if ( + process is not None + and process.poll() is not None + and not client.files.running() + ): + raise ServiceUnavailable( + f"Service startup failed (exit {process.returncode}); inspect {client.files.log}" + ) + if time.monotonic() >= deadline: + raise ServiceUnavailable( + f"Service did not become ready; inspect {client.files.log}. " + f"Last check: {last_failure}" + ) + time.sleep(0.1) + + +def _stop_detached( + files: ServiceFiles, *, timeout: float, cancel_running: bool +) -> dict: + if not files.running(): + return {"phase": "stopped"} + stopped = ServiceClient(files).call( + "stop", + {"timeout": timeout, "cancelRunning": cancel_running}, + timeout=timeout + 10, + ) + return _wait_stopped(files, stopped["instanceId"], timeout=15) + + +def _wait_stopped(files: ServiceFiles, instance_id: str, *, timeout: float) -> dict: + deadline = time.monotonic() + timeout + while files.running(): + current = files.read() + if current is not None and current[0].instance_id != instance_id: + raise ServiceOperationError( + "The stopped service has been replaced by another instance; the replacement was not stopped" + ) + if time.monotonic() >= deadline: + raise ServiceUnavailable( + "Service accepted stop but cleanup has not finished; inspect service logs" + ) + time.sleep(0.05) + return {"phase": "stopped"} + + +def _logs(files: ServiceFiles, *, lines: int, follow: bool) -> None: + offset = 0 + identity = None + first = True + while True: + try: + with os.fdopen(open_existing_private_file(files.log), "rb") as stream: + info = os.fstat(stream.fileno()) + current = (info.st_dev, info.st_ino) + if first: + stream.seek(max(0, info.st_size - 2 * 1024 * 1024)) + content = stream.read().decode("utf-8", errors="replace") + print("\n".join(content.splitlines()[-lines:]), flush=True) + else: + stream.seek( + 0 if current != identity or info.st_size < offset else offset + ) + print( + stream.read().decode("utf-8", errors="replace"), + end="", + flush=True, + ) + offset = stream.tell() + identity = current + first = False + except FileNotFoundError: + if not follow: + print("No service log yet.") + if not follow: + return + time.sleep(0.25) + + +def run(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="deepcode service", + description="Start, inspect, and stop the local DeepCode background service.", + ) + commands = parser.add_subparsers(dest="command", required=True) + for name in ( + "start", + "status", + "stop", + "restart", + "logs", + "install", + "uninstall", + "doctor", + "snapshot", + "prepare-upgrade", + "restore", + ): + command = commands.add_parser(name) + command.add_argument("--database", type=Path) + command.add_argument("--json", action="store_true") + if name in {"start", "restart", "install"}: + command.add_argument("--port", type=int) + if name in {"snapshot", "prepare-upgrade"}: + command.add_argument("--output", type=Path, required=True) + if name in {"snapshot", "prepare-upgrade", "restore"}: + command.add_argument( + "--sessions", + type=Path, + help="Canonical Session directory for a service without saved layout metadata", + ) + if name == "restore": + command.add_argument("--snapshot", type=Path, required=True) + command.add_argument( + "--replace-data", + action="store_true", + help="Explicitly replace runtime data with the snapshot; project files are not restored", + ) + if name in {"stop", "restart", "uninstall", "prepare-upgrade"}: + mode = command.add_mutually_exclusive_group() + mode.add_argument( + "--drain", + action="store_true", + help="Wait for running tasks and terminals (default)", + ) + mode.add_argument( + "--cancel-running", + action="store_true", + help="Cancel current service work before stopping", + ) + command.add_argument("--timeout", type=float, default=60.0) + if name == "install": + command.add_argument( + "--at-login", + action="store_true", + required=True, + help="Opt in to starting after operating-system user login", + ) + command.add_argument( + "--path", + help="PATH for tools in the managed service (default: current PATH)", + ) + if name == "logs": + command.add_argument("--lines", type=int, default=100) + command.add_argument("--follow", action="store_true") + args = parser.parse_args(argv) + if getattr(args, "port", None) is not None and not 0 <= args.port <= 65535: + parser.error("port must be between 0 and 65535") + if hasattr(args, "timeout") and not 0 <= args.timeout <= 300: + parser.error("timeout must be between 0 and 300 seconds") + files = ServiceFiles(args.database or default_database_path()) + try: + if args.command in {"snapshot", "prepare-upgrade", "restore"}: + from app_server.state_backup import ( + StatePaths, + create_snapshot, + restore_snapshot, + ) + + paths = StatePaths.current(files, sessions=args.sessions) + if args.command == "prepare-upgrade": + stop_service( + files, timeout=args.timeout, cancel_running=args.cancel_running + ) + result = ( + restore_snapshot(paths, args.snapshot, replace_data=args.replace_data) + if args.command == "restore" + else create_snapshot(paths, args.output) + ) + print( + json.dumps(result, ensure_ascii=False, indent=None if args.json else 2) + ) + return 0 + if args.command == "logs": + if not 1 <= args.lines <= 10_000: + parser.error("lines must be between 1 and 10000") + _logs(files, lines=args.lines, follow=args.follow) + return 0 + if args.command in {"install", "uninstall", "doctor"}: + agent = _service_manager(files) + if agent is None: + raise ServiceOperationError( + "No user service manager is available on this platform" + ) + with exclusive_file_lock(files.directory / "management.lock"): + if args.command == "install": + result = agent.install( + port=3081 if args.port is None else args.port, path=args.path + ) + elif args.command == "uninstall": + if agent.job()["loaded"]: + _stop_service( + files, + timeout=args.timeout, + cancel_running=args.cancel_running, + ) + result = agent.uninstall() + else: + result = agent.doctor() + if args.json: + print(json.dumps(result, ensure_ascii=False)) + else: + print( + f"{agent.name}: {'installed' if result['installed'] else 'not installed'}" + ) + print(f"File: {result['path']}") + if args.command == "install": + print( + "Starts with the user session. Run 'deepcode service start' to start now." + ) + if args.command == "doctor": + for check in result["checks"]: + print( + f" {check['name']}: {'ok' if check['ok'] else 'needs attention'}" + ) + print( + f"User session available: {result.get('sessionAvailable', result.get('guiSessionAvailable', False))} · job loaded: {result['loaded']}" + ) + if result["shellOnlyVariables"]: + print( + "Shell-only variables are not copied into the service manager: " + + ", ".join(result["shellOnlyVariables"]) + ) + return ( + 1 + if args.command == "doctor" + and any(not check["ok"] for check in result["checks"]) + else 0 + ) + if args.command == "status": + result = ( + ServiceClient(files).call("status") + if files.running() + else {"phase": "stopped"} + ) + agent = _service_manager(files) + if agent is not None: + result["supervision"] = { + "installed": agent.path.exists(), + **agent.job(), + } + elif args.command == "stop": + result = stop_service( + files, timeout=args.timeout, cancel_running=args.cancel_running + ) + else: + with exclusive_file_lock(files.directory / "management.lock"): + port = args.port + if args.command == "restart": + previous = files.read() + agent = _service_manager(files) + if ( + port is None + and previous is not None + and not (agent and agent.path.exists()) + ): + port = previous[0].port + _stop_service( + files, timeout=args.timeout, cancel_running=args.cancel_running + ) + result = _start_service(files, port=port, timeout=30) + if args.json: + print(json.dumps(result, ensure_ascii=False)) + else: + print(f"DeepCode service: {result['phase']}") + if "url" in result: + print(f"Management endpoint: {result['url']} · PID {result['pid']}") + print( + f"Active turns: {result['activeTurns']} · queued: {result['queuedTurns']} · terminals: {result['terminals']}" + ) + return 0 + except KeyboardInterrupt: + return 130 + except (ServiceUnavailable, ServiceOperationError, OSError, ValueError) as exc: + if args.json: + print(json.dumps({"error": str(exc)})) + else: + print(f"error: {exc}", file=sys.stderr) + return 1 diff --git a/cli/service_events.py b/cli/service_events.py new file mode 100644 index 000000000..704d37492 --- /dev/null +++ b/cli/service_events.py @@ -0,0 +1,221 @@ +"""Render durable service events through the existing TUI/exec event vocabulary.""" + +from __future__ import annotations + +from cli.rpc_models import from_view +from core.domain.event import DomainEvent +from core.events import ( + AgentMessage, + AgentMessageCompleted, + AgentMessageDelta, + AgentMessagePhase, + AgentReasoningCompleted, + AgentReasoningDelta, + AgentReasoningStarted, + ErrorEvent, + Event, + ModelUsageRecorded, + PlanStep, + PlanUpdated, + SkillLoaded, + TaskComplete, + ToolActivity, + ToolCompleted, + ToolStarted, + TurnStarted, +) +from core.reasoning import ReasoningChannel, ReasoningPayload +from core.skills.models import SkillInvocation, SkillInvocationKind + + +class ServiceEventRenderer: + """Keep only active item identities; the caller owns the contiguous cursor.""" + + def __init__(self): + self._items = {} + self._started = set() + self._skills = set() + + def seed(self, items): + for item in items: + self._remember( + item.id, + item.kind.value, + item.payload.get("messageId") or item.id, + item.turn_id, + ) + + def _remember(self, item_id, kind, message_id, turn_id): + if len(self._items) >= 2048 and item_id not in self._items: + raise RuntimeError("Too many active timeline items to render") + self._items[item_id] = (kind, message_id, turn_id) + + def convert(self, event: DomainEvent): + messages = self._messages(event) + return [ + Event(f"{event.id}:{index}", message) + for index, message in enumerate(messages) + ] + + def _messages(self, event): + payload = event.payload + turn = payload.get("turn", {}) + if ( + event.type in {"turn.started", "turn.updated"} + and turn.get("status") in {"running", "queued"} + and event.turn_id not in self._started + ): + self._started.add(event.turn_id) + return [TurnStarted()] + if event.type in { + "turn.completed", + "turn.failed", + "turn.interrupted", + "turn.recovered", + }: + self._started.discard(event.turn_id) + self._skills = {key for key in self._skills if key[0] != event.turn_id} + self._items = { + key: value + for key, value in self._items.items() + if value[2] != event.turn_id + } + return [ + TaskComplete( + None, turn.get("stopReason") or turn.get("status") or "interrupted" + ) + ] + if event.type == "turn.usage.recorded": + return [ + ModelUsageRecorded( + response_ordinal=payload["responseOrdinal"], usage=payload["usage"] + ) + ] + if event.type == "turn.plan.updated": + plan = payload.get("plan", {}) + return [ + PlanUpdated( + tuple(from_view(PlanStep, step) for step in plan.get("steps", [])), + plan.get("explanation"), + ) + ] + if event.type == "item.delta": + state = self._items.get(event.item_id) + if state is None: + raise RuntimeError("A streamed item is missing its initial state") + kind, message_id, _ = state + if kind == "assistant_message": + return [AgentMessageDelta(payload["delta"], message_id)] + if kind == "reasoning_summary": + return [ + AgentReasoningDelta( + event.item_id, + ReasoningChannel(payload["reasoningChannel"]), + payload["delta"], + ) + ] + return [] + if event.type not in {"item.created", "item.updated"}: + return [] + item = payload["item"] + kind, data = item["kind"], item["payload"] + identity = data.get("messageId") or item["id"] + running = item["status"] in {"in_progress", "pending"} + created = event.type == "item.created" + if running: + self._remember(item["id"], kind, identity, event.turn_id) + else: + self._items.pop(item["id"], None) + if kind == "user_message": + loaded = [] + for value in data.get("skills", []): + if not isinstance(value, dict): + continue + key = (event.turn_id, value["skillId"]) + if key not in self._skills: + self._skills.add(key) + loaded.append( + SkillLoaded( + SkillInvocation( + value["skillId"], + value["name"], + value["revision"], + value["source"], + SkillInvocationKind(value["invocation"]), + ) + ) + ) + return loaded + if kind == "assistant_message": + if running: + return ( + [AgentMessageDelta(data["text"], identity)] + if created and data.get("text") + else [] + ) + phase = AgentMessagePhase(data.get("phase", "final_answer")) + if phase is AgentMessagePhase.COMMENTARY: + return [ + AgentMessageCompleted( + identity, data.get("text", item["summary"]), phase + ) + ] + return [AgentMessage(data.get("text", item["summary"]), identity, phase)] + if kind == "reasoning_summary": + reasoning = ReasoningPayload.from_dict(data) + if running: + if not created: + return [] + messages = [AgentReasoningStarted(item["id"], reasoning.effort)] + if reasoning.summary_text: + messages.append( + AgentReasoningDelta( + item["id"], ReasoningChannel.SUMMARY, reasoning.summary_text + ) + ) + if reasoning.trace_text: + messages.append( + AgentReasoningDelta( + item["id"], + ReasoningChannel.PROVIDER_TRACE, + reasoning.trace_text, + ) + ) + return messages + return [ + AgentReasoningCompleted( + item["id"], + reasoning.summary_text, + reasoning.trace_text, + reasoning.availability, + reasoning.effort, + reasoning.duration_ms, + ) + ] + if ( + kind in {"tool_call", "command_execution", "file_change", "test_result"} + and "callId" in data + ): + if running and created: + return [ + ToolStarted( + data["callId"], + data["name"], + data.get("detail", ""), + from_view(ToolActivity, data["activity"]) + if data.get("activity") + else None, + ) + ] + if not running: + return [ + ToolCompleted( + data["callId"], + data["name"], + data.get("isError", item["status"] != "completed"), + data.get("resultPreview", ""), + ) + ] + if kind == "error": + return [ErrorEvent(data.get("message") or item["summary"])] + return [] diff --git a/cli/service_thread_client.py b/cli/service_thread_client.py new file mode 100644 index 000000000..c6b2cd0fd --- /dev/null +++ b/cli/service_thread_client.py @@ -0,0 +1,562 @@ +"""TUI attachment using the shared service, with no local execution owner.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from app_server.blocking_client import BlockingServiceClient +from app_server.service_state import ServiceFiles +from cli.rpc_models import from_view +from cli.service_catalogs import ( + ServiceGoals, + ServiceLLM, + ServiceMCP, + ServicePlugins, + ServiceSkills, +) +from cli.service_events import ServiceEventRenderer +from cli.service_turns import ServiceTurns +from cli.thread_client import ThreadListing, TurnDelivery, turn_access_summary +from core.application.errors import ProjectNotTrustedError +from core.application.interactive_turn_router import InteractiveTurnRouter +from core.domain.approval import Approval, ApprovalStatus +from core.domain.common import new_id +from core.domain.event import DomainEvent +from core.domain.execution_profile import ExecutionProfile +from core.domain.execution_security import ExecutionSecurityProfile +from core.domain.message_provenance import ClientSurface +from core.domain.project import Project, TrustState +from core.domain.thread import Thread +from core.events import ErrorEvent, Event +from core.persistence.database import default_database_path +from core.sessions import SessionStore + + +class ServiceThreadClient: + runtime_mode = "service" + + def __init__( + self, + *, + workspace, + model, + connection_id, + reasoning_effort, + max_iterations, + streaming, + trust_workspace=False, + resume_id=None, + store=None, + event_sink=None, + database=None, + surface="cli", + ): + if max_iterations is not None: + raise ValueError("--max-iterations is not supported by the shared service") + self.workspace = str(Path(workspace or Path.cwd()).expanduser().resolve()) + self._event_sink = event_sink + self._domain_task = None + self._renderer = ServiceEventRenderer() + self._domain_sink = None + self._selection_generation = 0 + self._sequence = 0 + self.rpc = BlockingServiceClient( + ServiceFiles(database or default_database_path()), surface=surface + ) + try: + self.llm = ServiceLLM(self.rpc) + self.skills = ServiceSkills(self.rpc) + self.plugins = ServicePlugins(self.rpc) + self.mcp = ServiceMCP(self.rpc) + self.goals = ServiceGoals(self.rpc) + self.turns = ServiceTurns(self.rpc) + self.router = InteractiveTurnRouter( + self.turns, client_surface=ClientSurface.CLI + ) + diagnostics = self.rpc.call("diagnostics/read", {})["diagnostics"] + self.store = store or SessionStore(Path(diagnostics["sessionStorePath"])) + if resume_id is None: + project = self.rpc.call( + "project/add", + { + "path": self.workspace, + "trustState": "trusted" if trust_workspace else "untrusted", + }, + )["project"] + self.project = from_view(Project, project) + if trust_workspace and not self.project_trusted: + self.project = from_view( + Project, + self.rpc.call( + "project/update", + {"projectId": self.project.id, "trustState": "trusted"}, + )["project"], + ) + self._require_trust() + value = self.rpc.call( + "thread/start", + { + "projectId": self.project.id, + "title": "New task", + "connectionId": connection_id, + "model": model, + "reasoningEffort": reasoning_effort, + }, + ) + self._replace_thread(from_view(Thread, value["thread"])) + else: + existing = from_view( + Thread, + self.rpc.call("thread/read", {"threadId": resume_id})["thread"], + ) + if workspace is None: + self.workspace = existing.workspace_path + project = ( + self.rpc.call("project/add", {"path": self.workspace})["project"] + if workspace is not None + else self.rpc.call( + "project/read", {"projectId": existing.project_id} + )["project"] + ) + self.project = from_view(Project, project) + if trust_workspace: + self.project = from_view( + Project, + self.rpc.call( + "project/update", + {"projectId": self.project.id, "trustState": "trusted"}, + )["project"], + ) + self._require_trust() + value = self.rpc.call( + "thread/resume", + {"sessionId": resume_id, "workspacePath": self.workspace}, + ) + self._replace_thread(from_view(Thread, value["thread"])) + if surface != "headless" and ( + model is not None + or connection_id is not None + or reasoning_effort is not None + ): + self.switch_execution( + connection_id=connection_id or self.thread.connection_id, + model=model or self.thread.model, + reasoning_effort=reasoning_effort + if reasoning_effort is not None + else self.thread.reasoning_effort, + context_window=self.thread.context_window, + ) + except BaseException: + self.rpc.close() + raise + + @property + def session_id(self): + return self.thread.id + + @property + def model(self): + return self.execution_profile.model_id + + @property + def project_trusted(self): + return self.project.trust_state is TrustState.TRUSTED + + @property + def access_preset_override(self): + return self.thread.access_preset_override + + def _require_trust(self): + if not self.project_trusted: + raise ProjectNotTrustedError( + "Project is untrusted; inspect it and use --trust explicitly" + ) + + def _replace_thread(self, thread): + project = from_view( + Project, + self.rpc.call("project/read", {"projectId": thread.project_id})["project"], + ) + if project.trust_state is not TrustState.TRUSTED: + raise ProjectNotTrustedError( + "Project is untrusted; inspect it and use --trust explicitly" + ) + profile = self.rpc.call("thread/execution/read", {"threadId": thread.id}) + execution = ExecutionProfile.from_dict(profile["executionProfile"]) + security = ExecutionSecurityProfile.from_dict(profile["securityProfile"]) + if execution is None or security is None: + raise ValueError("Service returned an invalid execution profile") + sequence = self.rpc.call("event/replay", {"threadId": thread.id, "limit": 1})[ + "headSequence" + ] + renderer = ServiceEventRenderer() + active = self.turns.executing_for_thread(thread.id) + if active: + renderer.seed(self.turns.read(active.id).items) + self.thread, self.project = thread, project + self.execution_profile, self._security = execution, security + self._sequence, self._renderer = sequence, renderer + self._selection_generation += 1 + + def access_summary(self): + override = self.thread.access_preset_override + if override is not None: + return override.value.replace("_", " ") + profile = self._security + if profile.access_preset is not None: + return f"default ({profile.access_preset.value.replace('_', ' ')})" + return f"legacy ({profile.permission_mode.value})" + + def frozen_access_summaries(self): + turns = self.turns.list_for_thread(self.thread.id) + current = next( + ( + turn + for turn in turns + if turn.status.value in {"running", "waiting_approval"} + ), + None, + ) + return turn_access_summary(current) if current else None, tuple( + turn_access_summary(turn) for turn in turns if turn.status.value == "queued" + ) + + def send(self, prompt, *, skill_ids=()): + active = self.turns.executing_for_thread(self.thread.id) + result = self.router.send( + self.thread.id, + prompt=prompt, + message_id=new_id("tinp"), + cached_active_turn_id=active.id if active else None, + skill_ids=skill_ids, + ) + if result.delivery.value in {"started", "queued"}: + self._title_from_first_prompt(prompt) + return TurnDelivery(result.delivery.value, result.turn) + + def queue(self, prompt, *, skill_ids=()): + snapshot = self.turns.enqueue( + self.thread.id, + prompt=prompt, + message_id=new_id("tinp"), + skill_ids=skill_ids, + ) + self._title_from_first_prompt(prompt) + return TurnDelivery("queued", snapshot.turn) + + def has_active_turn(self): + return self.turns.active_for_thread(self.thread.id) is not None + + def rename_thread(self, title): + self.thread = from_view( + Thread, + self.rpc.call( + "thread/rename", {"threadId": self.thread.id, "title": title} + )["thread"], + ) + return self.thread + + def delete_session(self, session_id): + if session_id == self.thread.id: + raise RuntimeError("cannot delete the current Session; /new first") + self.rpc.call("thread/delete", {"threadId": session_id}) + + def last_terminal_turn(self): + return next( + ( + turn + for turn in reversed(self.turns.list_for_thread(self.thread.id)) + if turn.status.is_terminal + ), + None, + ) + + def retry_turn(self, turn_id): + return self.turns.retry(turn_id, use_current_selection=True).turn + + def interrupt(self): + active = self.turns.active_for_thread(self.thread.id) + return self.turns.interrupt(self.thread.id, active.id) if active else None + + def pending_approval(self): + active = self.turns.executing_for_thread(self.thread.id) + if active is None: + return None + return next( + ( + approval + for approval in self.turns.read(active.id).approvals + if approval.status is ApprovalStatus.PENDING + ), + None, + ) + + def respond_to_approval(self, approval_id, decision): + return from_view( + Approval, + self.rpc.call( + "approval/respond", + {"approvalId": approval_id, "decision": decision.value}, + )["approval"], + ) + + async def wait_until_idle(self): + while await asyncio.to_thread(self.has_active_turn): + await asyncio.sleep(0.05) + await self.drain_events_async() + await asyncio.sleep(0) + + def _require_idle(self): + if self.has_active_turn(): + raise RuntimeError( + "the current Turn is still active; stop it before changing Session" + ) + + def new_thread(self, *, title=""): + self._require_idle() + value = self.rpc.call( + "thread/start", + { + "projectId": self.project.id, + "title": title.strip() or "New task", + "connectionId": self.execution_profile.connection_id, + "model": self.execution_profile.model_id, + "reasoningEffort": self.thread.reasoning_effort, + "contextWindow": self.thread.context_window, + }, + ) + self._replace_thread(from_view(Thread, value["thread"])) + return self.thread + + def resume(self, session_id): + self._require_idle() + existing = from_view( + Thread, self.rpc.call("thread/read", {"threadId": session_id})["thread"] + ) + project = from_view( + Project, + self.rpc.call("project/read", {"projectId": existing.project_id})[ + "project" + ], + ) + if project.trust_state is not TrustState.TRUSTED: + raise ProjectNotTrustedError("Project is untrusted") + value = self.rpc.call( + "thread/resume", {"sessionId": session_id, "workspacePath": self.workspace} + ) + self._replace_thread(from_view(Thread, value["thread"])) + self._require_trust() + return self.thread + + def list_recent(self, *, limit, include_all): + result = self.rpc.call( + "thread/list", + {"cwd": None if include_all else self.workspace, "limit": limit}, + ) + listings = [] + for value in result["threads"]: + thread = from_view(Thread, value) + session = self.store.get_session(thread.id) + if session and session.messages: + listings.append( + ThreadListing( + thread.id, + thread.title, + len(session.messages), + thread.updated_at, + thread.workspace_path, + thread.id == self.thread.id, + ) + ) + return listings + + def switch_execution( + self, *, connection_id, model, reasoning_effort, context_window + ): + value = self.rpc.call( + "thread/execution/update", + { + "threadId": self.thread.id, + "connectionId": connection_id, + "model": model, + "reasoningEffort": reasoning_effort, + "contextWindow": context_window, + }, + ) + self.thread = from_view(Thread, value["thread"]) + profile = self.rpc.call("thread/execution/read", {"threadId": self.thread.id}) + self.execution_profile = ExecutionProfile.from_dict(profile["executionProfile"]) + return self.execution_profile + + def set_access_preset(self, access_preset): + value = self.rpc.call( + "thread/permission/update", + { + "threadId": self.thread.id, + "accessPreset": access_preset.value if access_preset else None, + "riskAcknowledged": access_preset is not None + and access_preset.value == "full_access", + }, + ) + self.thread = from_view(Thread, value["thread"]) + return self.thread + + def set_agent_preset(self, preset_id): + self.rpc.call( + "preset/select", {"threadId": self.thread.id, "agentPreset": preset_id} + ) + return self.refresh_thread() + + def current_agent_preset_id(self): + return self.rpc.call("preset/current", {"threadId": self.thread.id})[ + "agentPreset" + ] + + def refresh_thread(self): + thread_id = self.thread.id + thread = from_view( + Thread, self.rpc.call("thread/read", {"threadId": thread_id})["thread"] + ) + profile = self.rpc.call("thread/execution/read", {"threadId": thread_id}) + execution = ExecutionProfile.from_dict(profile["executionProfile"]) + security = ExecutionSecurityProfile.from_dict(profile["securityProfile"]) + if execution is None or security is None: + raise ValueError("Service returned an invalid execution profile") + if self.thread.id == thread_id: + self.thread, self.execution_profile, self._security = ( + thread, + execution, + security, + ) + return self.thread + + def clear_context(self): + self.rpc.call("thread/context/clear", {"threadId": self.thread.id}) + + async def compact_context(self): + return await asyncio.to_thread( + self.rpc.call, "thread/context/compact", {"threadId": self.thread.id} + ) + + def set_event_loop(self, loop): + # Events are consumed on the TUI loop by the bounded pump below. + pass + + def _consume_page(self, page): + for value in page["events"]: + event = from_view(DomainEvent, value) + if event.sequence <= self._sequence: + continue + if event.sequence != self._sequence + 1: + raise RuntimeError("Thread event replay is not contiguous") + if self._domain_sink: + self._domain_sink(event) + for rendered in self._renderer.convert(event): + if self._event_sink: + self._event_sink(rendered) + self._sequence = event.sequence + + def drain_events(self): + through = None + while True: + params = {"threadId": self.thread.id, "after": self._sequence, "limit": 100} + if through is not None: + params["through"] = through + page = self.rpc.call("event/replay", params) + through = page["headSequence"] + self._consume_page(page) + if not page["hasMore"]: + return + + async def drain_events_async(self): + through = None + generation = self._selection_generation + while True: + params = {"threadId": self.thread.id, "after": self._sequence, "limit": 100} + if through is not None: + if through <= self._sequence: + return + params["through"] = through + page = await asyncio.to_thread(self.rpc.call, "event/replay", params) + if generation != self._selection_generation: + return + through = page["headSequence"] + self._consume_page(page) + if not page["hasMore"]: + return + + async def start_domain_events(self, sink): + if self._domain_task is not None and not self._domain_task.done(): + return + self._domain_sink = sink + + async def pump(): + through = None + failures = 0 + while True: + thread_id, generation = self.thread.id, self._selection_generation + params = {"threadId": thread_id, "after": self._sequence, "limit": 100} + if through is not None and through > self._sequence: + params["through"] = through + try: + page = await asyncio.to_thread( + self.rpc.call, "event/replay", params + ) + failures = 0 + except (RuntimeError, OSError) as exc: + failures += 1 + if failures == 1 and self._event_sink: + self._event_sink( + Event( + "connection", + ErrorEvent( + f"Service disconnected: {exc}. Admitted tasks may still be running." + ), + ) + ) + if failures >= 6: + if self._event_sink: + self._event_sink( + Event( + "connection", + ErrorEvent( + "Use /reconnect after the service is available." + ), + ) + ) + return + await asyncio.sleep(min(4, 0.25 * 2**failures)) + continue + if generation != self._selection_generation: + through = None + continue + self._consume_page(page) + through = page["headSequence"] if page["hasMore"] else None + await asyncio.sleep(0.05) + + self._domain_task = asyncio.create_task(pump()) + + async def reconnect(self): + await self.stop_domain_events() + await asyncio.to_thread(self.rpc.reconnect) + await self.start_domain_events(self._domain_sink) + + async def stop_domain_events(self): + if self._domain_task is not None: + self._domain_task.cancel() + await asyncio.gather(self._domain_task, return_exceptions=True) + self._domain_task = None + + async def close(self): + await self.stop_domain_events() + await asyncio.to_thread(self.rpc.close) + + def _title_from_first_prompt(self, prompt): + if self.thread.title == "New task" and prompt.strip(): + try: + self.rename_thread(prompt.strip().splitlines()[0][:60]) + except (RuntimeError, OSError): + # Admission already succeeded. A presentation write cannot + # make the composer report that the submitted input failed. + pass diff --git a/cli/service_turn.py b/cli/service_turn.py new file mode 100644 index 000000000..230dfe540 --- /dev/null +++ b/cli/service_turn.py @@ -0,0 +1,111 @@ +"""Headless service attachment preserving foreground exit and approval behavior.""" + +from __future__ import annotations + +import asyncio +import sys +import time +import threading + +from cli.thread_client import HeadlessTurnOptions, HeadlessTurnResult +from cli.service_thread_client import ServiceThreadClient +from core.domain.approval import ApprovalStatus +from core.domain.common import new_id + + +def run_service_turn( + options: HeadlessTurnOptions, + *, + on_event=None, + decide_approval=None, + detach=False, + detach_requested=None, +) -> HeadlessTurnResult: + prompt = options.prompt.strip() + if not prompt: + raise ValueError("prompt must not be empty") + client = ServiceThreadClient( + workspace=options.workspace, + model=options.model, + connection_id=options.connection_id, + reasoning_effort=options.reasoning_effort, + max_iterations=options.max_iterations, + streaming=False, + trust_workspace=options.trust_workspace, + resume_id=options.resume_id, + event_sink=on_event, + surface="headless", + ) + interrupted = False + try: + if options.access_preset is not None: + client.set_access_preset(options.access_preset) + if options.agent_preset is not None: + client.set_agent_preset(options.agent_preset) + skill_ids = tuple( + client.skills.select(client.project.id, value).id + for value in options.skill_identifiers + ) + snapshot = client.turns.start( + client.thread.id, + prompt=prompt, + message_id=new_id("tinp"), + skill_ids=skill_ids, + connection_id=options.connection_id, + model=options.model, + reasoning_effort=options.reasoning_effort, + ) + if not detach: + handled = set() + while not snapshot.turn.status.is_terminal: + if detach_requested is not None and detach_requested.is_set(): + raise InterruptedError( + "Client detached; the task continues in the service" + ) + client.drain_events() + for approval in snapshot.approvals: + if ( + approval.status is ApprovalStatus.PENDING + and approval.id not in handled + ): + handled.add(approval.id) + decision = ( + decide_approval(approval) + if decide_approval + else ApprovalStatus.DENIED + ) + client.respond_to_approval(approval.id, decision) + time.sleep(0.05) + snapshot = client.turns.read(snapshot.turn.id) + client.drain_events() + return HeadlessTurnResult(snapshot.turn, client.thread.id, client.workspace) + except KeyboardInterrupt: + interrupted = True + try: + client.interrupt() + except Exception: # transport failure must not replace the user's exit 130 + print( + "Task interruption could not be confirmed; reconnect to inspect its state.", + file=sys.stderr, + ) + raise + finally: + try: + asyncio.run(client.close()) + except Exception: + if not interrupted: + raise + print("The client connection could not finish closing.", file=sys.stderr) + + +async def run_service_turn_async( + options: HeadlessTurnOptions, **kwargs +) -> HeadlessTurnResult: + """Cancel the client waiter without cancelling an already admitted Turn.""" + detached = threading.Event() + try: + return await asyncio.to_thread( + run_service_turn, options, detach_requested=detached, **kwargs + ) + finally: + detached.set() diff --git a/cli/service_turns.py b/cli/service_turns.py new file mode 100644 index 000000000..ce29ad100 --- /dev/null +++ b/cli/service_turns.py @@ -0,0 +1,131 @@ +"""Typed Turn command port; shares the original InteractiveTurnRouter.""" + +from __future__ import annotations + +from cli.rpc_models import from_view +from core.application.turn_input_service import TurnInputReceipt +from core.application.turn_service import TurnSnapshot +from core.domain.turn import Turn + + +class ServiceTurns: + def __init__(self, rpc): + self.rpc = rpc + + def start( + self, + thread_id, + *, + prompt, + message_id, + skill_ids=(), + client_surface=None, + event_observer=None, + connection_id=None, + model=None, + reasoning_effort=None, + ): + return from_view( + TurnSnapshot, + self.rpc.call( + "turn/start", + { + "threadId": thread_id, + "prompt": prompt, + "messageId": message_id, + "skills": list(skill_ids), + **( + {"connectionId": connection_id} + if connection_id is not None + else {} + ), + **({"model": model} if model is not None else {}), + **( + {"reasoningEffort": reasoning_effort} + if reasoning_effort is not None + else {} + ), + }, + ), + ) + + def enqueue( + self, + thread_id, + *, + prompt, + message_id, + skill_ids=(), + client_surface=None, + event_observer=None, + ): + return from_view( + TurnSnapshot, + self.rpc.call( + "turn/enqueue", + { + "threadId": thread_id, + "prompt": prompt, + "messageId": message_id, + "skills": list(skill_ids), + }, + ), + ) + + def steer( + self, thread_id, *, expected_turn_id, prompt, message_id, client_surface=None + ): + return from_view( + TurnInputReceipt, + self.rpc.call( + "turn/steer", + { + "threadId": thread_id, + "expectedTurnId": expected_turn_id, + "prompt": prompt, + "messageId": message_id, + }, + ), + ) + + def read(self, turn_id): + return from_view(TurnSnapshot, self.rpc.call("turn/read", {"turnId": turn_id})) + + def list_for_thread(self, thread_id): + turns = [] + while True: + page = self.rpc.call( + "turn/list", {"threadId": thread_id, "offset": len(turns), "limit": 100} + ) + turns.extend(from_view(Turn, value) for value in page["turns"]) + if not page["hasMore"]: + return turns + if not page["turns"]: + raise RuntimeError("Turn listing made no progress") + + def active_for_thread(self, thread_id): + return self._first(thread_id, "active") + + def executing_for_thread(self, thread_id): + return self._first(thread_id, "executing") + + def _first(self, thread_id, state): + page = self.rpc.call( + "turn/list", {"threadId": thread_id, "state": state, "limit": 1} + ) + return from_view(Turn, page["turns"][0]) if page["turns"] else None + + def interrupt(self, thread_id, turn_id): + value = self.rpc.call( + "turn/interrupt", {"threadId": thread_id, "turnId": turn_id} + ) + return value["accepted"], from_view(Turn, value["turn"]) + + def retry(self, turn_id, *, use_current_selection=False): + return from_view( + TurnSnapshot, + self.rpc.call( + "turn/retry", + {"turnId": turn_id, "useCurrentSelection": use_current_selection}, + ), + ) diff --git a/cli/thread_client.py b/cli/thread_client.py new file mode 100644 index 000000000..1c86c6ec3 --- /dev/null +++ b/cli/thread_client.py @@ -0,0 +1,133 @@ +"""The command boundary consumed by TUI presentation.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Protocol + +from core.domain.approval import Approval, ApprovalStatus +from core.domain.event import DomainEvent +from core.domain.execution_profile import ExecutionProfile +from core.domain.execution_security import ExecutionAccessPreset +from core.domain.project import Project +from core.domain.thread import Thread +from core.domain.turn import Turn +from core.sessions import SessionStore + + +@dataclass(frozen=True, slots=True) +class HeadlessTurnOptions: + prompt: str + workspace: str | None = None + resume_id: str | None = None + connection_id: str | None = None + model: str | None = None + reasoning_effort: str | None = None + skill_identifiers: tuple[str, ...] = () + max_iterations: int | None = None + trust_workspace: bool = False + access_preset: ExecutionAccessPreset | None = None + agent_preset: str | None = None + + +@dataclass(frozen=True, slots=True) +class HeadlessTurnResult: + turn: Turn + session_id: str + workspace: str + + +@dataclass(frozen=True, slots=True) +class TurnDelivery: + kind: str + turn: Turn + + +@dataclass(frozen=True, slots=True) +class ThreadListing: + """One row of the resume picker — display data only.""" + + session_id: str + title: str + message_count: int + updated_at: datetime + workspace: str + is_current: bool + + +def turn_access_summary(turn: Turn) -> str: + profile = turn.execution_security_profile + if profile is not None: + if profile.access_preset is not None: + return profile.access_preset.value.replace("_", " ") + sandbox = "sandboxed" if profile.command_sandbox else "unsandboxed" + return f"legacy {profile.permission_mode.value.replace('_', ' ')} · {sandbox}" + if turn.execution_permission_mode is not None: + return f"legacy {turn.execution_permission_mode.value.replace('_', ' ')}" + return "legacy unknown" + + +class ThreadClient(Protocol): + """Presentation consumes commands and snapshots, never an execution owner.""" + + runtime_mode: str + workspace: str + thread: Thread + project: Project + execution_profile: ExecutionProfile + store: SessionStore + llm: Any + skills: Any + plugins: Any + mcp: Any + goals: Any + + @property + def session_id(self) -> str: ... + @property + def project_trusted(self) -> bool: ... + @property + def access_preset_override(self) -> ExecutionAccessPreset | None: ... + def access_summary(self) -> str: ... + def frozen_access_summaries(self) -> tuple[str | None, tuple[str, ...]]: ... + def send(self, prompt: str, *, skill_ids: tuple[str, ...] = ()) -> Any: ... + def queue(self, prompt: str, *, skill_ids: tuple[str, ...] = ()) -> Any: ... + def has_active_turn(self) -> bool: ... + def rename_thread(self, title: str) -> Thread: ... + def delete_session(self, session_id: str) -> None: ... + def last_terminal_turn(self) -> Turn | None: ... + def retry_turn(self, turn_id: str) -> Turn: ... + def interrupt(self) -> tuple[bool, Turn] | None: ... + def pending_approval(self) -> Approval | None: ... + def respond_to_approval( + self, approval_id: str, decision: ApprovalStatus + ) -> Approval: ... + async def wait_until_idle(self) -> None: ... + def new_thread(self, *, title: str = "") -> Thread: ... + def resume(self, session_id: str) -> Thread: ... + def list_recent(self, *, limit: int, include_all: bool) -> list[Any]: ... + def switch_execution( + self, + *, + connection_id: str | None, + model: str | None, + reasoning_effort: str | None, + context_window: int | None, + ) -> ExecutionProfile: ... + def set_access_preset( + self, access_preset: ExecutionAccessPreset | None + ) -> Thread: ... + def set_agent_preset(self, preset_id: str | None) -> Thread: ... + def current_agent_preset_id(self) -> str | None: ... + def refresh_thread(self) -> Thread: ... + def clear_context(self) -> None: ... + async def compact_context(self) -> dict: ... + def set_event_loop(self, loop: asyncio.AbstractEventLoop) -> None: ... + async def start_domain_events( + self, sink: Callable[[DomainEvent], None] + ) -> None: ... + async def stop_domain_events(self) -> None: ... + async def close(self) -> None: ... diff --git a/cli/tui/app.py b/cli/tui/app.py index 92d86571d..b84bd16d4 100644 --- a/cli/tui/app.py +++ b/cli/tui/app.py @@ -44,13 +44,12 @@ require_project_trusted, set_project_trusted, ) +from cli.thread_client import ThreadClient from cli.tui import commands, theme from cli.tui.goal_controller import TuiGoalController from cli.tui.input import InputInterrupted, InputReader, expand_file_refs from cli.tui.renderer import EventRenderer from cli.tui.session_bridge import SessionBridge -from cli.tui.thread_client import TuiThreadClient -from core.application.application import DeepCodeApplication from core.application.errors import ApplicationError from core.config import ConfigError from core.domain.approval import ApprovalStatus @@ -60,7 +59,6 @@ from core.file_lock import FileLease from core.providers.reasoning import normalize_reasoning_effort - _MODEL_CATALOG_PREVIEW = 4 # models shown per connection in /model # Sentinel: switch_model keeps the session's effort unless told otherwise. _KEEP_EFFORT: object = object() @@ -82,6 +80,7 @@ def __init__( trust_workspace: bool = False, access_preset: ExecutionAccessPreset | None = None, resume_id: str | None = None, + shared_service: bool = True, ) -> None: self.workspace = os.path.abspath(workspace) self.max_iterations = max_iterations @@ -94,6 +93,8 @@ def __init__( activity_probe=self.renderer.is_working, ) self._exit_requested = False + self._external_refresh_task: asyncio.Task | None = None + self._external_refresh_dirty = False self._last_send_delivered = True self._requested_model = model self._requested_connection = connection_id @@ -106,7 +107,15 @@ def __init__( self.selected_skill_ids: list[str] = [] self._session_activity: FileLease | None = None self._leased_session_id: str | None = None - self.thread_client = TuiThreadClient( + if shared_service: + from cli.service_thread_client import ServiceThreadClient + + client_type = ServiceThreadClient + else: + from cli.tui.thread_client import TuiThreadClient + + client_type = TuiThreadClient + self.thread_client: ThreadClient = client_type( workspace=self.workspace, model=self._requested_model, connection_id=self._requested_connection, @@ -257,7 +266,7 @@ def reasoning_options( Turn resolves its capabilities from, so the picker never advertises a ladder the switch would reject. """ - capabilities = self.thread_client.application.llm.model_reasoning( + capabilities = self.thread_client.llm.model_reasoning( connection_id or self.thread_client.execution_profile.connection_id, model_id, project_id=self.thread_client.project.id, @@ -292,9 +301,7 @@ async def switch_context_window(self, context_window: int | None) -> None: self._requested_context_window = context_window def connection_views(self) -> list[dict]: - data = self.thread_client.application.llm.list_connections( - self.thread_client.project.id - ) + data = self.thread_client.llm.list_connections(self.thread_client.project.id) return list(data.get("connections", [])) def model_overview(self) -> str: @@ -328,7 +335,7 @@ def model_overview(self) -> str: extra = len(models) - _MODEL_CATALOG_PREVIEW catalog = shown + (f", +{extra} more" if extra > 0 else "") else: - catalog = "no catalog configured — any model id accepted" + catalog = f"browse models with deepcode provider models {view['id']}" marker = " · current" if view.get("id") == profile.connection_id else "" lines.append(f" {str(view.get('id', '')):<{width}} {catalog}{marker}") return "\n".join(lines) @@ -342,7 +349,7 @@ def connection_model_catalog(self) -> list[tuple[str, list[dict]]]: offer one directory. A connection whose catalog cannot be read contributes nothing instead of failing the whole picker. """ - llm = self.thread_client.application.llm + llm = self.thread_client.llm catalog: list[tuple[str, list[dict]]] = [] for view in self.connection_views(): if not (view.get("configured") and view.get("enabled")): @@ -479,7 +486,7 @@ def set_agent_preset(self, preset_id: str | None) -> str: def list_skills(self) -> str: try: - skills = self.thread_client.application.skills.list( + skills = self.thread_client.skills.list( self.thread_client.project.id ).skills except (ApplicationError, OSError, ValueError) as exc: @@ -495,7 +502,7 @@ def list_skills(self) -> str: def select_skill(self, identifier: str) -> str: try: - skill = self.thread_client.application.skills.select( + skill = self.thread_client.skills.select( self.thread_client.project.id, identifier, ) @@ -511,7 +518,7 @@ def select_skill(self, identifier: str) -> str: def remove_skill(self, identifier: str) -> str: try: - skills = self.thread_client.application.skills.list( + skills = self.thread_client.skills.list( self.thread_client.project.id ).skills except (ApplicationError, OSError, ValueError) as exc: @@ -532,9 +539,7 @@ def remove_skill(self, identifier: str) -> str: return f"removed {skill.name} from the next turn" def _completion_skills(self): - return self.thread_client.application.skills.list( - self.thread_client.project.id - ).skills + return self.thread_client.skills.list(self.thread_client.project.id).skills def _complete_command_argument(self, name: str, prefix: str) -> tuple[str, ...]: command = commands.REGISTRY.get(name) @@ -551,7 +556,7 @@ def clear_skills(self) -> str: def list_plugins(self) -> str: try: - discovery = self.thread_client.application.plugins.list() + discovery = self.thread_client.plugins.list() except (ApplicationError, OSError, ValueError) as exc: return f"Plugin error: {exc}" if not discovery.plugins: @@ -570,9 +575,7 @@ def list_plugins(self) -> str: def list_mcp_servers(self) -> str: try: - inventory = self.thread_client.application.mcp.list( - self.thread_client.project.id - ) + inventory = self.thread_client.mcp.list(self.thread_client.project.id) except (ApplicationError, OSError, ValueError) as exc: return f"MCP error: {exc}" if not inventory.servers: @@ -601,7 +604,7 @@ async def manage_mcp(self, args: str) -> str: return usage try: if action == "presets": - presets = self.thread_client.application.mcp.list_presets(project_id) + presets = self.thread_client.mcp.list_presets(project_id) lines = ["", "Bundled MCP presets (add with /mcp add <id>):"] for preset in presets.presets: status = "configured" if preset.configured else "available" @@ -618,7 +621,7 @@ async def manage_mcp(self, args: str) -> str: return usage if action == "add": await asyncio.to_thread( - self.thread_client.application.mcp.add_preset, + self.thread_client.mcp.add_preset, target, project_id=project_id, ) @@ -628,7 +631,7 @@ async def manage_mcp(self, args: str) -> str: ) if action == "test": result = await asyncio.to_thread( - self.thread_client.application.mcp.probe, + self.thread_client.mcp.probe, target, project_id=project_id, ) @@ -640,7 +643,7 @@ async def manage_mcp(self, args: str) -> str: ) if action == "login": flow = await asyncio.to_thread( - self.thread_client.application.mcp.oauth_start, + self.thread_client.mcp.oauth_start, target, project_id=project_id, open_browser=True, @@ -655,7 +658,7 @@ async def manage_mcp(self, args: str) -> str: return f"Browser authorization started for {target}.{suffix}" if action == "logout": removed = await asyncio.to_thread( - self.thread_client.application.mcp.oauth_logout, + self.thread_client.mcp.oauth_logout, target, project_id=project_id, ) @@ -666,7 +669,7 @@ async def manage_mcp(self, args: str) -> str: ) if action == "cancel": await asyncio.to_thread( - self.thread_client.application.mcp.oauth_cancel, + self.thread_client.mcp.oauth_cancel, target, project_id=project_id, ) @@ -675,13 +678,13 @@ async def manage_mcp(self, args: str) -> str: if action != "remove": enabled = action == "enable" await asyncio.to_thread( - self.thread_client.application.mcp.set_enabled, + self.thread_client.mcp.set_enabled, target, enabled=enabled, project_id=project_id, ) return f"{'enabled' if enabled else 'disabled'} MCP server {target}" - inventory = self.thread_client.application.mcp.list(project_id) + inventory = self.thread_client.mcp.list(project_id) matches = [ server for server in inventory.servers @@ -694,7 +697,7 @@ async def manage_mcp(self, args: str) -> str: return "Plugin MCP servers are managed through /plugins" scope = "project" if server.source == "project" else "user" await asyncio.to_thread( - self.thread_client.application.mcp.remove, + self.thread_client.mcp.remove, name=server.name, scope=scope, project_id=project_id, @@ -711,9 +714,18 @@ async def run_goal_command(self, args: str) -> str: return result.message async def _reload_current_session(self) -> None: - self.thread_client.refresh_thread() + await asyncio.to_thread(self.thread_client.refresh_thread) self._sync_thread_state() + async def reconnect_service(self) -> str: + if self.thread_client.runtime_mode != "service": + return "This connection does not support reconnection." + try: + await self.thread_client.reconnect() + return "Reconnected to the shared service." + except (RuntimeError, OSError) as exc: + return f"Reconnect failed: {exc}" + def request_exit(self) -> None: self._exit_requested = True @@ -807,6 +819,31 @@ def _respond_to_pending_approval(self, text: str) -> str | None: return f"Approval {outcome}." def _on_domain_event(self, event: DomainEvent) -> None: + if self.thread_client.runtime_mode == "service" and event.type.startswith( + "thread." + ): + thread = event.payload.get("thread") + if ( + isinstance(thread, dict) + and thread.get("id") == self.thread_client.session_id + ): + self._external_refresh_dirty = True + if ( + self._external_refresh_task is None + or self._external_refresh_task.done() + ): + + async def refresh(): + try: + while self._external_refresh_dirty: + self._external_refresh_dirty = False + await self._reload_current_session() + except (ApplicationError, OSError, ValueError) as exc: + self.console.print( + f"Service state refresh failed: {escape(str(exc))}" + ) + + self._external_refresh_task = asyncio.create_task(refresh()) if event.type != "approval.requested": return approval = event.payload.get("approval") @@ -858,6 +895,7 @@ def _banner(self) -> None: ) self.console.print( f" [{theme.META_STYLE}]session {escape(self.bridge.session_id)} · " + f"runtime {self.thread_client.runtime_mode} · " f"access {escape(self.thread_client.access_summary())} · " f"effort {escape(self.requested_reasoning_effort)}[/]", soft_wrap=True, @@ -876,6 +914,7 @@ async def repl(self) -> int: if self.reader.interactive: self._banner() try: + self.render_resume_tail() while not self._exit_requested: try: line = await self.reader.read() @@ -929,6 +968,11 @@ async def repl(self) -> int: self.console.print(f"[{theme.META_STYLE}]bye[/]") return 0 finally: + if self._external_refresh_task is not None: + self._external_refresh_task.cancel() + await asyncio.gather( + self._external_refresh_task, return_exceptions=True + ) self.goal_controller.close() await self.thread_client.close() if self._session_activity is not None: @@ -937,7 +981,7 @@ async def repl(self) -> int: self._leased_session_id = None -def main(argv: list[str] | None = None) -> int: +def main(argv: list[str] | None = None, *, shared_service: bool = True) -> int: parser = argparse.ArgumentParser( prog="deepcode", description="Interactive DeepCode coding agent (multi-turn TUI).", @@ -949,16 +993,17 @@ def main(argv: list[str] | None = None) -> int: add_access_preset_argument(parser) add_workspace_trust_argument(parser) parser.add_argument("--resume", "-r", default=None, help="Session id to resume.") - parser.add_argument( - "--max-iterations", - type=int, - default=None, - help="Optional model-sampling limit for diagnostics (unlimited by default).", - ) + parser.set_defaults(max_iterations=None) + if not shared_service: + parser.add_argument( + "--max-iterations", type=int, help="Optional model-sampling limit." + ) args = parser.parse_args(argv) _bootstrap_quiet_logging() - if not _prepare_workspace_trust(args.workspace, grant=args.trust): + if not shared_service and not _prepare_workspace_trust( + args.workspace, grant=args.trust + ): return 1 try: @@ -971,6 +1016,7 @@ def main(argv: list[str] | None = None) -> int: trust_workspace=args.trust, access_preset=parse_access_preset(args.access), resume_id=args.resume, + shared_service=shared_service, ) except ConfigError as exc: print(format_config_error(exc), file=sys.stderr) @@ -1034,6 +1080,8 @@ def _silence_console_logging() -> None: def _prepare_workspace_trust(workspace: str, *, grant: bool) -> bool: """Persist trust before creating a Session, avoiding empty denied threads.""" + from core.application.application import DeepCodeApplication + application = DeepCodeApplication.open( host_surface="cli-trust", run_automation_scheduler=False, diff --git a/cli/tui/commands.py b/cli/tui/commands.py index 6a7d1bbd5..82e8c3356 100644 --- a/cli/tui/commands.py +++ b/cli/tui/commands.py @@ -16,8 +16,8 @@ from datetime import UTC, datetime from typing import Any -from cli.transcript import TranscriptMode from cli.execution_options import parse_context_window +from cli.transcript import TranscriptMode from cli.tui import theme from cli.tui.picker import Picker, PickerItem, PickerScope, PickerVariant from cli.tui.text import fit_head, short_path @@ -41,6 +41,10 @@ class Command: _HELP_USAGE_COLUMN_CAP = 30 +async def _cmd_reconnect(app, args: str) -> str: + return await app.reconnect_service() + + async def _cmd_help(app, args: str) -> str | None: """Aligned command table: the usage column fits the widest short usage, and an over-long usage moves its description to a wrapped second line @@ -681,9 +685,7 @@ async def _cmd_skills(app, args: str) -> str | None: async def _skill_via_picker(app) -> str | None: try: - skills = app.thread_client.application.skills.list( - app.thread_client.project.id - ).skills + skills = app.thread_client.skills.list(app.thread_client.project.id).skills except (OSError, RuntimeError, ValueError) as exc: return f"Skill listing failed: {exc}" selected = set(app.selected_skill_ids) @@ -760,6 +762,9 @@ async def _cmd_exit(app, args: str) -> str | None: c.name: c for c in ( Command("help", "/help", "show this help", _cmd_help), + Command( + "reconnect", "/reconnect", "Reconnect to the shared service", _cmd_reconnect + ), Command("new", "/new [title]", "start a new conversation", _cmd_new), Command( "resume", diff --git a/cli/tui/goal_controller.py b/cli/tui/goal_controller.py index bfc38a23c..6d6e7005a 100644 --- a/cli/tui/goal_controller.py +++ b/cli/tui/goal_controller.py @@ -7,7 +7,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from core.application.application import DeepCodeApplication from core.application.errors import ApplicationError from core.config import ConfigError from core.domain.message_provenance import ClientSurface @@ -211,15 +210,15 @@ def request_stop() -> None: refresh_session=True, ) - def _context(self) -> tuple[DeepCodeApplication, str]: + def _context(self): return ( - self.owner.thread_client.application, + self.owner.thread_client, self.owner.thread_client.session_id, ) @staticmethod def _required( - application: DeepCodeApplication, + application, thread_id: str, ) -> ThreadGoal: goal = application.goals.read(thread_id) diff --git a/cli/tui/thread_client.py b/cli/tui/thread_client.py index bc76c2c06..62c1326b1 100644 --- a/cli/tui/thread_client.py +++ b/cli/tui/thread_client.py @@ -4,9 +4,8 @@ import asyncio from collections.abc import Callable -from dataclasses import dataclass -from datetime import datetime +from cli.thread_client import ThreadListing, TurnDelivery, turn_access_summary from cli.project_trust import ( open_workspace_project, require_project_trusted, @@ -27,7 +26,6 @@ from core.domain.execution_profile import ExecutionProfile, ExecutionSelection from core.domain.execution_security import ( ExecutionAccessPreset, - ExecutionSecurityProfile, ) from core.domain.message_provenance import ClientSurface from core.domain.project import TrustState @@ -38,24 +36,6 @@ from core.sessions import SessionStore, get_default_store -@dataclass(frozen=True, slots=True) -class TuiDelivery: - kind: str - turn: Turn - - -@dataclass(frozen=True, slots=True) -class ThreadListing: - """One row of the resume picker — display data only.""" - - session_id: str - title: str - message_count: int - updated_at: datetime - workspace: str - is_current: bool - - class TuiThreadClient: """Own one application and expose only interactive CLI operations.""" @@ -123,6 +103,28 @@ def __init__( self.application.close() raise + runtime_mode = "compatibility" + + @property + def llm(self): + return self.application.llm + + @property + def skills(self): + return self.application.skills + + @property + def plugins(self): + return self.application.plugins + + @property + def mcp(self): + return self.application.mcp + + @property + def goals(self): + return self.application.goals + @property def session_id(self) -> str: return self.thread.id @@ -162,8 +164,8 @@ def frozen_access_summaries(self) -> tuple[str | None, tuple[str, ...]]: ) queued = tuple(turn for turn in turns if turn.status is TurnStatus.QUEUED) return ( - _turn_access_summary(current) if current is not None else None, - tuple(_turn_access_summary(turn) for turn in queued), + turn_access_summary(current) if current is not None else None, + tuple(turn_access_summary(turn) for turn in queued), ) def set_event_loop(self, loop: asyncio.AbstractEventLoop) -> None: @@ -211,7 +213,7 @@ def send( prompt: str, *, skill_ids: tuple[str, ...] = (), - ) -> TuiDelivery: + ) -> TurnDelivery: active = self.application.turns.executing_for_thread(self.thread.id) result: InteractiveTurnResult = self.router.send( self.thread.id, @@ -225,14 +227,14 @@ def send( InteractiveDelivery.QUEUED, }: self._title_from_first_prompt(prompt) - return TuiDelivery(result.delivery.value, result.turn) + return TurnDelivery(result.delivery.value, result.turn) def queue( self, prompt: str, *, skill_ids: tuple[str, ...] = (), - ) -> TuiDelivery: + ) -> TurnDelivery: snapshot = self.application.turns.enqueue( self.thread.id, prompt=prompt, @@ -241,7 +243,7 @@ def queue( client_surface=ClientSurface.CLI, ) self._title_from_first_prompt(prompt) - return TuiDelivery("queued", snapshot.turn) + return TurnDelivery("queued", snapshot.turn) def has_active_turn(self) -> bool: return self.application.turns.active_for_thread(self.thread.id) is not None @@ -568,16 +570,4 @@ def _require_idle(self) -> None: ) -def _turn_access_summary(turn: Turn) -> str: - profile: ExecutionSecurityProfile | None = turn.execution_security_profile - if profile is not None: - if profile.access_preset is not None: - return profile.access_preset.value.replace("_", " ") - sandbox = "sandboxed" if profile.command_sandbox else "unsandboxed" - return f"legacy {profile.permission_mode.value.replace('_', ' ')} · {sandbox}" - if turn.execution_permission_mode is not None: - return f"legacy {turn.execution_permission_mode.value.replace('_', ' ')}" - return "legacy unknown" - - -__all__ = ["TuiDelivery", "TuiThreadClient"] +__all__ = ["TuiThreadClient"] diff --git a/cli/web_cli.py b/cli/web_cli.py new file mode 100644 index 000000000..1e9be16ff --- /dev/null +++ b/cli/web_cli.py @@ -0,0 +1,50 @@ +"""Open the packaged browser client using an instance-local one-time link.""" + +from __future__ import annotations + +import argparse +import json +import webbrowser +from pathlib import Path + +from app_server.service_client import ServiceClient +from app_server.service_state import ServiceFiles +from app_server.web_surface import read_web_build +from cli.service_cli import start_service +from core.persistence.database import default_database_path + + +def run(argv=None) -> int: + parser = argparse.ArgumentParser(prog="deepcode web") + parser.add_argument("--database", type=Path, default=default_database_path()) + parser.add_argument("--port", type=int, default=None) + parser.add_argument( + "--no-open", + action="store_true", + help="Print the short-lived link without opening a browser", + ) + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + files = ServiceFiles(args.database) + if not files.running() and read_web_build() is None: + parser.error( + "Web assets are missing or incompatible. Install a complete release, " + "or run npm run build:web in desktop/." + ) + status = start_service(files, port=args.port) + ticket = ServiceClient(files).call("auth/issue", instance_id=status["instanceId"]) + url = status["url"] + "/#ticket=" + ticket["ticket"] + print( + json.dumps( + { + "url": url, + "expiresIn": ticket["expiresIn"], + "instanceId": status["instanceId"], + } + ) + if args.json + else f"Open within {ticket['expiresIn']} seconds (one use):\n{url}" + ) + if not args.no_open: + webbrowser.open(url) + return 0 diff --git a/core/agent_runtime/injections.py b/core/agent_runtime/injections.py index 025f9e0f8..12cca1204 100644 --- a/core/agent_runtime/injections.py +++ b/core/agent_runtime/injections.py @@ -255,6 +255,9 @@ def deactivate(self, turn_id: str) -> None: return self._active_turn_id = None self._state = MailboxState.CLOSED + for entry in self._pending: + if not entry.ready: + self._seen.pop(entry.message_id, None) self._pending.clear() self._pending_by_id.clear() self._pending_chars = 0 @@ -273,13 +276,17 @@ def reserve(self, value: TurnRuntimeInput) -> TurnInputReservation | None: # message is appended. Wait for that ordering boundary rather # than rejecting a valid early Steer or persisting it first. self._condition.wait() - previous = self._seen.get(clean_value.message_id) - if previous is not None: + while (previous := self._seen.get(clean_value.message_id)) is not None: if previous != digest: raise TurnInputConflictError( "message_id was already used with different content" ) - return None + pending = self._pending_by_id.get(clean_value.message_id) + if pending is None or pending.ready: + return None + # A reservation is not an acceptance. Wait for commit/cancel, + # releasing the condition so the original producer can finish. + self._condition.wait() if self._state is not MailboxState.OPEN: raise TurnInputClosedError(self._state) if self._active_turn_id != clean_value.target_turn_id: @@ -319,9 +326,10 @@ def commit(self, reservation: TurnInputReservation) -> None: def cancel(self, reservation: TurnInputReservation) -> None: with self._condition: - entry = self._pending_by_id.pop(reservation.message_id, None) - if entry is None: + entry = self._pending_by_id.get(reservation.message_id) + if entry is None or entry.ready: return + self._pending_by_id.pop(reservation.message_id) self._pending = deque( candidate for candidate in self._pending diff --git a/core/agent_setup.py b/core/agent_setup.py index 43beb1e28..ddfec4960 100644 --- a/core/agent_setup.py +++ b/core/agent_setup.py @@ -17,6 +17,7 @@ from __future__ import annotations import inspect +from collections.abc import Awaitable, Callable import os from typing import Any @@ -258,6 +259,7 @@ def build_agent_session( permission_mode_override: PermissionMode | None = None, execution_security_profile: ExecutionSecurityProfile | None = None, runtime: DeepCodeRuntime | None = None, + provider_cleanup: Callable[[], Awaitable[None]] | None = None, skill_runtime: Any | None = None, project_trusted: bool = False, plugin_mcp_servers: tuple[Any, ...] = (), @@ -499,6 +501,7 @@ def _permission_checker(name: str, arguments: dict[str, Any]): goal_runtime.closure_prompt if goal_runtime is not None else None ), mcp_runtime=mcp_runtime, + provider_cleanup=provider_cleanup, ) if control is not None: # `session.history` is a @property (a list), so it must be wrapped in a diff --git a/core/application/agent_adapter.py b/core/application/agent_adapter.py index e6705f9b0..6df90af78 100644 --- a/core/application/agent_adapter.py +++ b/core/application/agent_adapter.py @@ -136,7 +136,10 @@ def create( options["tool_filter"] = preset_filter if agent_preset.allow_spawn is not None: options["allow_spawn"] = agent_preset.allow_spawn - runtime = DeepCodeRuntime(load_config_for_workspace(workspace)) + runtime = DeepCodeRuntime( + load_config_for_workspace(workspace), + config_loader=lambda: load_config_for_workspace(workspace), + ) plugin_mcp_servers = ( self._plugin_mcp_provider(Path(workspace)) if self._plugin_mcp_provider is not None @@ -163,6 +166,7 @@ def create( else None ), runtime=runtime, + provider_cleanup=runtime.aclose, project_trusted=True, plugin_mcp_servers=plugin_mcp_servers, mcp_status_observer=self._mcp_status_observer, @@ -189,6 +193,12 @@ def runtime_key( ( execution_profile.connection_id, execution_profile.config_revision, + execution_profile.provider_revision, + execution_profile.protocol, + execution_profile.model_id, + execution_profile.input_modalities, + execution_profile.tool_calling, + execution_profile.reasoning_supported, execution_profile.context_window, execution_profile.max_output_tokens, execution_profile.max_tokens, diff --git a/core/application/application.py b/core/application/application.py index 998ef86e7..84aab8536 100644 --- a/core/application/application.py +++ b/core/application/application.py @@ -302,6 +302,9 @@ def open( event_relay_batch_size=event_relay_batch_size, ) application._application_lease = lease + if database.restore_recovery_marker.exists(): + application._pause_restored_scheduling() + database.restore_recovery_marker.unlink() application.event_relay.start() application.execution_coordinator.start(background=False) if lease.recovery_owner: @@ -329,6 +332,35 @@ def open( lease.close() raise + def _pause_restored_scheduling(self) -> None: + """A data rollback cannot roll back tool effects in project directories.""" + from core.domain.automation import ( + AutomationActivationStatus, + AutomationScheduleKind, + AutomationStatus, + ) + from core.domain.thread_goal import ThreadGoalStatus + + for goal in self.goals.store.list_current(): + if goal.status is ThreadGoalStatus.ACTIVE: + self.goals.pause(goal.thread_id, expected_goal_id=goal.id) + definitions = [] + offset = 0 + while True: + page = self.automations.list(limit=100, offset=offset) + definitions.extend(page.automations) + if not page.has_more: + break + offset = page.next_offset + for definition in definitions: + if ( + definition.status is AutomationStatus.ENABLED + and definition.schedule_kind is AutomationScheduleKind.INTERVAL + ): + self.automations.update( + definition.id, status=AutomationActivationStatus.PAUSED + ) + def close(self) -> None: errors: list[Exception] = [] @@ -347,6 +379,7 @@ def attempt(stage: str, operation) -> None: "execution runtime", lambda: self.executions.close(cleanup=self.turns.close_live_sessions), ) + attempt("Provider login", self.llm.close) attempt("Plugin service", self.plugins.close) attempt("Skill service", self.skills.close) attempt("Skill workspace registry", self.skill_hosts.close) diff --git a/core/application/config_store.py b/core/application/config_store.py index fca25428e..7b52acd43 100644 --- a/core/application/config_store.py +++ b/core/application/config_store.py @@ -6,14 +6,17 @@ import json import os import threading -import uuid from collections.abc import Callable, Iterator from contextlib import contextmanager from pathlib import Path from typing import Any from core.config import DeepCodeConfig, home_config_path -from core.private_storage import ensure_private_directory, open_private_file +from core.private_storage import ( + atomic_write_private_json, + ensure_private_directory, + open_private_file, +) class ConfigRevisionConflict(RuntimeError): @@ -85,29 +88,7 @@ def mutate( return updated def _replace(self, value: dict[str, Any]) -> None: - ensure_private_directory(self.path.parent) - temporary = self.path.with_name(f".{self.path.name}.{uuid.uuid4().hex}.tmp") - payload = (json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode() - descriptor = open_private_file( - temporary, - os.O_WRONLY | os.O_CREAT | os.O_EXCL, - ) - try: - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, self.path) - try: - os.chmod(self.path, 0o600) - except OSError: - pass - _fsync_directory(self.path.parent) - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass + atomic_write_private_json(self.path, value) def deep_merge(base: dict[str, Any], patch: dict[str, Any]) -> dict[str, Any]: @@ -151,13 +132,3 @@ def _file_lock(path: Path) -> Iterator[None]: fcntl.flock(descriptor, fcntl.LOCK_UN) finally: os.close(descriptor) - - -def _fsync_directory(path: Path) -> None: - if os.name == "nt": - return - descriptor = os.open(path, os.O_RDONLY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) diff --git a/core/application/errors.py b/core/application/errors.py index a25aeed45..74b11afed 100644 --- a/core/application/errors.py +++ b/core/application/errors.py @@ -221,6 +221,15 @@ class DuplicateMessageConflictError(ApplicationError): code = "DUPLICATE_MESSAGE_CONFLICT" +class InputDeliveryPendingError(ApplicationError): + code = "INPUT_DELIVERY_PENDING" + retryable = True + + +class InputDeliveryUncertainError(ApplicationError): + code = "INPUT_DELIVERY_UNCERTAIN" + + class TurnInterruptTimeoutError(ApplicationError): code = "TURN_INTERRUPT_TIMEOUT" retryable = True diff --git a/core/application/event_service.py b/core/application/event_service.py index 2a5915e8f..f338827c7 100644 --- a/core/application/event_service.py +++ b/core/application/event_service.py @@ -35,6 +35,7 @@ class ReplayPage: events: tuple[DomainEvent, ...] next_after: int | None has_more: bool + head_sequence: int | None = None @dataclass(slots=True) @@ -174,15 +175,28 @@ def replay( return list(self.replay_page(thread_id, after=after, limit=limit).events) def replay_page( - self, thread_id: str, *, after: int = 0, limit: int = 500 + self, + thread_id: str, + *, + after: int = 0, + limit: int = 500, + through: int | None = None, ) -> ReplayPage: if after < 0: raise ValueError("after must not be negative") if not 1 <= limit <= 1000: raise ValueError("limit must be between 1 and 1000") + if through is not None and through < 0: + raise ValueError("through must not be negative") with self.database.read() as connection: - events = EventRepository(connection).replay( - thread_id, after=after, limit=limit + 1 + repository = EventRepository(connection) + # Capture the cutoff before reading the page. Later commits cannot + # extend this replay round; clients carry the cutoff to later pages. + head = repository.sequence_head(thread_id) + if through is not None: + head = min(head, through) + events = repository.replay( + thread_id, after=after, limit=limit + 1, through=head ) has_more = len(events) > limit page = tuple(events[:limit]) @@ -190,6 +204,7 @@ def replay_page( events=page, next_after=page[-1].sequence if has_more and page else None, has_more=has_more, + head_sequence=head, ) diff --git a/core/application/execution_coordinator.py b/core/application/execution_coordinator.py index d345bcbe3..af1497fcb 100644 --- a/core/application/execution_coordinator.py +++ b/core/application/execution_coordinator.py @@ -147,6 +147,8 @@ def __init__( ) self._lock = threading.RLock() + self._admission_lock = threading.RLock() + self._admission_paused = False self._stop = threading.Event() self._wake = threading.Event() self._thread: threading.Thread | None = None @@ -287,7 +289,28 @@ def dispatch_once( ) -> tuple[ExecutionDispatch, ...]: """Claim eligible work, then send admitted Turns to the injected starter.""" - self._require_dispatchable() + with self._admission_lock: + self._require_dispatchable() + if self._admission_paused: + return () + return self._dispatch_once(candidate_limit=candidate_limit) + + def pause_admission(self) -> None: + """Fence new starts while keeping heartbeat and cancellation processing.""" + + with self._admission_lock: + self._require_dispatchable() + self._admission_paused = True + + def resume_admission(self) -> None: + """Resume after a cancelled or timed-out drain.""" + + with self._admission_lock: + self._require_dispatchable() + self._admission_paused = False + self.offer() + + def _dispatch_once(self, *, candidate_limit: int) -> tuple[ExecutionDispatch, ...]: now = self._now() claimed: list[ExecutionDispatch] = [] with self.database.transaction() as connection: diff --git a/core/application/goal_extension.py b/core/application/goal_extension.py index ddd627afd..05ad39c21 100644 --- a/core/application/goal_extension.py +++ b/core/application/goal_extension.py @@ -130,6 +130,22 @@ def is_turn_accounted( turn_id=turn_id, ) + def execution_settled(self, goal: ThreadGoal) -> bool: + """A terminal Goal is ready only after its deciding Turn is accounted.""" + if goal.status is ThreadGoalStatus.ACTIVE: + return False + outcome = self.read_outcome(goal.thread_id) + deciding_turn_id = outcome.decided_by_turn_id if outcome else None + if deciding_turn_id is None: + return True + try: + turn = self.turns.read(deciding_turn_id).turn + except TurnNotFoundError: + return True + return turn.status.is_terminal and self.is_turn_accounted( + goal.thread_id, goal_id=goal.id, turn_id=turn.id + ) + def are_turns_accounted( self, thread_id: str, diff --git a/core/application/input_identity.py b/core/application/input_identity.py new file mode 100644 index 000000000..edeba371c --- /dev/null +++ b/core/application/input_identity.py @@ -0,0 +1,51 @@ +"""Stable submission identity, independent of changing runtime defaults.""" + +from __future__ import annotations + +import hashlib +import json + +from core.domain.execution_permission import ExecutionPermissionMode +from core.domain.execution_security import ExecutionSecurityProfile +from core.domain.message_provenance import TurnInputDelivery, TurnInputSource +from core.domain.runtime_coordination import ExecutionClass +from core.skills.models import SkillSelection + + +def submission_fingerprint( + *, + prompt: str, + skill_ids: tuple[str, ...], + connection_id: str | None, + model: str | None, + reasoning_effort: str | None, + source: TurnInputSource, + delivery: TurnInputDelivery, + execution_class: ExecutionClass, + security_override: ExecutionSecurityProfile | None, + permission_override: ExecutionPermissionMode | None, +) -> str: + # Requested fields, not the resolved profile or Goal-inherited skills: + # a lost response must stay recoverable after those defaults change. + payload = { + "prompt": prompt.strip(), + "skills": [SkillSelection(skill_id=s).skill_id for s in skill_ids], + "connectionId": connection_id, + "model": model, + "reasoningEffort": reasoning_effort, + "source": source.value, + "delivery": delivery.value, + "executionClass": execution_class.value, + "securityOverride": security_override.to_dict() if security_override else None, + "permissionOverride": permission_override.value + if permission_override + else None, + } + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode() + return "v1:" + hashlib.sha256(encoded).hexdigest() diff --git a/core/application/llm_configuration_service.py b/core/application/llm_configuration_service.py index f28c7ac27..5c2f96ed2 100644 --- a/core/application/llm_configuration_service.py +++ b/core/application/llm_configuration_service.py @@ -14,6 +14,11 @@ ) from core.application.errors import ConflictError, InvalidArgumentError from core.application.project_service import ProjectService +from core.application.provider_verification import ( + verification_stage as _verification_stage, + model_error_detail as _model_error_detail, + verify_agent, +) from core.config import ( ConnectionProfileConfig, DeepCodeConfig, @@ -21,9 +26,10 @@ load_config_for_workspace, ) from core.domain.execution_profile import ExecutionProfile, ExecutionSelection -from core.providers.catalog_service import ModelCatalogService +from core.providers.catalog_service import ModelCatalogService, ModelCatalog from core.providers.credentials import CredentialStore from core.providers.profiles import ConnectionResolver, validate_connection_id +from core.providers.oauth import ProviderOAuthManager from core.providers.reasoning import infer_reasoning_capabilities from core.providers.registry import PROVIDERS, find_by_name @@ -32,6 +38,9 @@ "label", "template", "adapter", + "protocol", + "auth", + "compat", "apiBase", "apiKeyEnv", "apiKey", @@ -63,6 +72,7 @@ def __init__( self.config_store = config_store or ConfigStore() self.credentials = credential_store or CredentialStore() self.catalog = catalog or ModelCatalogService() + self.oauth = ProviderOAuthManager(self.credentials) def list_connections(self, project_id: str | None = None) -> dict[str, Any]: config = self._config(project_id=project_id) @@ -134,6 +144,10 @@ def transform(current: dict[str, Any]) -> dict[str, Any]: existing=existing, providers=providers, ) + if normalized["auth"] == "oauth" and api_key is not None: + raise InvalidArgumentError( + "Use provider login for OAuth connections, or select API-key authentication" + ) profiles[connection_id] = normalized providers["profiles"] = profiles return {**current, "providers": providers} @@ -155,6 +169,9 @@ def transform(current: dict[str, Any]) -> dict[str, Any]: ) if not credential_only_builtin: self._mutate_config(transform, expected_revision) + self.credentials.begin_login( + connection_id + ) # invalidate pending flows after configuration changes if clear_api_key: self.credentials.clear(connection_id) if api_key is not None: @@ -209,6 +226,36 @@ def transform(current: dict[str, Any]) -> dict[str, Any]: **self.list_connections(), } + def close(self) -> None: + self.oauth.close() + + def login_start(self, connection_id: str, *, open_browser: bool = False) -> dict: + connection = self._resolver().resolve_connection(connection_id) + if connection.auth != "oauth" or not connection.enabled: + raise InvalidArgumentError( + "Save an enabled OpenRouter OAuth connection before signing in" + ) + return self.oauth.start(connection.id, open_browser=open_browser) + + def login_poll(self, flow_id: str) -> dict: + return self.oauth.poll(flow_id) + + def login_cancel(self, flow_id: str) -> dict: + return self.oauth.cancel(flow_id) + + def logout(self, connection_id: str) -> dict: + connection = self._resolver().resolve_connection(connection_id) + if connection.auth != "oauth": + raise InvalidArgumentError( + "The selected connection does not use Provider login" + ) + self.credentials.clear(connection.id) + return { + "disconnected": True, + "remoteRevoked": False, + "manageUrl": "https://openrouter.ai/settings/keys", + } + def discover_models( self, *, @@ -217,6 +264,7 @@ def discover_models( api_base: str | None = None, api_key: str | None = None, project_id: str | None = None, + draft: dict[str, Any] | None = None, ) -> dict[str, Any]: """Probe an endpoint AS SHOWN in an editor form; writes nothing. @@ -226,9 +274,15 @@ def discover_models( leaves memory — discovery returns candidates, adopting writes (the dsh rule). """ - resolver = self._resolver(project_id=project_id) + resolver = ( + self._draft_resolver(draft, project_id) + if draft is not None + else self._resolver(project_id=project_id) + ) try: - if connection_id and connection_id.strip(): + if draft is not None: + base = resolver.resolve_connection(str(draft["id"])) + elif connection_id and connection_id.strip(): base = resolver.resolve_connection(connection_id.strip()) elif template and template.strip(): base = resolver.template_connection(template.strip()) @@ -296,6 +350,8 @@ def test( *, project_id: str | None = None, model_id: str | None = None, + draft: dict[str, Any] | None = None, + mode: str = "quick", ) -> dict[str, Any]: """Check one connection and optionally run a minimal real model request. @@ -305,8 +361,20 @@ def test( credential may call a particular model. """ + if mode not in {"quick", "agent"}: + raise InvalidArgumentError("Unknown verification mode") + if mode == "agent" and not _clean_optional(model_id): + raise InvalidArgumentError("Agent verification requires a model") + if draft is not None and draft.get("id") != connection_id: + raise InvalidArgumentError( + "Draft connection ID does not match the selected connection" + ) try: - resolver = self._resolver(project_id=project_id) + resolver = ( + self._draft_resolver(draft, project_id) + if draft is not None + else self._resolver(project_id=project_id) + ) connection = resolver.resolve_connection(connection_id) except ValueError as exc: raise InvalidArgumentError(str(exc)) from exc @@ -338,7 +406,32 @@ def test( _credential_detail(connection.credential_source), ) catalog_started = time.monotonic() - catalog = self.catalog.list_models(connection, refresh=True) + if draft is None: + catalog = self.catalog.list_models(connection, refresh=True) + else: + # Form probes never persist catalogs, credentials or route revisions. + try: + models = ( + self.catalog.probe(connection) + if connection.model_catalog != "manual" + else () + ) + catalog = ModelCatalog( + connection_id=connection.id, + models=models, + source="remote" + if connection.model_catalog != "manual" + else "manual", + stale=False, + ) + except Exception as exc: + catalog = ModelCatalog( + connection_id=connection.id, + models=(), + source="fallback", + stale=True, + error=_safe_configuration_error(exc), + ) catalog_latency = round((time.monotonic() - catalog_started) * 1000) if catalog.source == "manual" and not catalog.stale: catalog_stage = _verification_stage( @@ -372,7 +465,12 @@ def test( "Choose a model to run a minimal verification request", model_id=clean_model, ) - if clean_model is not None: + agent_stages = [] + if clean_model is not None and mode == "agent": + agent_stages = self._verify_agent( + resolver, connection.id, clean_model, catalog + ) + elif clean_model is not None: model_stage = self._verify_model( resolver, connection_id=connection.id, @@ -380,7 +478,18 @@ def test( catalog=catalog, ) - if clean_model is not None: + if mode == "agent": + required = {"stream", "tool", "continuation"} + ok = all( + any( + stage["id"] == name and stage["status"] == "passed" + for stage in agent_stages + ) + for name in required + ) and not any(stage["status"] == "failed" for stage in agent_stages) + status = "ready" if ok else "error" + error = None if ok else "Agent compatibility verification did not pass" + elif clean_model is not None: ok = model_stage["status"] == "passed" status = "ready" if ok else "error" error = None if ok else str(model_stage["detail"]) @@ -404,7 +513,9 @@ def test( started=started, model_count=len(catalog.models), error=error, - stages=(credential, catalog_stage, model_stage), + stages=(credential, catalog_stage, *agent_stages) + if mode == "agent" + else (credential, catalog_stage, model_stage), ) def _verify_model( @@ -426,6 +537,7 @@ def _verify_model( model_id=model_id, ), phase="implementation", + persist_revision=False, model_limits=( ( catalog_model.context_window, @@ -447,17 +559,22 @@ def _verify_model( model_id=model_id, ) - started = time.monotonic() - try: - response = _run_probe_isolated( - provider.chat( + async def probe(): + try: + return await provider.chat( messages=[{"role": "user", "content": _MODEL_PROBE_PROMPT}], model=profile.model_id, max_tokens=min(16, profile.max_output_tokens), temperature=0, reasoning_effort=None, - ), - timeout=_MODEL_PROBE_TIMEOUT_SECONDS, + ) + finally: + await provider.aclose() + + started = time.monotonic() + try: + response = _run_probe_isolated( + probe(), timeout=_MODEL_PROBE_TIMEOUT_SECONDS ) except TimeoutError: return _verification_stage( @@ -543,6 +660,47 @@ def resolve_phases( except ValueError as exc: raise InvalidArgumentError(str(exc)) from exc + def _draft_resolver( + self, draft: dict[str, Any], project_id: str | None + ) -> ConnectionResolver: + connection_id, key, clear = self._parse_mutation(draft) + config = self._config(project_id=project_id).model_copy(deep=True) + providers = config.providers.model_dump(by_alias=True, exclude_none=True) + existing = providers.get("profiles", {}).get(connection_id) + normalized = self._normalize_profile( + draft, connection_id=connection_id, existing=existing, providers=providers + ) + config.providers.profiles[connection_id] = ( + ConnectionProfileConfig.model_validate(normalized) + ) + overrides = {connection_id: key} if key is not None or clear else {} + return ConnectionResolver( + config, self.credentials, credential_overrides=overrides + ) + + def _verify_agent(self, resolver, connection_id, model_id, catalog): + selected = next((item for item in catalog.models if item.id == model_id), None) + try: + profile = resolver.execution_profile( + ExecutionSelection(connection_id, model_id), + model_limits=(selected.context_window, selected.max_output_tokens) + if selected + else None, + reasoning_capabilities=selected.reasoning if selected else None, + persist_revision=False, + ) + provider = resolver.build_provider(profile) + return _run_probe_isolated(verify_agent(provider, profile), timeout=95) + except Exception as exc: + return [ + _verification_stage( + "stream", + "failed", + _safe_configuration_error(exc), + model_id=model_id, + ) + ] + def _resolver(self, project_id: str | None = None) -> ConnectionResolver: return ConnectionResolver(self._config(project_id=project_id), self.credentials) @@ -604,6 +762,9 @@ def _normalize_profile( "label", "template", "adapter", + "protocol", + "auth", + "compat", "apiBase", "apiKeyEnv", "extraHeaders", @@ -637,25 +798,6 @@ def _clean_optional(value: Any) -> str | None: return clean or None -def _verification_stage( - stage_id: str, - status: str, - detail: str, - *, - latency_ms: int | None = None, - model_count: int | None = None, - model_id: str | None = None, -) -> dict[str, Any]: - return { - "id": stage_id, - "status": status, - "detail": detail[:300], - "latencyMs": latency_ms, - "modelCount": model_count, - "modelId": model_id, - } - - def _verification_result( connection_id: str, *, @@ -684,7 +826,9 @@ def _credential_detail(source: str) -> str: "environment": "Credential resolved from the configured environment variable", "credential_store": "Credential loaded from DeepCode private storage", "legacy_config": "Credential loaded from legacy DeepCode configuration", - "not_required": "This local or direct connection does not require a credential", + "not_required": "This connection does not require a credential", + "request": "Unsaved credential supplied for this verification only", + "oauth": "Credential bound to the signed-in OpenRouter account", }.get(source, "Credential is configured") @@ -712,27 +856,6 @@ def _safe_configuration_error(exc: Exception) -> str: return f"{type(exc).__name__}: model verification could not be completed" -def _model_error_detail(response: Any) -> str: - status = response.error_status_code - if status == 401: - return "The provider rejected the API credential" - if status == 403: - return "The credential does not have access to this model" - if status == 404: - return "The endpoint or selected model was not found" - if status == 408: - return "The model verification request timed out" - if status == 429: - return "The provider reported a rate, quota, or balance limit" - if isinstance(status, int) and status >= 500: - return "The provider is temporarily unavailable" - if response.error_kind == "timeout": - return "The model verification request timed out" - if response.error_kind == "connection": - return "DeepCode could not connect to the model endpoint" - return "The provider rejected the model verification request" - - def _profile_seed( connection_id: str, *, @@ -756,6 +879,9 @@ def _profile_seed( "label": spec.label, "template": spec.name, "adapter": spec.backend, + "protocol": legacy.get("protocol", "auto"), + "auth": legacy.get("auth", "api_key"), + "compat": legacy.get("compat", {}), "apiBase": legacy.get("apiBase", legacy.get("api_base")) or spec.default_api_base or None, diff --git a/core/application/provider_verification.py b/core/application/provider_verification.py new file mode 100644 index 000000000..5739dadfc --- /dev/null +++ b/core/application/provider_verification.py @@ -0,0 +1,253 @@ +"""Bounded, read-only model compatibility probes using the production adapters.""" + +from __future__ import annotations + +import asyncio +import json +import secrets +import time +from typing import Any + +from core.agent_runtime.helpers import build_assistant_message +from core.domain.execution_profile import ExecutionProfile +from core.providers.base import LLMProvider + + +def verification_stage( + stage_id: str, + status: str, + detail: str, + *, + latency_ms: int | None = None, + model_count: int | None = None, + model_id: str | None = None, +) -> dict[str, Any]: + return { + "id": stage_id, + "status": status, + "detail": detail[:300], + "latencyMs": latency_ms, + "modelCount": model_count, + "modelId": model_id, + } + + +def model_error_detail(response: Any) -> str: + status = response.error_status_code + if status == 401: + return "The provider rejected the API credential" + if status == 403: + return "The credential does not have access to this model" + if status == 404: + return "The endpoint or selected model was not found" + if status == 408: + return "The model verification request timed out" + if status == 429: + return "The provider reported a rate, quota, or balance limit" + if isinstance(status, int) and status >= 500: + return "The provider is temporarily unavailable" + if response.error_kind == "timeout": + return "The model verification request timed out" + if response.error_kind == "connection": + return "DeepCode could not connect to the model endpoint" + return "The provider rejected the model verification request" + + +async def verify_agent(provider: LLMProvider, profile: ExecutionProfile) -> list[dict]: + """At most three requests, 90 seconds total, no shell/file/network tools. + + The second request consumes a random value revealed only by the local + tool result, so merely accepting a tool schema cannot pass the probe. + """ + stages = [] + stage_ids = ("stream", "tool", "continuation", "reasoning", "image") + current = "stream" + started = time.monotonic() + observed_reasoning = False + streamed_text = False + + def record(stage, status, detail): + stages.append( + verification_stage( + stage, + status, + detail, + latency_ms=round((time.monotonic() - started) * 1000), + model_id=profile.model_id, + ) + ) + + async def content_delta(text): + nonlocal streamed_text + streamed_text = streamed_text or bool(text) + + async def reasoning_delta(text, _channel): + nonlocal observed_reasoning + observed_reasoning = observed_reasoning or bool(text) + + async def request(messages, **kwargs): + response = await provider.chat_stream( + messages=messages, + model=profile.model_id, + max_tokens=min(1024, profile.max_output_tokens), + temperature=0, + reasoning_effort=profile.reasoning_effort, + on_content_delta=content_delta, + on_reasoning_delta=reasoning_delta, + **kwargs, + ) + if response.finish_reason == "error": + raise ValueError(model_error_detail(response)) + return response + + try: + async with asyncio.timeout(90): + if profile.tool_calling is False: + record( + "tool", + "skipped", + "Model declared without tool calling; Agent compatibility cannot be established", + ) + record("continuation", "skipped", "Tool calling is disabled") + reply = await request([{"role": "user", "content": "Reply with OK"}]) + record( + "stream", + "passed" if streamed_text and reply.content else "failed", + "Text stream checked", + ) + else: + tools = [ + { + "type": "function", + "function": { + "name": "deepcode_probe", + "description": "Return a verification nonce. Call once with value 7.", + "parameters": { + "type": "object", + "properties": { + "value": {"type": "integer", "enum": [7]} + }, + "required": ["value"], + "additionalProperties": False, + }, + }, + } + ] + messages = [ + { + "role": "user", + "content": "Call deepcode_probe once with value 7, then reply with exactly the nonce returned by the tool.", + } + ] + first = await request( + messages, + tools=tools, + tool_choice="auto", + ) + current = "tool" + if not first.should_execute_tools or len(first.tool_calls) != 1: + raise ValueError("Expected one completed tool call") + call = first.tool_calls[0] + if ( + call.name != "deepcode_probe" + or call.arguments != {"value": 7} + or not call.id + ): + raise ValueError("The tool call name, ID or arguments were invalid") + record( + "tool", + "passed", + "One valid tool call executed by the local verification function", + ) + nonce = secrets.token_hex(12) + messages += [ + build_assistant_message( + first.content, + tool_calls=[call.to_openai_tool_call()], + reasoning_content=first.reasoning_content, + reasoning_summary=first.reasoning_summary, + provider_state=first.provider_state, + thinking_blocks=first.thinking_blocks, + ), + { + "role": "tool", + "tool_call_id": call.id, + "name": call.name, + "content": json.dumps({"nonce": nonce}), + }, + ] + current = "continuation" + second = await request(messages, tools=tools, tool_choice="auto") + if second.tool_calls or (second.content or "").strip() != nonce: + raise ValueError( + "The model did not consume and reproduce the tool result" + ) + record( + "continuation", + "passed", + "The model consumed the tool result with production reasoning-history serialization", + ) + record( + "stream", + "passed" if streamed_text else "failed", + "Text deltas received" + if streamed_text + else "No text deltas received", + ) + record( + "reasoning", + "passed" if observed_reasoning else "skipped", + "Provider reasoning deltas observed and history accepted" + if observed_reasoning + else "No reasoning deltas observed; reasoning remains unverified", + ) + current = "image" + if ( + profile.input_modalities is None + or "image" not in profile.input_modalities + ): + record("image", "skipped", "Image input is not explicitly declared") + else: + # A valid 64x64 PNG; this checks protocol acceptance, not vision quality. + png = "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAXklEQVR4nO3PMQ0AMAzAsPInvYLYYVWKESTzjhsd8KsBrQGtAa0BrQGtAa0BrQGtAa0BrQGtAa0BrQGtAa0BrQGtAa0BrQGtAa0BrQGtAa0BrQGtAa0BrQGtAa0BbQHKU9LC7/CP1AAAAABJRU5ErkJggg==" + image = await request( + [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "An image is attached. Reply OK.", + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," + png + }, + }, + ], + } + ] + ) + record( + "image", + "passed" if image.content else "failed", + "Image request accepted; visual understanding is not evaluated", + ) + except TimeoutError: + record( + current, + "failed", + "Compatibility verification exceeded its 90 second budget", + ) + except ValueError as exc: + record(current, "failed", str(exc)) + except Exception: + record(current, "failed", "Compatibility verification could not be completed") + finally: + await provider.aclose() + recorded = {stage["id"] for stage in stages} + for stage in stage_ids: + if stage not in recorded: + record(stage, "not_run", "An earlier stage did not complete") + return sorted(stages, key=lambda stage: stage_ids.index(stage["id"])) diff --git a/core/application/service_lifecycle.py b/core/application/service_lifecycle.py new file mode 100644 index 000000000..c47a14c5a --- /dev/null +++ b/core/application/service_lifecycle.py @@ -0,0 +1,77 @@ +"""Drain and activity queries over the existing application execution owners.""" + +from __future__ import annotations + +import threading +import time + +from core.application.application import DeepCodeApplication +from core.domain.turn import TurnStatus +from core.persistence.execution_repository import TurnRepository + + +class ServiceLifecycle: + def __init__(self, application: DeepCodeApplication) -> None: + self.application = application + self._paused = False + self._scheduler_was_active = False + + def resume(self) -> None: + """Restore admission after a supervisor could not finish a stop.""" + if self._paused: + self.application.execution_coordinator.resume_admission() + if self._scheduler_was_active: + self.application.automation_scheduler.start() + self._paused = False + self._scheduler_was_active = False + + def activity(self) -> dict[str, int | bool]: + app = self.application + worker_id = app.execution_coordinator.worker_id + with app.database.read() as connection: + turns = TurnRepository(connection).list_active() + owned = [ + turn + for turn in turns + if turn.execution_owner_id == worker_id + or (turn.execution_owner_id is None and turn.home_worker_id == worker_id) + ] + return { + "activeTurns": sum(turn.status != TurnStatus.QUEUED for turn in owned), + "queuedTurns": sum(turn.status == TurnStatus.QUEUED for turn in owned), + "terminals": app.terminals.active_count, + "schedulerActive": app.automation_scheduler.active, + "schedulerLeader": app.automation_scheduler.leader, + } + + def drain(self, timeout: float, interrupt: threading.Event) -> bool: + """Wait for admitted work, restoring admission if the wait does not finish. + + Queued work stays durable and unstarted. Normal application shutdown + settles it using the existing worker/Goal recovery semantics. + """ + app = self.application + if self._paused: + return True + self._scheduler_was_active = app.automation_scheduler.active + app.automation_scheduler.close() + drained = False + try: + app.execution_coordinator.pause_admission() + self._paused = True + deadline = time.monotonic() + timeout + while not interrupt.is_set(): + if ( + not app.execution_coordinator.active_claims + and not app.terminals.active_count + ): + drained = True + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + interrupt.wait(min(0.05, remaining)) + return False + finally: + if not drained: + self.resume() diff --git a/core/application/terminal_service.py b/core/application/terminal_service.py index ecacd3c6f..f1a23876f 100644 --- a/core/application/terminal_service.py +++ b/core/application/terminal_service.py @@ -4,13 +4,16 @@ import codecs import os +import select import signal import struct import subprocess import threading +import time import uuid +from collections import OrderedDict from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -22,6 +25,7 @@ ThreadNotFoundError, ) from core.application.workspace_service import WorkspaceService +from core.application.views import terminal_info_view from core.file_lock import FileLease from core.sessions import SessionStore @@ -48,9 +52,19 @@ class TerminalInfo: class _TerminalSession: info: TerminalInfo process: subprocess.Popen[bytes] - master_fd: int + master_fd: int | None activity_lease: FileLease closing: bool = False + output: bytearray = field(default_factory=bytearray) + offset: int = 0 + exited: bool = False + exit_code: int | None = None + reader: threading.Thread | None = None + io_lock: threading.Lock = field(default_factory=threading.Lock) + + @property + def head(self) -> int: + return self.offset + len(self.output) class TerminalService: @@ -60,12 +74,19 @@ def __init__( sessions: SessionStore, *, max_sessions: int = 8, + output_capacity: int = 256 * 1024, + retained_exits: int = 8, ) -> None: + if max_sessions < 1 or output_capacity < 4 or retained_exits < 0: + raise ValueError("invalid terminal retention limits") self.workspaces = workspaces self.sessions = sessions self.max_sessions = max_sessions + self.output_capacity = output_capacity + self.retained_exits = retained_exits self._lock = threading.RLock() self._sessions: dict[str, _TerminalSession] = {} + self._finished: OrderedDict[str, _TerminalSession] = OrderedDict() self._listeners: dict[str, TerminalListener] = {} self._creating = 0 @@ -142,12 +163,22 @@ def create( self._sessions[terminal_id] = session self._creating -= 1 registered = True - threading.Thread( + session.reader = threading.Thread( target=self._read_output, args=(session,), name=f"deepcode-terminal-{terminal_id[-8:]}", daemon=True, - ).start() + ) + try: + session.reader.start() + except BaseException: + with self._lock: + self._sessions.pop(terminal_id, None) + self._terminate(process) + os.close(master_fd) + session.master_fd = None + activity_lease.close() + raise return info except OSError as exc: raise ConflictError(f"terminal could not start: {exc}") from exc @@ -169,17 +200,23 @@ def write(self, thread_id: str, terminal_id: str, data: str) -> int: if len(encoded) > 64 * 1024: raise InvalidArgumentError("terminal input exceeds 64 KiB") session = self._owned(thread_id, terminal_id) - try: - return os.write(session.master_fd, encoded) - except OSError as exc: - raise ConflictError("terminal is no longer writable") from exc + with session.io_lock: + if session.master_fd is None or session.closing: + raise ConflictError("terminal is no longer writable") + try: + return os.write(session.master_fd, encoded) + except OSError as exc: + raise ConflictError("terminal is no longer writable") from exc def resize( self, thread_id: str, terminal_id: str, *, columns: int, rows: int ) -> TerminalInfo: self._validate_size(columns, rows) session = self._owned(thread_id, terminal_id) - self._set_size(session.master_fd, columns, rows) + with session.io_lock: + if session.master_fd is None or session.closing: + raise ConflictError("terminal has closed") + self._set_size(session.master_fd, columns, rows) with self._lock: session.info = TerminalInfo( id=session.info.id, @@ -198,12 +235,73 @@ def close(self, thread_id: str, terminal_id: str) -> bool: return False session.closing = True self._terminate(session.process) - try: - os.close(session.master_fd) - except OSError: - pass return True + def list(self, thread_id: str) -> list[dict[str, Any]]: + """Discover live terminals and a bounded set of completed output windows.""" + self._require_thread(thread_id) + with self._lock: + return [ + { + "terminal": terminal_info_view(session.info), + "exited": session.exited, + "exitCode": session.exit_code, + } + for session in (*self._finished.values(), *self._sessions.values()) + if session.info.thread_id == thread_id + ] + + def _require_thread(self, thread_id: str) -> None: + if self.sessions.get_session(thread_id) is None: + raise ThreadNotFoundError(f"thread not found: {thread_id}") + + def read( + self, + thread_id: str, + terminal_id: str, + *, + offset: int = 0, + limit: int = 16 * 1024, + through: int | None = None, + ) -> dict[str, Any]: + if ( + offset < 0 + or not 4 <= limit <= 64 * 1024 + or (through is not None and through < offset) + ): + raise InvalidArgumentError("invalid terminal output range") + self._require_thread(thread_id) + with self._lock: + session = self._owned(thread_id, terminal_id, include_finished=True) + head = session.head + if offset > head: + raise InvalidArgumentError("terminal cursor is ahead of its output") + start = max(offset, session.offset) + # If the original window was evicted during paging, report the new + # lower bound even when it has moved beyond the old cutoff. + end = max(start, min(head, through if through is not None else head)) + if end < head and session.output[end - session.offset] & 0xC0 == 0x80: + raise InvalidArgumentError("terminal cutoff splits a UTF-8 character") + relative = start - session.offset + raw = bytes(session.output[relative : relative + min(limit, end - start)]) + if raw and raw[0] & 0xC0 == 0x80: + raise InvalidArgumentError("terminal cursor splits a UTF-8 character") + data = raw.decode("utf-8", errors="ignore") + next_offset = start + len(data.encode("utf-8")) + return { + "terminalId": terminal_id, + "threadId": thread_id, + "data": data, + "offset": start, + "nextOffset": next_offset, + "availableFrom": session.offset, + "headOffset": head, + "hasMore": next_offset < end, + "truncated": offset < session.offset, + "exited": session.exited, + "exitCode": session.exit_code, + } + def active_for_thread(self, thread_id: str) -> bool: with self._lock: return any( @@ -211,30 +309,54 @@ def active_for_thread(self, thread_id: str) -> bool: for session in self._sessions.values() ) + @property + def active_count(self) -> int: + """Number of terminals this application still owns.""" + with self._lock: + return len(self._sessions) + def close_all(self) -> None: with self._lock: sessions = list(self._sessions.values()) for session in sessions: - if session.process.poll() is None: - session.closing = True - self._terminate(session.process) - try: - os.close(session.master_fd) - except OSError: - pass + session.closing = True + self._terminate(session.process) + for session in sessions: + if ( + session.reader is not None + and session.reader is not threading.current_thread() + ): + session.reader.join(timeout=3) + with self._lock: + self._finished.clear() - def _owned(self, thread_id: str, terminal_id: str) -> _TerminalSession: + def _owned( + self, thread_id: str, terminal_id: str, *, include_finished: bool = False + ) -> _TerminalSession: with self._lock: session = self._sessions.get(terminal_id) + if session is None and include_finished: + session = self._finished.get(terminal_id) if session is None or session.info.thread_id != thread_id: raise TerminalNotFoundError(f"terminal not found for Thread: {terminal_id}") return session def _read_output(self, session: _TerminalSession) -> None: decoder = codecs.getincrementaldecoder("utf-8")("replace") + close_deadline = None try: while True: + if session.closing: + if close_deadline is None: + close_deadline = time.monotonic() + 0.75 + if time.monotonic() >= close_deadline: + break try: + ready, _, _ = select.select([session.master_fd], [], [], 0.1) + if not ready: + if session.closing: + break + continue raw = os.read(session.master_fd, 16 * 1024) except OSError: break @@ -242,33 +364,30 @@ def _read_output(self, session: _TerminalSession) -> None: break text = decoder.decode(raw) if text: - self._publish( - "terminal.output", - { - "terminalId": session.info.id, - "threadId": session.info.thread_id, - "data": text, - }, - ) + self._record_output(session, text) trailing = decoder.decode(b"", final=True) if trailing: - self._publish( - "terminal.output", - { - "terminalId": session.info.id, - "threadId": session.info.thread_id, - "data": trailing, - }, - ) + self._record_output(session, trailing) finally: + # The reader exclusively closes the descriptor, including natural + # exit. Producer operations use io_lock to avoid descriptor reuse. + with session.io_lock: + if session.master_fd is not None: + os.close(session.master_fd) + session.master_fd = None try: exit_code = session.process.wait(timeout=1) except subprocess.TimeoutExpired: self._terminate(session.process) exit_code = session.process.returncode with self._lock: + session.exited = True + session.exit_code = exit_code if self._sessions.get(session.info.id) is session: self._sessions.pop(session.info.id, None) + self._finished[session.info.id] = session + while len(self._finished) > self.retained_exits: + self._finished.popitem(last=False) session.activity_lease.close() self._publish( "terminal.exit", @@ -276,9 +395,33 @@ def _read_output(self, session: _TerminalSession) -> None: "terminalId": session.info.id, "threadId": session.info.thread_id, "exitCode": exit_code, + "nextOffset": session.head, }, ) + def _record_output(self, session: _TerminalSession, text: str) -> None: + encoded = text.encode("utf-8") + with self._lock: + offset = session.head + session.output.extend(encoded) + trim = max(0, len(session.output) - self.output_capacity) + while trim < len(session.output) and session.output[trim] & 0xC0 == 0x80: + trim += 1 + if trim: + del session.output[:trim] + session.offset += trim + next_offset = session.head + self._publish( + "terminal.output", + { + "terminalId": session.info.id, + "threadId": session.info.thread_id, + "data": text, + "offset": offset, + "nextOffset": next_offset, + }, + ) + def _publish(self, method: str, payload: dict[str, Any]) -> None: with self._lock: listeners = tuple(self._listeners.values()) diff --git a/core/application/thread_service.py b/core/application/thread_service.py index 8c7ead3db..cb959fe37 100644 --- a/core/application/thread_service.py +++ b/core/application/thread_service.py @@ -61,6 +61,15 @@ _UNSET = object() +def _visible_conversation_items(items: list[Item]) -> list[Item]: + """Match the canonical text conversation, retaining tool-only timeline items. + + An assistant record carrying toolCalls can have no text. Its reconstructed + item belongs to the timeline, but neither side counts it as spoken text. + """ + return [item for item in items if str(item.payload.get("text", item.summary))] + + def _projected_item_kind(message: SessionMessage) -> ItemKind: """The item kind a rebuilt-from-JSONL record should carry. @@ -85,7 +94,22 @@ def _projected_item_kind(message: SessionMessage) -> ItemKind: def _projected_item_payload(message: SessionMessage) -> dict[str, object]: """Payload matching what the live projection stores for the same kind.""" if message.role != "tool": - return {"text": message.content, "projectedFromSession": True} + payload = {"text": message.content, "projectedFromSession": True} + if message.role == "user": + # Keep admission receipts when rebuilding disposable SQLite state. + metadata = message.metadata or {} + for key in ( + "messageId", + "requestFingerprint", + "expectedTurnId", + "deliveryState", + "source", + "delivery", + "client", + ): + if isinstance(metadata.get(key), str): + payload[key] = metadata[key] + return payload metadata = message.metadata or {} name = str(metadata.get("name") or "tool") return { @@ -781,7 +805,9 @@ def _reconcile_transcript( if message.role in {"user", "assistant"} and message.content ] items = ItemRepository(connection) - projected_items = items.conversation_for_thread(thread.id) + projected_items = _visible_conversation_items( + items.conversation_for_thread(thread.id) + ) projected = [ ( "user" if item.kind is ItemKind.USER_MESSAGE else "assistant", @@ -1381,7 +1407,12 @@ def _is_context_note(message: SessionMessage) -> bool: projection conflict. """ metadata = message.metadata or {} - return "delivery" in metadata + # User input also carries delivery provenance (current_turn/next_turn). + # Only the context-note sink's markers identify internal user-role notes. + return message.role == "user" and metadata.get("delivery") in ( + "mid_turn", + "between_turns", + ) def _merge_projection_tail( self, @@ -1390,6 +1421,7 @@ def _merge_projection_tail( *, projection_thread_id: str, ) -> bool: + projected_items = _visible_conversation_items(projected_items) canonical_pairs = [ (message.role, message.content) for message in canonical.messages diff --git a/core/application/turn_input_service.py b/core/application/turn_input_service.py index 4f552910c..eb320ee19 100644 --- a/core/application/turn_input_service.py +++ b/core/application/turn_input_service.py @@ -3,7 +3,8 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace +import logging from core.agent_runtime.injections import ( GoalObjectiveUpdated, @@ -18,6 +19,8 @@ DuplicateMessageConflictError, EmptyInputError, ExpectedTurnMismatchError, + InputDeliveryPendingError, + InputDeliveryUncertainError, InputTooLargeError, NoActiveTurnError, ThreadNotFoundError, @@ -32,6 +35,7 @@ from core.domain.item import Item, ItemKind, ItemStatus from core.domain.message_provenance import ( ClientSurface, + InputDeliveryState, TurnInputDelivery, TurnInputSource, ) @@ -43,6 +47,7 @@ from core.sessions import SessionStore EventPublisher = Callable[[list[DomainEvent]], None] +logger = logging.getLogger(__name__) @dataclass(frozen=True, slots=True) @@ -91,6 +96,7 @@ def steer( thread_id, prompt=clean_prompt, message_id=clean_message_id, + expected_turn_id=clean_expected, ) if duplicate is not None: return duplicate @@ -135,23 +141,52 @@ def steer( ) from exc if reservation is None: - return TurnInputReceipt( + duplicate = self._persisted_input( + thread_id, + prompt=clean_prompt, message_id=clean_message_id, - delivery=TurnInputDelivery.CURRENT_TURN.value, - turn=executing, - duplicate=True, + expected_turn_id=clean_expected, + ) + if duplicate is not None: + return duplicate + raise InputDeliveryUncertainError( + "mailbox input has no durable delivery confirmation", + details={ + "threadId": thread_id, + "expectedTurnId": clean_expected, + "messageId": clean_message_id, + }, ) + item = None try: - self._persist_live_input( + item, events = self._record_input_intent( executing, prompt=clean_prompt, message_id=clean_message_id, client_surface=client_surface, ) + self._publish(events) + self._append_canonical_input(item) self.session_runtimes.commit_input(thread_id, reservation) - except BaseException: + self._set_delivery_state(item.id, InputDeliveryState.ACCEPTED) + except BaseException as exc: self.session_runtimes.cancel_input(thread_id, reservation) + if item is not None: + try: + self._set_delivery_state(item.id, InputDeliveryState.UNKNOWN) + except Exception: + logger.exception("Could not settle Steer receipt %s", item.id) + if isinstance(exc, Exception): + raise InputDeliveryUncertainError( + "Steer delivery could not be confirmed; query its original receipt", + details={ + "threadId": thread_id, + "expectedTurnId": clean_expected, + "messageId": clean_message_id, + "itemId": item.id, + }, + ) from exc raise return TurnInputReceipt( message_id=clean_message_id, @@ -217,19 +252,16 @@ def _persisted_input( *, prompt: str, message_id: str, + expected_turn_id: str, ) -> TurnInputReceipt | None: with self.database.read() as connection: - if ThreadRepository(connection).get(thread_id) is None: - raise ThreadNotFoundError(f"thread not found: {thread_id}") - item = ItemRepository(connection).find_user_message_by_message_id( - thread_id, - message_id, - ) + item = self._find_input(connection, thread_id, message_id) if item is None: return None if ( item.payload.get("text") != prompt or item.payload.get("source") != TurnInputSource.STEER.value + or item.payload.get("expectedTurnId", item.turn_id) != expected_turn_id ): raise DuplicateMessageConflictError( "messageId was already used with different content" @@ -239,6 +271,25 @@ def _persisted_input( raise DuplicateMessageConflictError( "idempotent input references a missing Turn" ) + state = self._delivery_state(item, turn) + if state is not InputDeliveryState.ACCEPTED: + error = ( + InputDeliveryPendingError + if state is InputDeliveryState.PENDING + else InputDeliveryUncertainError + ) + raise error( + "Steer delivery is pending confirmation" + if state is InputDeliveryState.PENDING + else "Steer delivery is uncertain; this input will not be submitted again", + details={ + "threadId": thread_id, + "expectedTurnId": expected_turn_id, + "messageId": message_id, + "itemId": item.id, + "deliveryState": state.value, + }, + ) return TurnInputReceipt( message_id=message_id, delivery=TurnInputDelivery.CURRENT_TURN.value, @@ -246,14 +297,56 @@ def _persisted_input( duplicate=True, ) - def _persist_live_input( + def read(self, thread_id: str, message_id: str) -> Item | None: + """Locate an admitted input after a lost response without submitting work.""" + message_id = message_id.strip() + if not message_id: + raise EmptyInputError("messageId must not be empty") + with self.database.read() as connection: + item = self._find_input(connection, thread_id, message_id) + if ( + item is None + or item.payload.get("source") != TurnInputSource.STEER.value + ): + return item + turn = TurnRepository(connection).get(item.turn_id) + return replace( + item, + payload={ + **item.payload, + "deliveryState": self._delivery_state(item, turn).value, + }, + ) + + @staticmethod + def _delivery_state(item: Item, turn: Turn | None) -> InputDeliveryState: + state = item.payload.get("deliveryState") + if state == InputDeliveryState.ACCEPTED.value: + return InputDeliveryState.ACCEPTED + if ( + state == InputDeliveryState.PENDING.value + and turn is not None + and not turn.status.is_terminal + ): + return InputDeliveryState.PENDING + return InputDeliveryState.UNKNOWN + + @staticmethod + def _find_input(connection, thread_id: str, message_id: str) -> Item | None: + if ThreadRepository(connection).get(thread_id) is None: + raise ThreadNotFoundError(f"thread not found: {thread_id}") + return ItemRepository(connection).find_user_message_by_message_id( + thread_id, message_id + ) + + def _record_input_intent( self, turn: Turn, *, prompt: str, message_id: str, client_surface: ClientSurface, - ) -> None: + ) -> tuple[Item, list[DomainEvent]]: now = utc_now() events: list[DomainEvent] = [] with self.database.transaction() as connection: @@ -281,6 +374,11 @@ def _persist_live_input( ) items = ItemRepository(connection) + if ( + items.find_user_message_by_message_id(turn.thread_id, message_id) + is not None + ): + raise DuplicateMessageConflictError("messageId was already recorded") item = Item( thread_id=turn.thread_id, turn_id=turn.id, @@ -294,54 +392,113 @@ def _persist_live_input( "client": client_surface.value, "delivery": TurnInputDelivery.CURRENT_TURN.value, "source": TurnInputSource.STEER.value, + "expectedTurnId": turn.id, + "deliveryState": InputDeliveryState.PENDING.value, }, created_at=now, updated_at=now, ) items.add(item) event_repo = EventRepository(connection) - events.extend( - ( - event_repo.append( - thread_id=turn.thread_id, - turn_id=turn.id, - item_id=item.id, - type="item.created", - payload={"item": item_view(item)}, - ), - event_repo.append( - thread_id=turn.thread_id, - turn_id=turn.id, - item_id=item.id, - type="turn.steered", - payload={ - "turnId": turn.id, - "messageId": message_id, - "delivery": TurnInputDelivery.CURRENT_TURN.value, - }, - ), + events.append( + event_repo.append( + thread_id=turn.thread_id, + turn_id=turn.id, + item_id=item.id, + type="item.created", + payload={"item": item_view(item)}, ) ) - self._publish(events) + return item, events + def _append_canonical_input(self, item: Item) -> None: stored = self.session_store.append_message( - turn.thread_id, + item.thread_id, "user", - prompt, + item.payload["text"], metadata={ "schemaVersion": 3, - "client": client_surface.value, - "turnId": turn.id, - "messageId": message_id, + "client": item.payload["client"], + "turnId": item.turn_id, + "messageId": item.payload["messageId"], "delivery": TurnInputDelivery.CURRENT_TURN.value, "source": TurnInputSource.STEER.value, + "expectedTurnId": item.turn_id, + # A transcript proves persistence, not mailbox acceptance. A + # rebuilt projection must not fabricate delivery confirmation. + "deliveryState": InputDeliveryState.UNKNOWN.value, }, ) if stored is None: raise ThreadNotFoundError( - f"canonical session disappeared: {turn.thread_id}" + f"canonical session disappeared: {item.thread_id}" ) - self.session_runtimes.mark_persisted(turn.thread_id) + self.session_runtimes.mark_persisted(item.thread_id) + + def _set_delivery_state(self, item_id: str, state: InputDeliveryState) -> None: + with self.database.transaction() as connection: + items = ItemRepository(connection) + item = items.get(item_id) + if item is None: + raise InputDeliveryUncertainError("Steer receipt disappeared") + if item.payload.get("deliveryState") == InputDeliveryState.ACCEPTED.value: + return + events = self._update_delivery_state(connection, item, state) + if state is InputDeliveryState.ACCEPTED: + events.append( + EventRepository(connection).append( + thread_id=item.thread_id, + turn_id=item.turn_id, + item_id=item.id, + type="turn.steered", + payload={ + "turnId": item.turn_id, + "messageId": item.payload["messageId"], + "delivery": TurnInputDelivery.CURRENT_TURN.value, + "deliveryState": state.value, + }, + ) + ) + self._publish(events) + + @staticmethod + def _update_delivery_state( + connection, item: Item, state: InputDeliveryState + ) -> list[DomainEvent]: + if item.payload.get("deliveryState") == state.value: + return [] + updated = replace( + item, + payload={**item.payload, "deliveryState": state.value}, + updated_at=utc_now(), + ) + ItemRepository(connection).update(updated) + return [ + EventRepository(connection).append( + thread_id=item.thread_id, + turn_id=item.turn_id, + item_id=item.id, + type="item.updated", + payload={"item": item_view(updated)}, + ) + ] + + def settle_pending(self, connection, turn_id: str) -> list[DomainEvent]: + """A terminating Turn cannot leave an unconfirmed input pending forever.""" + events = [] + for item in ItemRepository(connection).list_for_turn(turn_id): + if ( + item.kind is ItemKind.USER_MESSAGE + and item.payload.get("source") == TurnInputSource.STEER.value + and item.payload.get("deliveryState") + == InputDeliveryState.PENDING.value + ): + events.extend( + self._update_delivery_state( + connection, item, InputDeliveryState.UNKNOWN + ) + ) + return events __all__ = ["TurnInputReceipt", "TurnInputService"] diff --git a/core/application/turn_service.py b/core/application/turn_service.py index 14f231816..919573640 100644 --- a/core/application/turn_service.py +++ b/core/application/turn_service.py @@ -51,6 +51,7 @@ GoalTurnAssociation, ) from core.application.llm_configuration_service import LLMConfigurationService +from core.application.input_identity import submission_fingerprint from core.application.session_runtime import SessionRuntimeRegistry from core.application.turn_input_service import ( TurnInputReceipt, @@ -350,6 +351,7 @@ def start( queue_if_busy=False, event_observer=event_observer, input_message_id=message_id, + requested_skill_ids=skill_ids, client_surface=client_surface, input_source=input_source, input_delivery=TurnInputDelivery.CURRENT_TURN, @@ -455,6 +457,7 @@ def enqueue( queue_if_busy=True, event_observer=event_observer, input_message_id=message_id, + requested_skill_ids=skill_ids, client_surface=client_surface, input_source=TurnInputSource.QUEUE, input_delivery=TurnInputDelivery.NEXT_TURN, @@ -485,6 +488,7 @@ def _submit( goal_id: str | None = None, goal_turn_settlement_ids: frozenset[str] = frozenset(), input_message_id: str | None = None, + requested_skill_ids: tuple[str, ...] | None = None, client_surface: ClientSurface = ClientSurface.INTERNAL, input_source: TurnInputSource = TurnInputSource.START, input_delivery: TurnInputDelivery = TurnInputDelivery.CURRENT_TURN, @@ -509,6 +513,24 @@ def _submit( ) except (TypeError, ValueError) as exc: raise InvalidArgumentError(str(exc)) from exc + fingerprint = ( + submission_fingerprint( + prompt=clean_prompt, + skill_ids=requested_skill_ids + if requested_skill_ids is not None + else clean_skill_ids, + connection_id=connection_id, + model=model, + reasoning_effort=reasoning_effort, + source=input_source, + delivery=input_delivery, + execution_class=execution_class, + security_override=execution_security_profile_override, + permission_override=execution_permission_mode_override, + ) + if input_message_id is not None + else None + ) events: list[DomainEvent] = [] schedule_now = False participant_contribution: ( @@ -565,6 +587,11 @@ def _submit( if ( existing_item.payload.get("text") != clean_prompt or existing_item.payload.get("source") != input_source.value + or ( + existing_item.payload.get("requestFingerprint") is not None + and existing_item.payload["requestFingerprint"] + != fingerprint + ) ): raise DuplicateMessageConflictError( "messageId was already used with different content" @@ -574,6 +601,23 @@ def _submit( raise DuplicateMessageConflictError( "idempotent input references a missing Turn" ) + if existing_item.payload.get("requestFingerprint") is None: + # Older receipts did not preserve the requested selection. + # Check supplied selectors against their execution snapshot; + # omitted defaults continue to refer to the original Turn. + profile = existing_turn.execution_profile + if clean_skill_ids != existing_turn.skill_ids or any( + value is not None + and (profile is None or value != getattr(profile, name)) + for value, name in ( + (connection_id, "connection_id"), + (model, "model_id"), + (reasoning_effort, "reasoning_effort"), + ) + ): + raise DuplicateMessageConflictError( + "messageId was already used with a different selection" + ) return _TurnSubmission( TurnSnapshot( existing_turn, @@ -653,7 +697,7 @@ def _submit( "delivery": input_delivery.value, "source": input_source.value, **( - {"messageId": input_message_id} + {"messageId": input_message_id, "requestFingerprint": fingerprint} if input_message_id is not None else {} ), @@ -1051,13 +1095,22 @@ def executing_for_thread(self, thread_id: str) -> Turn | None: raise ThreadNotFoundError(f"thread not found: {thread_id}") return TurnRepository(connection).executing_for_thread(thread_id) - def list_for_thread(self, thread_id: str) -> tuple[Turn, ...]: + def list_for_thread( + self, + thread_id: str, + *, + limit: int | None = None, + offset: int = 0, + state: str = "all", + ) -> tuple[Turn, ...]: """Return the durable Turn queue in ordinal order for client status UI.""" with self.database.read() as connection: if ThreadRepository(connection).get(thread_id) is None: raise ThreadNotFoundError(f"thread not found: {thread_id}") - turns = TurnRepository(connection).list_for_thread(thread_id) + turns = TurnRepository(connection).list_for_thread( + thread_id, limit=limit, offset=offset, state=state + ) return tuple(turns) def list_for_goal(self, thread_id: str, goal_id: str) -> tuple[Turn, ...]: @@ -1079,6 +1132,9 @@ def may_resume_queued_after_restart(self, turn: Turn) -> bool: for item in ItemRepository(connection).list_for_turn(turn.id) ) + def read_input(self, thread_id: str, message_id: str) -> Item | None: + return self.turn_inputs.read(thread_id, message_id) + def steer( self, thread_id: str, @@ -1331,7 +1387,13 @@ async def approve( initial_input_metadata = ( { key: initial_item.payload[key] - for key in ("messageId", "client", "delivery", "source") + for key in ( + "messageId", + "requestFingerprint", + "client", + "delivery", + "source", + ) if key in initial_item.payload } if initial_item is not None @@ -1839,6 +1901,7 @@ def _finish( payload={"turnId": turn_id}, ) ) + events.extend(self.turn_inputs.settle_pending(connection, turn_id)) if recover_active: approvals = ApprovalRepository(connection) for approval in approvals.pending_for_turn(turn_id): @@ -2035,6 +2098,7 @@ def recover_incomplete( if turn.status is TurnStatus.QUEUED and should_resume(turn): continue now = utc_now() + events.extend(self.turn_inputs.settle_pending(connection, turn.id)) for approval in approvals.pending_for_turn(turn.id): cancelled = replace( approval, diff --git a/core/application/workflow_service.py b/core/application/workflow_service.py index 4445628a2..fab6dc2fc 100644 --- a/core/application/workflow_service.py +++ b/core/application/workflow_service.py @@ -68,7 +68,6 @@ from core.persistence.workflow_repository import ArtifactRepository, WorkflowRepository from core.sessions import SessionStore - SUPPORTED_KINDS = frozenset({"paper2code"}) SUPPORTED_SOURCE_TYPES = frozenset({"local", "url", "repository", "requirement"}) MAX_SOURCE_LENGTH = 16_384 @@ -853,28 +852,32 @@ async def _execute( load_config_for_workspace(workspace), credential_store=self.llm_configuration.credentials, phase_execution_profiles=_execution_profiles(run.input), + config_loader=lambda: load_config_for_workspace(workspace), ) - with use_runtime(runtime): - outcome = await self.runner.run( - request, - WorkflowCallbacks( - progress=lambda stage, current, total, message, metadata: ( - self._progress( + try: + with use_runtime(runtime): + outcome = await self.runner.run( + request, + WorkflowCallbacks( + progress=lambda stage, current, total, message, metadata: ( + self._progress( + run.id, + stage=stage, + current=current, + total=total, + message=message, + metadata=metadata, + ) + ), + interact=lambda interaction: self._interact( run.id, - stage=stage, - current=current, - total=total, - message=message, - metadata=metadata, - ) - ), - interact=lambda interaction: self._interact( - run.id, - interaction, - claim=claim, + interaction, + claim=claim, + ), ), - ), - ) + ) + finally: + await runtime.aclose() if outcome.status == "completed": self._finish( run.id, diff --git a/core/compat/runtime.py b/core/compat/runtime.py index 1e83e67a0..5162ee1e0 100644 --- a/core/compat/runtime.py +++ b/core/compat/runtime.py @@ -17,9 +17,10 @@ from __future__ import annotations +import asyncio import hashlib import threading -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass @@ -39,7 +40,6 @@ from core.providers.credentials import CredentialStore from core.providers.profiles import ConnectionResolver - _runtime_lock = threading.Lock() _runtime: "DeepCodeRuntime | None" = None @@ -74,14 +74,18 @@ def __init__( *, credential_store: CredentialStore | None = None, phase_execution_profiles: dict[str, ExecutionProfile] | None = None, + config_loader: Callable[[], DeepCodeConfig] | None = None, ) -> None: self.config = config self.credential_store = credential_store or CredentialStore() - self.connection_resolver = ConnectionResolver(config, self.credential_store) + self.connection_resolver = ConnectionResolver( + config, self.credential_store, config_loader=config_loader + ) self.model_catalog = ModelCatalogService() self.phase_execution_profiles = dict(phase_execution_profiles or {}) self.logger = logger self._provider_cache: dict[tuple[str, ...], LLMProvider] = {} + self._closed = False # MCP servers materialised on construction so legacy callers can # mutate ``args`` in place (workflows.environment, plugin code, ...). self._mcp_servers = config.mcp_servers @@ -92,7 +96,10 @@ def __init__( @classmethod def load(cls, config_path: str | None = None) -> "DeepCodeRuntime": """Read ``deepcode_config.json`` and build a fresh runtime.""" - return cls(load_config(config_path=config_path)) + return cls( + load_config(config_path=config_path), + config_loader=lambda: load_config(config_path=config_path), + ) def provider_for( self, @@ -104,6 +111,8 @@ def provider_for( execution_profile: ExecutionProfile | None = None, ) -> LLMProvider: """Return a cached :class:`LLMProvider` for the requested combination.""" + if self._closed: + raise RuntimeError("The provider runtime is closed") if execution_profile is None and connection_id is None and model is None: execution_profile = self.phase_execution_profiles.get(phase) if execution_profile is not None or connection_id is not None: @@ -120,6 +129,10 @@ def provider_for( profile.connection_id, profile.model_id, profile.config_revision, + profile.provider_revision or "", + repr(profile.input_modalities), + repr(profile.tool_calling), + repr(profile.reasoning_supported), credential_revision, str(profile.max_tokens), repr(profile.temperature), @@ -148,6 +161,31 @@ def provider_for( self._provider_cache[cache_key] = provider return provider + async def aclose(self) -> None: + """Close only this runtime's pools after its sessions and children finish.""" + self._closed = True + providers = list( + { + id(provider): provider for provider in self._provider_cache.values() + }.values() + ) + results = await asyncio.gather( + *(provider.aclose() for provider in providers), return_exceptions=True + ) + failed = { + id(provider) + for provider, result in zip(providers, results, strict=True) + if isinstance(result, BaseException) + } + self._provider_cache = { + key: provider + for key, provider in self._provider_cache.items() + if id(provider) in failed + } + errors = [result for result in results if isinstance(result, BaseException)] + if errors: + raise BaseExceptionGroup("Provider cleanup failed", errors) + def resolve_execution_profile( self, *, diff --git a/core/config.py b/core/config.py index 023453394..4126bd723 100644 --- a/core/config.py +++ b/core/config.py @@ -36,13 +36,18 @@ from typing import Any, Literal from loguru import logger -from pydantic import AliasChoices, BaseModel, ConfigDict, Field +from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator from pydantic.alias_generators import to_camel from pydantic_settings import BaseSettings from core.agent_runtime.tools.mcp import MCPServerConfig from core.mcp.models import McpServerDefinition from core.providers.base import GenerationSettings, LLMProvider +from core.providers.protocol_config import ( + ProviderCompat, + ProviderProtocol, + protocol_adapter, +) from core.providers.registry import ( PROVIDERS, ProviderSpec, @@ -67,6 +72,7 @@ class _Base(BaseModel): alias_generator=to_camel, populate_by_name=True, extra="ignore", + hide_input_in_errors=True, ) @@ -145,6 +151,18 @@ class ProviderConfig(_Base): api_key: str | None = None api_base: str | None = None extra_headers: dict[str, str] | None = None + protocol: ProviderProtocol = "auto" + compat: ProviderCompat = Field(default_factory=ProviderCompat) + auth: Literal["api_key", "none"] = "api_key" + + @model_validator(mode="after") + def validate_wire(self): + self.compat.validate_protocol(self.protocol) + if self.auth == "none" and self.protocol == "anthropic_messages": + raise ValueError( + "Unauthenticated endpoints currently require an OpenAI protocol" + ) + return self class ManualModelConfig(_Base): @@ -167,6 +185,19 @@ class ManualModelConfig(_Base): context_window: int | None = None max_output_tokens: int | None = None reasoning_efforts: list[str] | Literal[False] | None = None + input_modalities: list[Literal["text", "image"]] | None = Field( + default=None, min_length=1, max_length=2 + ) + tool_calling: bool | None = None + compat: ProviderCompat = Field(default_factory=ProviderCompat) + + @model_validator(mode="after") + def validate_modalities(self): + if self.input_modalities is not None and len(set(self.input_modalities)) != len( + self.input_modalities + ): + raise ValueError("Input modalities must be unique") + return self class ConnectionProfileConfig(_Base): @@ -179,6 +210,9 @@ class ConnectionProfileConfig(_Base): label: str = "" template: str = "custom" adapter: Literal["openai_compat", "anthropic"] | None = None + protocol: ProviderProtocol = "auto" + auth: Literal["api_key", "none", "oauth"] = "api_key" + compat: ProviderCompat = Field(default_factory=ProviderCompat) api_base: str | None = None api_key_env: str | None = None extra_headers: dict[str, str] = Field(default_factory=dict) @@ -188,6 +222,48 @@ class ConnectionProfileConfig(_Base): manual_models: list[str | ManualModelConfig] = Field(default_factory=list) enabled: bool = True + @model_validator(mode="after") + def validate_wire(self): + spec = find_by_name(self.template) + legacy = self.adapter or (spec.backend if spec else "openai_compat") + effective = protocol_adapter(self.protocol, legacy) + if ( + self.protocol != "auto" + and self.adapter is not None + and self.adapter != effective + ): + raise ValueError( + "Explicit protocol and legacy adapter disagree; clear the adapter or select the matching protocol" + ) + if self.auth == "none" and effective == "anthropic": + raise ValueError( + "Unauthenticated endpoints currently require an OpenAI protocol" + ) + if self.auth == "oauth" and ( + self.template != "openrouter" + or self.protocol not in {"auto", "openai_chat"} + or effective != "openai_compat" + or self.api_base + not in { + None, + "https://openrouter.ai/api/v1", + "https://openrouter.ai/api/v1/", + } + or self.api_key_env + or any( + key.lower() in {"authorization", "x-api-key"} + for key in self.extra_headers + ) + ): + raise ValueError( + "OAuth currently requires the official OpenRouter Chat endpoint without credential overrides" + ) + self.compat.validate_protocol(self.protocol) + for model in self.manual_models: + if isinstance(model, ManualModelConfig): + model.compat.validate_protocol(self.protocol) + return self + class ProvidersConfig(_Base): """Per-provider connection blocks. Add new providers by extending here @@ -879,12 +955,18 @@ def make_llm_provider( "Set agents.defaults.provider or fill in the matching providers.<name>.apiKey." ) - backend = spec.backend + protocol = provider_cfg.protocol if provider_cfg else "auto" + backend = protocol_adapter(protocol, spec.backend) api_key = provider_cfg.api_key if provider_cfg else None api_base = provider_cfg.api_base if provider_cfg else None extra_headers = provider_cfg.extra_headers if provider_cfg else None - needs_key = not (spec.is_oauth or spec.is_local or spec.is_direct) + auth_mode = provider_cfg.auth if provider_cfg else "api_key" + if auth_mode == "none" and backend == "anthropic": + raise ConfigError("Anthropic Messages requires an API key") + needs_key = auth_mode != "none" and not ( + spec.is_oauth or spec.is_local or spec.is_direct + ) if needs_key and not api_key: raise ConfigError( f"Provider '{spec.name}' (phase '{phase}') requires providers.{spec.name}.apiKey " @@ -901,6 +983,7 @@ def make_llm_provider( api_base=effective_base, default_model=chosen_model, extra_headers=extra_headers, + compat=provider_cfg.compat if provider_cfg else None, ) elif backend == "openai_compat": from core.providers.openai_compat import OpenAICompatProvider @@ -911,6 +994,9 @@ def make_llm_provider( default_model=chosen_model, extra_headers=extra_headers, spec=spec, + protocol=protocol, + compat=provider_cfg.compat if provider_cfg else None, + auth_mode=auth_mode, ) else: raise ValueError( diff --git a/core/domain/execution_profile.py b/core/domain/execution_profile.py index 01e3f4d2b..8f0ba8e9b 100644 --- a/core/domain/execution_profile.py +++ b/core/domain/execution_profile.py @@ -3,6 +3,7 @@ from __future__ import annotations import math +import re from dataclasses import dataclass from typing import Any @@ -58,8 +59,35 @@ class ExecutionProfile: temperature: float reasoning_effort: str | None config_revision: str + protocol: str = "auto" + provider_revision: str | None = None + input_modalities: tuple[str, ...] | None = None + tool_calling: bool | None = None + reasoning_supported: bool | None = None def __post_init__(self) -> None: + if self.provider_revision is not None and ( + not isinstance(self.provider_revision, str) + or not re.fullmatch(r"[a-f0-9]{64}", self.provider_revision) + ): + raise ValueError("invalid provider revision") + if self.reasoning_supported is not None and not isinstance( + self.reasoning_supported, bool + ): + raise ValueError("reasoning_supported must be a boolean") + if self.protocol not in { + "auto", + "openai_chat", + "openai_responses", + "anthropic_messages", + }: + raise ValueError("invalid provider protocol") + if self.input_modalities is not None and ( + not self.input_modalities or set(self.input_modalities) - {"text", "image"} + ): + raise ValueError("invalid input modalities") + if self.tool_calling is not None and not isinstance(self.tool_calling, bool): + raise ValueError("tool_calling must be a boolean") for value, name in ( (self.connection_id, "connection_id"), (self.provider_name, "provider_name"), @@ -90,6 +118,27 @@ def to_dict(self) -> dict[str, Any]: "temperature": self.temperature, "reasoningEffort": self.reasoning_effort, "configRevision": self.config_revision, + **({"protocol": self.protocol} if self.protocol != "auto" else {}), + **( + {"providerRevision": self.provider_revision} + if self.provider_revision is not None + else {} + ), + **( + {"inputModalities": list(self.input_modalities)} + if self.input_modalities is not None + else {} + ), + **( + {"toolCalling": self.tool_calling} + if self.tool_calling is not None + else {} + ), + **( + {"reasoningSupported": self.reasoning_supported} + if self.reasoning_supported is not None + else {} + ), } @classmethod @@ -114,6 +163,13 @@ def from_dict(cls, value: Any) -> "ExecutionProfile | None": else None ), config_revision=str(value["configRevision"]), + protocol=value.get("protocol", "auto"), + provider_revision=value.get("providerRevision"), + input_modalities=tuple(value["inputModalities"]) + if value.get("inputModalities") is not None + else None, + tool_calling=value.get("toolCalling"), + reasoning_supported=value.get("reasoningSupported"), ) except (KeyError, TypeError, ValueError): return None diff --git a/core/domain/message_provenance.py b/core/domain/message_provenance.py index f7988a7d0..10628f831 100644 --- a/core/domain/message_provenance.py +++ b/core/domain/message_provenance.py @@ -8,6 +8,7 @@ class ClientSurface(StrEnum): CLI = "cli" DESKTOP = "desktop" + WEB = "web" HEADLESS = "headless" AUTOMATION = "automation" APP_SERVER = "app_server" @@ -29,8 +30,17 @@ class TurnInputDelivery(StrEnum): NEXT_TURN = "next_turn" +class InputDeliveryState(StrEnum): + """Confirmation that a durable Steer was accepted by its live mailbox.""" + + PENDING = "pending" + ACCEPTED = "accepted" + UNKNOWN = "unknown" + + __all__ = [ "ClientSurface", "TurnInputDelivery", + "InputDeliveryState", "TurnInputSource", ] diff --git a/core/events/session.py b/core/events/session.py index 8cc8510b8..88e1b94ea 100644 --- a/core/events/session.py +++ b/core/events/session.py @@ -19,6 +19,7 @@ from __future__ import annotations import asyncio +from collections.abc import Awaitable, Callable from contextvars import ContextVar from functools import partial from pathlib import Path @@ -399,9 +400,11 @@ def __init__( tool_filter: Any | None = None, closure_callback: Any | None = None, mcp_runtime: McpSessionRuntime | None = None, + provider_cleanup: Callable[[], Awaitable[None]] | None = None, ) -> None: self._runner = AgentRunner(provider) self._provider = provider + self._provider_cleanup = provider_cleanup self._tools = tools self._model = model self._system_prompt = system_prompt @@ -692,6 +695,9 @@ async def aclose(self) -> None: if self._mcp_runtime is not None: await self._mcp_runtime.aclose() await self._tools.aclose() + if self._provider_cleanup is not None: + await self._provider_cleanup() + self._provider_cleanup = None async def _cancel_turn_subagents(self) -> None: """Stop delegated work without letting repeated Stop interrupt teardown.""" diff --git a/core/observability/bus.py b/core/observability/bus.py index 31d476018..baf95ee51 100644 --- a/core/observability/bus.py +++ b/core/observability/bus.py @@ -26,7 +26,7 @@ import threading from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TextIO from loguru import logger as _loguru_logger @@ -98,6 +98,7 @@ def setup_logging( *, workspace_root: Path | None = None, force: bool = False, + console_sink: TextIO | logging.Handler | None = None, ) -> None: """Wire up loguru sinks. Idempotent unless ``force=True``. @@ -108,6 +109,9 @@ def setup_logging( ``workspace_root`` controls where the global log file lives (``logs/`` is created relative to it). When omitted the current working directory is used. + + ``console_sink`` lets a headless host redirect the console channel without + replacing the shared stdlib/loguru bridge or duplicating its routing. """ global _INITIALISED, _LLM_PREVIEW_CHARS, _MCP_PREVIEW_CHARS @@ -146,7 +150,7 @@ def setup_logging( if "console" in transports or not transports: sid = _loguru_logger.add( - sys.stderr, + console_sink if console_sink is not None else sys.stderr, level=level, format=_console_format, backtrace=False, diff --git a/core/persistence/database.py b/core/persistence/database.py index 0cc397cbd..2b1d12af2 100644 --- a/core/persistence/database.py +++ b/core/persistence/database.py @@ -15,6 +15,7 @@ from core.file_lock import exclusive_file_lock from core.persistence.migrations import ( LATEST_SCHEMA_VERSION, + MigrationError, current_version, migrate, ) @@ -37,10 +38,27 @@ def __init__(self, path: Path | str | None = None) -> None: Path(path).expanduser().resolve() if path else default_database_path() ) + @property + def restore_marker(self) -> Path: + return self.path.with_name(self.path.name + ".restore.json") + + @property + def restore_recovery_marker(self) -> Path: + return self.path.with_name(self.path.name + ".restored.json") + def initialize(self, *, target_version: int = LATEST_SCHEMA_VERSION) -> None: ensure_private_directory(self.path.parent) with exclusive_file_lock(self._migration_lock_path()): + if self.restore_marker.exists(): + raise RuntimeError( + "A state restore is pending. Resume it with deepcode service restore before starting the application." + ) had_existing_database = self._has_existing_database() + installed = self.schema_version() + if installed > target_version: + raise MigrationError( + f"database schema {installed} is newer than supported {target_version}" + ) connection = self._connect() try: self._enable_wal(connection) @@ -61,8 +79,11 @@ def schema_version(self) -> int: if not self._has_existing_database(): return 0 - with self.read() as connection: + connection = sqlite3.connect(self.path.as_uri() + "?mode=ro", uri=True) + try: return current_version(connection) + finally: + connection.close() @staticmethod def _enable_wal(connection: sqlite3.Connection) -> None: diff --git a/core/persistence/event_repository.py b/core/persistence/event_repository.py index 5b07e74af..a9ac09266 100644 --- a/core/persistence/event_repository.py +++ b/core/persistence/event_repository.py @@ -61,11 +61,19 @@ def replay( *, after: int = 0, limit: int = 500, + through: int | None = None, ) -> list[DomainEvent]: + upper_bound = " AND sequence <= ?" if through is not None else "" + parameters = ( + (thread_id, after, through, limit) + if through is not None + else (thread_id, after, limit) + ) rows = self.connection.execute( - "SELECT * FROM event_log WHERE thread_id = ? AND sequence > ? " + "SELECT * FROM event_log WHERE thread_id = ? AND sequence > ?" + f"{upper_bound} " "ORDER BY sequence LIMIT ?", - (thread_id, after, limit), + parameters, ).fetchall() return [self._from_row(row) for row in rows] diff --git a/core/persistence/execution_repository.py b/core/persistence/execution_repository.py index c18ef2949..eaa4223c2 100644 --- a/core/persistence/execution_repository.py +++ b/core/persistence/execution_repository.py @@ -226,10 +226,37 @@ def list_active(self) -> list[Turn]: ).fetchall() return [self._from_row(row) for row in rows] - def list_for_thread(self, thread_id: str) -> list[Turn]: - rows = self.connection.execute( - "SELECT * FROM turns WHERE thread_id = ? ORDER BY ordinal", (thread_id,) - ).fetchall() + def list_for_thread( + self, + thread_id: str, + *, + limit: int | None = None, + offset: int = 0, + state: str = "all", + ) -> list[Turn]: + filters = { + "all": "", + "active": " AND status IN ('queued', 'running', 'waiting_approval')", + "executing": " AND status IN ('running', 'waiting_approval')", + } + if state not in filters or offset < 0 or (limit is not None and limit < 1): + raise ValueError("Invalid Turn page") + order = ( + "CASE status WHEN 'running' THEN 0 WHEN 'waiting_approval' THEN 0 ELSE 1 END, ordinal" + if state == "active" + else "ordinal" + ) + query = ( + "SELECT * FROM turns WHERE thread_id = ?" + + filters[state] + + " ORDER BY " + + order + ) + arguments = [thread_id] + if limit is not None: + query += " LIMIT ? OFFSET ?" + arguments.extend((limit, offset)) + rows = self.connection.execute(query, arguments).fetchall() return [self._from_row(row) for row in rows] def list_for_goal(self, thread_id: str, goal_id: str) -> list[Turn]: diff --git a/core/private_storage.py b/core/private_storage.py index d67899529..2ab496b99 100644 --- a/core/private_storage.py +++ b/core/private_storage.py @@ -102,6 +102,10 @@ def _run_icacls(executable: str, path: Path, *arguments: str) -> bool: errors="replace", timeout=15, check=True, + # A detached service has no console to inherit. Avoid allocating + # a new console for every ACL helper it launches. + creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, + stdin=subprocess.DEVNULL, ) except (OSError, subprocess.SubprocessError): return False @@ -320,3 +324,30 @@ def _chmod(path: Path, mode: int, *, force: bool = False) -> None: "open_existing_private_file", "open_private_file", ] + + +def atomic_write_private_json(path: Path, value: object) -> None: + """Publish private JSON durably; callers own any cross-process mutation lock.""" + import json + import uuid + + ensure_private_directory(path.parent) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + payload = ( + json.dumps(value, indent=2, ensure_ascii=False, allow_nan=False) + "\n" + ).encode() + descriptor = open_private_file(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + if os.name != "nt": + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + temporary.unlink(missing_ok=True) diff --git a/core/providers/anthropic.py b/core/providers/anthropic.py index 675d9c1eb..2108b2119 100644 --- a/core/providers/anthropic.py +++ b/core/providers/anthropic.py @@ -14,22 +14,25 @@ from core.observability import log_llm_call from core.providers.base import ( LLMProvider, + ProviderCapabilityError, + ProviderConfigurationError, LLMResponse, ReasoningDeltaCallback, ToolCallRequest, ) +from core.providers.protocol_config import ProviderCompat from core.providers.reasoning import ( ANTHROPIC_THINKING_BLOCKS, infer_reasoning_capabilities, normalize_reasoning_effort, ) -from core.reasoning import ReasoningChannel from core.providers.timeouts import ( StreamIdleTimeoutError, iter_with_stream_idle_timeout, resolve_stream_idle_timeout_s, wait_for_stream_activity, ) +from core.reasoning import ReasoningChannel _ALNUM = string.ascii_letters + string.digits @@ -87,16 +90,20 @@ def __init__( api_base: str | None = None, default_model: str = "claude-sonnet-4-20250514", extra_headers: dict[str, str] | None = None, + compat: ProviderCompat | None = None, ): super().__init__(api_key, api_base) self.default_model = default_model self.extra_headers = extra_headers or {} + self.compat = compat or ProviderCompat() + self.compat.validate_protocol("anthropic_messages") from anthropic import AsyncAnthropic client_kw: dict[str, Any] = {} if api_key: client_kw["api_key"] = api_key + client_kw["auth_token"] = "" if api_base: client_kw["base_url"] = api_base if extra_headers: @@ -104,9 +111,28 @@ def __init__( # Keep retries centralized in LLMProvider._run_with_retry to avoid retry amplification. client_kw["max_retries"] = 0 self._client = AsyncAnthropic(**client_kw) + if api_key: + self._client.auth_token = None + + async def aclose(self) -> None: + await self._client.close() + + def _set_runtime_credential(self, key: str | None) -> None: + self.api_key = key + self._client.api_key = key + self._client.auth_token = None @classmethod def _handle_error(cls, e: Exception) -> LLMResponse: + if isinstance(e, (ProviderCapabilityError, ProviderConfigurationError)): + return LLMResponse( + content=str(e), + finish_reason="error", + error_kind="capability" + if isinstance(e, ProviderCapabilityError) + else "configuration", + error_should_retry=False, + ) response = getattr(e, "response", None) headers = getattr(response, "headers", None) payload = ( @@ -460,6 +486,7 @@ def _build_kwargs( tool_choice: str | dict[str, Any] | None, supports_caching: bool = True, ) -> dict[str, Any]: + self.validate_request_capabilities(messages, tools) model_name = self._strip_prefix(model or self.default_model) system, anthropic_msgs = self._convert_messages( self._sanitize_empty_content(messages) @@ -475,7 +502,12 @@ def _build_kwargs( max_tokens = max(1, max_tokens) effort = normalize_reasoning_effort(reasoning_effort) - summarized_thinking = self._uses_summarized_thinking(model_name, effort) + if self.reasoning_supported is False: + effort = None + summarized_thinking = ( + self.reasoning_supported is not False + and self._uses_summarized_thinking(model_name, effort) + ) thinking_enabled = summarized_thinking or effort not in {None, "auto", "none"} kwargs: dict[str, Any] = { @@ -502,6 +534,15 @@ def _build_kwargs( else: kwargs["temperature"] = temperature + if self.compat.temperature is False: + kwargs.pop("temperature", None) + elif self.compat.temperature is True: + kwargs.setdefault("temperature", temperature) + if "temperature" in kwargs: + # SDK 1.x removed the typed parameter. The documented extra_body + # path preserves the same legacy wire value on both SDK generations. + kwargs["extra_body"] = {"temperature": kwargs.pop("temperature")} + if anthropic_tools: kwargs["tools"] = anthropic_tools tc = self._convert_tool_choice(tool_choice, thinking_enabled) @@ -630,18 +671,20 @@ async def chat( model_name = self._strip_prefix(model or self.default_model) effort = normalize_reasoning_effort(reasoning_effort) summarized_thinking = self._uses_summarized_thinking(model_name, effort) - kwargs = self._build_kwargs( - messages, - tools, - model, - max_tokens, - temperature, - reasoning_effort, - tool_choice, - ) started = time.monotonic() result: LLMResponse | None = None try: + await self.refresh_request_credentials() + kwargs = self._build_kwargs( + messages, + tools, + model, + max_tokens, + temperature, + reasoning_effort, + tool_choice, + ) + response = await self._client.messages.create(**kwargs) result = self._parse_response( response, @@ -649,7 +692,9 @@ async def chat( ) return result except Exception as e: - result = self._handle_error(e) + result = self.redact_error( + self._handle_error(e), e, [self.api_key, *self.extra_headers.values()] + ) return result finally: self._emit_observability( @@ -675,19 +720,21 @@ async def chat_stream( model_name = self._strip_prefix(model or self.default_model) effort = normalize_reasoning_effort(reasoning_effort) summarized_thinking = self._uses_summarized_thinking(model_name, effort) - kwargs = self._build_kwargs( - messages, - tools, - model, - max_tokens, - temperature, - reasoning_effort, - tool_choice, - ) idle_timeout_s = resolve_stream_idle_timeout_s() started = time.monotonic() result: LLMResponse | None = None try: + await self.refresh_request_credentials() + kwargs = self._build_kwargs( + messages, + tools, + model, + max_tokens, + temperature, + reasoning_effort, + tool_choice, + ) + async with self._client.messages.stream(**kwargs) as stream: async for event in iter_with_stream_idle_timeout( stream, timeout_s=idle_timeout_s @@ -724,7 +771,9 @@ async def chat_stream( ) return result except Exception as e: - result = self._handle_error(e) + result = self.redact_error( + self._handle_error(e), e, [self.api_key, *self.extra_headers.values()] + ) return result finally: self._emit_observability( diff --git a/core/providers/base.py b/core/providers/base.py index 79f2b6ff9..2d1f35bdb 100644 --- a/core/providers/base.py +++ b/core/providers/base.py @@ -14,7 +14,6 @@ from core.reasoning import ReasoningChannel - ReasoningDeltaCallback = Callable[[str, ReasoningChannel], Awaitable[None]] _CONTEXT_WINDOW_MARKERS = ( @@ -94,6 +93,14 @@ def to_openai_tool_call(self) -> dict[str, Any]: return tool_call +class ProviderConfigurationError(ValueError): + """A frozen route or live credential is no longer valid for this Turn.""" + + +class ProviderCapabilityError(ValueError): + """The user explicitly declared this request capability unsupported.""" + + @dataclass class LLMResponse: """Response from an LLM provider.""" @@ -117,6 +124,7 @@ class LLMResponse: error_code: str | None = None error_retry_after_s: float | None = None error_should_retry: bool | None = None + partial_output: bool = False @property def has_tool_calls(self) -> bool: @@ -223,6 +231,72 @@ class LLMProvider(ABC): _SENTINEL = object() + request_guard = None + + async def refresh_request_credentials(self) -> None: + if self.request_guard is not None: + import asyncio + + try: + key = await asyncio.to_thread(self.request_guard) + except ValueError as exc: + raise ProviderConfigurationError(str(exc)) from exc + self._set_runtime_credential(key) + + async def aclose(self) -> None: + """Release transport resources owned by this provider instance.""" + + def _set_runtime_credential(self, key: str | None) -> None: + self.api_key = key + + @staticmethod + def redact_error(response: LLMResponse, error: Exception, values) -> LLMResponse: + """Keep echoed request credentials out of owned errors and observability.""" + secrets = {value for value in values if isinstance(value, str) and value} + try: + request = getattr(error, "request", None) + except RuntimeError: + request = None + headers = getattr(request, "headers", {}) + for name in ("authorization", "proxy-authorization", "x-api-key", "api-key"): + value = headers.get(name) + if isinstance(value, str) and value: + secrets.add(value) + if name.endswith("authorization") and " " in value: + token = value.split(" ", 1)[1] + if token: + secrets.add(token) + for attribute in ("content", "error_type", "error_code"): + text = getattr(response, attribute) + if isinstance(text, str): + for secret in sorted(secrets, key=len, reverse=True): + text = text.replace(secret, "[redacted]") + setattr(response, attribute, text) + return response + + input_modalities: tuple[str, ...] | None = None + tool_calling: bool | None = None + reasoning_supported: bool | None = None + + def validate_request_capabilities( + self, messages: list[dict], tools: list[dict] | None + ) -> None: + if tools and self.tool_calling is False: + raise ProviderCapabilityError( + "The selected model is declared not to support tool calling" + ) + if self.input_modalities is not None and "image" not in self.input_modalities: + for message in messages: + content = message.get("content") + if isinstance(content, list) and any( + isinstance(block, dict) + and block.get("type") in {"image", "image_url", "input_image"} + for block in content + ): + raise ProviderCapabilityError( + "The selected model is declared not to support image input" + ) + def __init__(self, api_key: str | None = None, api_base: str | None = None): self.api_key = api_key self.api_base = api_base @@ -562,19 +636,45 @@ async def chat_stream( response.reasoning_content, ReasoningChannel.PROVIDER_TRACE, ) - if on_content_delta and response.content: + if on_content_delta and response.content and response.finish_reason != "error": await on_content_delta(response.content) return response async def _safe_chat_stream(self, **kwargs: Any) -> LLMResponse: + delivered = False + content_callback = kwargs.get("on_content_delta") + reasoning_callback = kwargs.get("on_reasoning_delta") + + async def content_delta(text): + nonlocal delivered + delivered = delivered or bool(text) + if content_callback: + await content_callback(text) + + async def reasoning_delta(text, channel): + nonlocal delivered + delivered = delivered or bool(text) + if reasoning_callback: + await reasoning_callback(text, channel) + try: - return await self.chat_stream(**kwargs) + response = await self.chat_stream( + **{ + **kwargs, + "on_content_delta": content_delta, + "on_reasoning_delta": reasoning_delta, + } + ) except asyncio.CancelledError: raise except Exception as exc: - return LLMResponse( + response = LLMResponse( content=f"Error calling LLM: {exc}", finish_reason="error" ) + if response.finish_reason == "error" and delivered: + response.partial_output = True + response.error_should_retry = False + return response async def chat_stream_with_retry( self, @@ -770,7 +870,11 @@ async def _run_with_retry( while True: attempt += 1 response = await call(**kw) - if response.finish_reason != "error": + if ( + response.finish_reason != "error" + or response.partial_output + or response.error_kind in {"capability", "configuration"} + ): return response last_response = response error_key = (response.content or "").strip().lower() or None diff --git a/core/providers/catalog_service.py b/core/providers/catalog_service.py index e03c30351..e0828bee5 100644 --- a/core/providers/catalog_service.py +++ b/core/providers/catalog_service.py @@ -21,6 +21,7 @@ from core.providers.reasoning import ( ModelReasoningCapabilities, infer_reasoning_capabilities, + declared_reasoning_capabilities, ) @@ -32,6 +33,8 @@ class CatalogModel: max_output_tokens: int supported_parameters: tuple[str, ...] = () reasoning: ModelReasoningCapabilities | None = None + input_modalities: tuple[str, ...] | None = None + tool_calling: bool | None = None def to_dict(self) -> dict[str, Any]: return { @@ -41,6 +44,16 @@ def to_dict(self) -> dict[str, Any]: "maxOutputTokens": self.max_output_tokens, "supportedParameters": list(self.supported_parameters), "reasoning": self.reasoning.to_dict() if self.reasoning else None, + **( + {"inputModalities": list(self.input_modalities)} + if self.input_modalities is not None + else {} + ), + **( + {"toolCalling": self.tool_calling} + if self.tool_calling is not None + else {} + ), } @classmethod @@ -64,6 +77,10 @@ def from_dict(cls, value: Any) -> CatalogModel | None: for item in value.get("supportedParameters", []) if isinstance(item, str) ), + input_modalities=tuple(value["inputModalities"]) + if value.get("inputModalities") is not None + else None, + tool_calling=value.get("toolCalling"), reasoning=( ModelReasoningCapabilities.from_dict(value.get("reasoning")) or infer_reasoning_capabilities( @@ -472,6 +489,9 @@ def _has_declarations(entry: ManualModelConfig) -> bool: or entry.context_window is not None or entry.max_output_tokens is not None or entry.reasoning_efforts is not None + or entry.input_modalities is not None + or entry.tool_calling is not None + or bool(entry.compat.model_dump(exclude_none=True)) ) @@ -484,31 +504,22 @@ def _declared_model( anything it leaves unsaid falls through — to the discovered row when overlaying a remote catalog, otherwise to the built-in cascade.""" base = base if base is not None else _offline_model(entry.id) - if entry.reasoning_efforts is None: - reasoning = base.reasoning - elif entry.reasoning_efforts is False: - # Declared non-reasoning: block the inference fallback with an - # explicit "no controls" answer instead of an absent one. - reasoning = ModelReasoningCapabilities() - else: - levels = tuple( - dict.fromkeys( - level.strip().lower() - for level in entry.reasoning_efforts - if level.strip() - ) - ) - reasoning = ModelReasoningCapabilities( - supported_efforts=tuple(level for level in levels if level != "off"), - default_enabled=True, - mandatory="off" not in levels, - ) + reasoning = declared_reasoning_capabilities( + entry.reasoning_efforts, base.reasoning or ModelReasoningCapabilities() + ) return CatalogModel( id=entry.id, name=entry.label or base.name, context_window=entry.context_window or base.context_window, max_output_tokens=entry.max_output_tokens or base.max_output_tokens, reasoning=reasoning, + supported_parameters=base.supported_parameters, + input_modalities=tuple(entry.input_modalities) + if entry.input_modalities is not None + else base.input_modalities, + tool_calling=entry.tool_calling + if entry.tool_calling is not None + else base.tool_calling, ) diff --git a/core/providers/credentials.py b/core/providers/credentials.py index 7ddd25bf9..b4e406ab2 100644 --- a/core/providers/credentials.py +++ b/core/providers/credentials.py @@ -5,17 +5,16 @@ import hashlib import json import os +import secrets import threading -import uuid from pathlib import Path from typing import Any from core.config import deepcode_home from core.file_lock import exclusive_file_lock from core.private_storage import ( - ensure_private_directory, + atomic_write_private_json, open_existing_private_file, - open_private_file, ) @@ -49,16 +48,17 @@ def set(self, connection_id: str, api_key: str) -> None: clean = api_key.strip() if not clean: raise ValueError("api key must not be empty") - self._mutate( - lambda data: { + + def transform(data): + self._invalidate_login(data, connection_id) + data.setdefault("accounts", {}).pop(connection_id, None) + return { **data, "version": 1, - "connections": { - **_connections(data), - connection_id: clean, - }, + "connections": {**_connections(data), connection_id: clean}, } - ) + + self._mutate(transform) def clear(self, connection_id: str) -> bool: removed = False @@ -66,12 +66,78 @@ def clear(self, connection_id: str) -> bool: def transform(data: dict[str, Any]) -> dict[str, Any]: nonlocal removed connections = _connections(data) + self._invalidate_login(data, connection_id) + data.setdefault("accounts", {}).pop(connection_id, None) removed = connections.pop(connection_id, None) is not None return {**data, "version": 1, "connections": connections} self._mutate(transform) return removed + def oauth_credential(self, connection_id: str) -> tuple[str | None, str | None]: + data = self._read() + account = data.get("accounts", {}).get(connection_id, {}) + key = data.get("connections", {}).get(connection_id) + if ( + not isinstance(account, dict) + or account.get("provider") != "openrouter" + or not isinstance(account.get("accountId"), str) + or not account["accountId"] + or not isinstance(key, str) + or not key + ): + return None, None + return key, account.get("accountId") + + @staticmethod + def _invalidate_login(data, connection_id): + generation = secrets.token_hex(24) + data.setdefault("loginGenerations", {})[connection_id] = generation + return generation + + def begin_login(self, connection_id: str) -> str: + generation = None + + def transform(data): + nonlocal generation + generation = self._invalidate_login(data, connection_id) + return data + + self._mutate(transform) + return generation + + def cancel_login(self, connection_id: str, generation: str) -> None: + def transform(data): + if data.get("loginGenerations", {}).get(connection_id) == generation: + self._invalidate_login(data, connection_id) + return data + + self._mutate(transform) + + def complete_login( + self, connection_id: str, generation: str, *, api_key: str, account_id: str + ) -> None: + if not api_key or not account_id or len(account_id) > 256: + raise ValueError("The provider returned an invalid account identity") + + def transform(data): + if data.get("loginGenerations", {}).get(connection_id) != generation: + raise ValueError("This login was cancelled or superseded") + existing = data.get("accounts", {}).get(connection_id) + if existing and existing.get("accountId") != account_id: + raise ValueError( + "A different account was selected. Disconnect the existing account before switching." + ) + data.setdefault("connections", {})[connection_id] = api_key + data.setdefault("accounts", {})[connection_id] = { + "provider": "openrouter", + "accountId": account_id, + } + self._invalidate_login(data, connection_id) + return data + + self._mutate(transform) + def revision(self) -> str: """Return a non-secret fingerprint suitable for runtime invalidation.""" @@ -102,6 +168,9 @@ def _read(self) -> dict[str, Any]: raise ValueError("unsupported DeepCode credentials version") if not isinstance(value.get("connections", {}), dict): raise TypeError("credentials.connections must be an object") + for field in ("accounts", "loginGenerations"): + if not isinstance(value.get(field, {}), dict): + raise TypeError(f"credentials.{field} must be an object") return value def _mutate(self, transform) -> None: @@ -110,25 +179,7 @@ def _mutate(self, transform) -> None: self._replace(updated) def _replace(self, value: dict[str, Any]) -> None: - ensure_private_directory(self.path.parent) - temporary = self.path.with_name(f".{self.path.name}.{uuid.uuid4().hex}.tmp") - descriptor = open_private_file( - temporary, - os.O_WRONLY | os.O_CREAT | os.O_EXCL, - ) - try: - payload = (json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode() - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, self.path) - _fsync_directory(self.path.parent) - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass + atomic_write_private_json(self.path, value) def _connections(data: dict[str, Any]) -> dict[str, str]: @@ -136,14 +187,4 @@ def _connections(data: dict[str, Any]) -> dict[str, str]: return dict(value) if isinstance(value, dict) else {} -def _fsync_directory(path: Path) -> None: - if os.name == "nt": - return - descriptor = os.open(path, os.O_RDONLY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - - __all__ = ["CredentialStore", "default_credentials_path"] diff --git a/core/providers/oauth.py b/core/providers/oauth.py new file mode 100644 index 000000000..42708c56e --- /dev/null +++ b/core/providers/oauth.py @@ -0,0 +1,297 @@ +"""OpenRouter's PKCE-to-API-key login, with user-private account binding. + +OpenRouter issues a user-controlled key, not a refresh token. This adapter +therefore never invents refresh or remote-revocation operations. +""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import secrets +import threading +import time +import webbrowser +from dataclasses import dataclass, field +from urllib.parse import urlencode + +import httpx +from aiohttp import web + +from core.providers.credentials import CredentialStore + +FLOW_TTL = 300 + + +@dataclass(slots=True) +class _Flow: + id: str + connection_id: str + generation: str + expires_at: float + status: str = "starting" + url: str | None = None + error: str | None = None + account_id: str | None = None + ready: threading.Event = field(default_factory=threading.Event) + cancel: threading.Event = field(default_factory=threading.Event) + thread: threading.Thread | None = None + + +class ProviderOAuthManager: + def __init__(self, credentials: CredentialStore): + self.credentials = credentials + self._flows: dict[str, _Flow] = {} + self._lock = threading.RLock() + self._closed = False + + def start(self, connection_id: str, *, open_browser: bool = False) -> dict: + with self._lock: + if self._closed: + raise ValueError("Provider login service is closed") + self._flows = { + key: flow + for key, flow in self._flows.items() + if flow.expires_at + 300 > time.monotonic() + } + active = [ + flow + for flow in self._flows.values() + if flow.status in {"starting", "pending", "exchanging"} + ] + if len(active) >= 8: + raise ValueError( + "Too many pending provider logins; cancel a login first" + ) + for flow in active: + if flow.connection_id == connection_id: + self.cancel(flow.id) + flow = _Flow( + secrets.token_urlsafe(24), + connection_id, + self.credentials.begin_login(connection_id), + time.monotonic() + FLOW_TTL, + ) + self._flows[flow.id] = flow + # Keep a bounded terminal history without dropping active sockets. + terminal = [ + key + for key, item in self._flows.items() + if item.status not in {"starting", "pending", "exchanging"} + ] + for key in terminal[:-24]: + self._flows.pop(key) + flow.thread = threading.Thread( + target=self._run, + args=(flow,), + name="deepcode-provider-login", + daemon=True, + ) + flow.thread.start() + if not flow.ready.wait(5): + self.cancel(flow.id) + raise ValueError("The local provider login callback did not start") + result = self.poll(flow.id) + if open_browser and result["authorizationUrl"]: + webbrowser.open(result["authorizationUrl"]) + return result + + def poll(self, flow_id: str) -> dict: + with self._lock: + flow = self._flows.get(flow_id) + if flow is None: + raise ValueError("Unknown or expired provider login") + return { + "flowId": flow.id, + "connectionId": flow.connection_id, + "provider": "openrouter", + "status": flow.status, + "authorizationUrl": flow.url if flow.status == "pending" else None, + "expiresInSeconds": max(0, int(flow.expires_at - time.monotonic())), + "error": flow.error, + "accountId": flow.account_id, + "refreshSupported": False, + } + + def cancel(self, flow_id: str) -> dict: + with self._lock: + flow = self._flows.get(flow_id) + if flow is None: + raise ValueError("Unknown or expired provider login") + if flow.status in {"starting", "pending", "exchanging"}: + self.credentials.cancel_login(flow.connection_id, flow.generation) + flow.cancel.set() + flow.status = "cancelled" + flow.url = None + return self.poll(flow_id) + + def close(self): + with self._lock: + self._closed = True + flows = list(self._flows.values()) + for flow in flows: + self.cancel(flow.id) + deadline = time.monotonic() + 3 + for flow in flows: + if flow.thread is not None: + flow.thread.join(max(0, deadline - time.monotonic())) + + def _run(self, flow): + try: + asyncio.run(self._authorize(flow)) + except ( + Exception + ): # callback startup/transport errors must not expose codes or keys + with self._lock: + if flow.status != "cancelled": + flow.status = "failed" + flow.error = "Provider authorization could not be completed" + finally: + flow.url = None + flow.ready.set() + + async def _authorize(self, flow): + verifier = secrets.token_urlsafe(48) + challenge = ( + base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()) + .decode() + .rstrip("=") + ) + state = secrets.token_urlsafe(32) + callback_path = "/auth/provider/" + state + done = asyncio.Event() + exchange = None + allowed_hosts = set() + + async def callback(request): + nonlocal exchange + if request.host not in allowed_hosts or request.remote != "127.0.0.1": + raise web.HTTPForbidden() + with self._lock: + if flow.status != "pending" or flow.cancel.is_set(): + raise web.HTTPConflict(text="This login is no longer pending") + codes = request.query.getall("code", []) + if "error" in request.query: + flow.status = "cancelled" + self.credentials.cancel_login(flow.connection_id, flow.generation) + done.set() + return web.Response(text="Authorization cancelled") + if len(codes) != 1 or not 1 <= len(codes[0]) <= 4096: + raise web.HTTPBadRequest(text="Missing authorization code") + flow.status = "exchanging" + exchange = asyncio.current_task() + try: + async with httpx.AsyncClient( + timeout=20, follow_redirects=False + ) as client: + response = await client.post( + "https://openrouter.ai/api/v1/auth/keys", + json={ + "code": codes[0], + "code_verifier": verifier, + "code_challenge_method": "S256", + }, + ) + response.raise_for_status() + if len(response.content) > 65536: + raise ValueError("Invalid authorization response") + value = response.json() + key = value.get("key") + account = value.get("user_id") + if ( + not isinstance(key, str) + or not key + or not isinstance(account, str) + or not account + ): + raise ValueError( + "The provider returned no verifiable account identity" + ) + with self._lock: + if flow.cancel.is_set(): + raise ValueError("This login was cancelled") + self.credentials.complete_login( + flow.connection_id, + flow.generation, + api_key=key, + account_id=account, + ) + flow.account_id = account + flow.status = "authenticated" + except ValueError as exc: + with self._lock: + if flow.status != "cancelled": + flow.status = "failed" + # Only our credential-store errors are projected; never an HTTP body. + flow.error = ( + str(exc) + if str(exc) + in { + "This login was cancelled or superseded", + "A different account was selected. Disconnect the existing account before switching.", + "The provider returned no verifiable account identity", + } + else "The provider returned an invalid authorization response" + ) + except Exception: + with self._lock: + if flow.status != "cancelled": + flow.status = "failed" + flow.error = ( + "The provider rejected or could not complete authorization" + ) + finally: + done.set() + nonce = secrets.token_urlsafe(18) + return web.Response( + content_type="text/html", + text=f'<!doctype html><meta charset="utf-8"><title>DeepCode

Authorization processed. Return to DeepCode to check its status.

', + headers={ + "Cache-Control": "no-store", + "Referrer-Policy": "no-referrer", + "Content-Security-Policy": f"default-src 'none'; script-src 'nonce-{nonce}'; frame-ancestors 'none'", + }, + ) + + app = web.Application(client_max_size=8192) + app.router.add_get(callback_path, callback) + runner = web.AppRunner(app, access_log=None, shutdown_timeout=1) + await runner.setup() + try: + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = runner.addresses[0][1] + allowed_hosts.update({f"127.0.0.1:{port}", f"localhost:{port}"}) + url = "https://openrouter.ai/auth?" + urlencode( + { + "callback_url": f"http://localhost:{port}{callback_path}", + "code_challenge": challenge, + "code_challenge_method": "S256", + } + ) + with self._lock: + if not flow.cancel.is_set(): + flow.url = url + flow.status = "pending" + flow.ready.set() + while ( + not done.is_set() + and not flow.cancel.is_set() + and time.monotonic() < flow.expires_at + ): + await asyncio.sleep(0.1) + with self._lock: + if flow.status in {"pending", "exchanging"}: + flow.status = "expired" if not flow.cancel.is_set() else "cancelled" + self.credentials.cancel_login(flow.connection_id, flow.generation) + if ( + exchange is not None + and not exchange.done() + and (flow.cancel.is_set() or flow.status == "expired") + ): + exchange.cancel() + if done.is_set(): + await asyncio.sleep(0.1) # allow the bounded callback response to flush + finally: + await runner.cleanup() diff --git a/core/providers/openai_compat.py b/core/providers/openai_compat.py index f63010868..330ca1384 100644 --- a/core/providers/openai_compat.py +++ b/core/providers/openai_compat.py @@ -2,9 +2,9 @@ from __future__ import annotations -import json import hashlib import importlib.util +import json import os import secrets import string @@ -15,6 +15,7 @@ import json_repair from loguru import logger +from openai import Omit from core.observability import log_llm_call @@ -32,6 +33,8 @@ from core.providers.base import ( LLMProvider, + ProviderCapabilityError, + ProviderConfigurationError, LLMResponse, ReasoningDeltaCallback, ToolCallRequest, @@ -43,13 +46,17 @@ convert_tools, parse_response_output, ) -from core.providers.reasoning import OPENROUTER_REASONING_DETAILS -from core.reasoning import ReasoningChannel +from core.providers.protocol_config import ProviderCompat, apply_chat_compat +from core.providers.reasoning import ( + OPENROUTER_REASONING_DETAILS, + infer_reasoning_capabilities, +) from core.providers.timeouts import ( StreamIdleTimeoutError, iter_with_stream_idle_timeout, resolve_stream_idle_timeout_s, ) +from core.reasoning import ReasoningChannel if TYPE_CHECKING: from core.providers.registry import ProviderSpec @@ -231,11 +238,20 @@ def __init__( default_model: str = "gpt-4o", extra_headers: dict[str, str] | None = None, spec: ProviderSpec | None = None, + protocol: str = "auto", + compat: ProviderCompat | None = None, + auth_mode: str = "api_key", ): super().__init__(api_key, api_base) self.default_model = default_model self.extra_headers = extra_headers or {} self._spec = spec + if protocol not in {"auto", "openai_chat", "openai_responses"}: + raise ValueError("Unsupported OpenAI-compatible protocol") + self.protocol = protocol + self.auth_mode = auth_mode + self.compat = compat or ProviderCompat() + self.compat.validate_protocol(protocol) # The credential travels only on this instance and its client. It is # deliberately never exported to os.environ: in a long-lived App @@ -252,9 +268,17 @@ def __init__( default_headers.update(_DEFAULT_REQUESTY_HEADERS) if extra_headers: default_headers.update(extra_headers) + if auth_mode == "none": + # Omit is the SDK's supported way to remove a default header. + # An empty key is rejected by some SDK versions at construction. + default_headers = { + key: value + for key, value in default_headers.items() + if key.lower() != "authorization" + } self._client = AsyncOpenAI( - api_key=api_key or "no-key", + api_key="no-key" if auth_mode == "none" else api_key or "no-key", base_url=effective_base, default_headers=default_headers, max_retries=0, @@ -266,6 +290,16 @@ def __init__( self._responses_failures: dict[str, int] = {} self._responses_tripped_at: dict[str, float] = {} + async def aclose(self) -> None: + await self._client.close() + + def _set_runtime_credential(self, key: str | None) -> None: + self.api_key = key + self._client.api_key = "no-key" if self.auth_mode == "none" else key or "no-key" + + def _request_headers(self) -> dict: + return {"Authorization": Omit()} if self.auth_mode == "none" else {} + @classmethod def _apply_cache_control( cls, @@ -427,6 +461,7 @@ def _build_kwargs( :func:`resolve_model_compat`; this method only assembles the payload from that value — no ``model_name.lower()`` branching inline. """ + self.validate_request_capabilities(messages, tools) model_name = model or self.default_model spec = self._spec @@ -482,14 +517,26 @@ def _build_kwargs( if msg.get("role") == "assistant" and "reasoning_content" not in msg: msg["reasoning_content"] = "" - return kwargs + return apply_chat_compat( + kwargs, + self.compat.model_copy(update={"reasoning_field": "omit"}) + if self.reasoning_supported is False + else self.compat, + max_tokens=max_tokens, + temperature=temperature, + reasoning_effort=reasoning_effort, + ) def _should_use_responses_api( self, model: str | None, reasoning_effort: str | None, ) -> bool: - """Use Responses API only for direct OpenAI requests that benefit from it.""" + """Explicit choices are authoritative; auto retains the legacy heuristic.""" + if self.protocol == "openai_responses": + return True + if self.protocol == "openai_chat": + return False if self._spec and self._spec.name != "openai": return False if not _is_direct_openai_base(self._effective_base): @@ -573,7 +620,8 @@ def _build_responses_body( reasoning_effort: str | None, tool_choice: str | dict[str, Any] | None, ) -> dict[str, Any]: - """Build a Responses API body for direct OpenAI requests.""" + """Build a Responses API body for the explicitly selected or legacy route.""" + self.validate_request_capabilities(messages, tools) model_name = model or self.default_model sanitized_messages = self._sanitize_messages( self._sanitize_empty_content(messages), preserve_provider_state=True @@ -595,10 +643,22 @@ def _build_responses_body( compat = resolve_model_compat( model_name=model_name, spec=self._spec, reasoning_effort=reasoning_effort ) - if compat.include_temperature: + if self.compat.temperature is True or ( + self.compat.temperature is None and compat.include_temperature + ): body["temperature"] = temperature - if self._should_use_responses_api(model, reasoning_effort): + wants_reasoning = ( + self.protocol == "auto" + or self.compat.reasoning_field == "reasoning" + or reasoning_effort not in {None, "auto", "none"} + or infer_reasoning_capabilities(model_name) is not None + ) + if ( + wants_reasoning + and self.compat.reasoning_field != "omit" + and self.reasoning_supported is not False + ): body["reasoning"] = {"summary": "auto"} if reasoning_effort: body["reasoning"]["effort"] = reasoning_effort.lower() @@ -607,6 +667,8 @@ def _build_responses_body( if tools: body["tools"] = convert_tools(tools) body["tool_choice"] = tool_choice or "auto" + if self.compat.parallel_tool_calls is not None: + body["parallel_tool_calls"] = self.compat.parallel_tool_calls return body @@ -1105,6 +1167,15 @@ def _handle_error( spec: ProviderSpec | None = None, api_base: str | None = None, ) -> LLMResponse: + if isinstance(e, (ProviderCapabilityError, ProviderConfigurationError)): + return LLMResponse( + content=str(e), + finish_reason="error", + error_kind="capability" + if isinstance(e, ProviderCapabilityError) + else "configuration", + error_should_retry=False, + ) body = ( getattr(e, "doc", None) or getattr(e, "body", None) @@ -1161,6 +1232,7 @@ async def chat( started = time.monotonic() response: LLMResponse | None = None try: + await self.refresh_request_credentials() if self._should_use_responses_api(model, reasoning_effort): try: body = self._build_responses_body( @@ -1173,13 +1245,20 @@ async def chat( tool_choice, ) result = parse_response_output( - await self._client.responses.create(**body) + await self._client.responses.create( + **body, extra_headers=self._request_headers() + ) ) self._record_responses_success(model, reasoning_effort) response = result return result except Exception as responses_error: - if not self._should_fallback_from_responses_error(responses_error): + if ( + self.protocol == "openai_responses" + or not self._should_fallback_from_responses_error( + responses_error + ) + ): raise self._record_responses_failure(model, reasoning_effort) @@ -1192,10 +1271,18 @@ async def chat( reasoning_effort, tool_choice, ) - response = self._parse(await self._client.chat.completions.create(**kwargs)) + response = self._parse( + await self._client.chat.completions.create( + **kwargs, extra_headers=self._request_headers() + ) + ) return response except Exception as e: - response = self._handle_error(e, spec=self._spec, api_base=self.api_base) + response = self.redact_error( + self._handle_error(e, spec=self._spec, api_base=self.api_base), + e, + [self.api_key, *self.extra_headers.values()], + ) return response finally: self._emit_observability( @@ -1221,7 +1308,9 @@ async def chat_stream( idle_timeout_s = resolve_stream_idle_timeout_s() started = time.monotonic() response: LLMResponse | None = None + stream = None try: + await self.refresh_request_credentials() if self._should_use_responses_api(model, reasoning_effort): try: body = self._build_responses_body( @@ -1234,7 +1323,9 @@ async def chat_stream( tool_choice, ) body["stream"] = True - stream = await self._client.responses.create(**body) + stream = await self._client.responses.create( + **body, extra_headers=self._request_headers() + ) async def _timed_stream(): async for event in iter_with_stream_idle_timeout( @@ -1265,7 +1356,13 @@ async def _timed_stream(): ) return response except Exception as responses_error: - if not self._should_fallback_from_responses_error(responses_error): + if ( + stream is not None + or self.protocol == "openai_responses" + or not self._should_fallback_from_responses_error( + responses_error + ) + ): raise self._record_responses_failure(model, reasoning_effort) @@ -1280,7 +1377,9 @@ async def _timed_stream(): ) kwargs["stream"] = True kwargs["stream_options"] = {"include_usage": True} - stream = await self._client.chat.completions.create(**kwargs) + stream = await self._client.chat.completions.create( + **kwargs, extra_headers=self._request_headers() + ) chunks: list[Any] = [] async for chunk in iter_with_stream_idle_timeout( stream, timeout_s=idle_timeout_s @@ -1295,6 +1394,12 @@ async def _timed_stream(): chunk.choices[0].delta, on_reasoning_delta, ) + if not any( + choice.finish_reason is not None + for chunk in chunks + for choice in chunk.choices + ): + raise RuntimeError("Chat stream ended before its terminal event") response = self._parse_chunks(chunks) return response except StreamIdleTimeoutError: @@ -1308,9 +1413,15 @@ async def _timed_stream(): ) return response except Exception as e: - response = self._handle_error(e, spec=self._spec, api_base=self.api_base) + response = self.redact_error( + self._handle_error(e, spec=self._spec, api_base=self.api_base), + e, + [self.api_key, *self.extra_headers.values()], + ) return response finally: + if stream is not None: + await stream.close() self._emit_observability( model=model, messages=messages, diff --git a/core/providers/openai_responses/parsing.py b/core/providers/openai_responses/parsing.py index 823e5c634..57d915e4b 100644 --- a/core/providers/openai_responses/parsing.py +++ b/core/providers/openai_responses/parsing.py @@ -250,6 +250,7 @@ async def consume_sdk_stream( tool_calls: list[ToolCallRequest] = [] tool_call_buffers: dict[str, dict[str, Any]] = {} finish_reason = "stop" + terminal_seen = False usage: dict[str, int] = {} reasoning_content: str | None = None reasoning_items: list[dict[str, Any]] = [] @@ -316,7 +317,8 @@ async def consume_sdk_stream( arguments=args, ) ) - elif event_type == "response.completed": + elif event_type in {"response.completed", "response.incomplete"}: + terminal_seen = True resp = getattr(event, "response", None) status = getattr(resp, "status", None) if resp else None finish_reason = map_finish_reason(status) @@ -351,6 +353,8 @@ async def consume_sdk_stream( ) raise RuntimeError(f"Response failed: {str(detail)[:500]}") + if not terminal_seen: + raise RuntimeError("Responses stream ended before its terminal event") provider_state = ( {OPENAI_RESPONSE_REASONING_ITEMS: reasoning_items} if reasoning_items else None ) diff --git a/core/providers/profiles.py b/core/providers/profiles.py index 1b910b83a..199f6e9d0 100644 --- a/core/providers/profiles.py +++ b/core/providers/profiles.py @@ -6,7 +6,8 @@ import json import os import re -from dataclasses import dataclass +from collections.abc import Callable +from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any from core.config import ConfigError, ManualModelConfig @@ -14,12 +15,15 @@ from core.providers.base import GenerationSettings, LLMProvider from core.providers.catalog import resolve_model_info from core.providers.credentials import CredentialStore -from core.providers.registry import PROVIDERS, ProviderSpec, find_by_model, find_by_name +from core.providers.protocol_config import ProviderCompat, protocol_adapter from core.providers.reasoning import ( ModelReasoningCapabilities, infer_reasoning_capabilities, + declared_reasoning_capabilities, resolve_reasoning_effort, ) +from core.providers.registry import PROVIDERS, ProviderSpec, find_by_model, find_by_name +from core.providers.revisions import ProviderRevisionStore, credential_digest if TYPE_CHECKING: from core.config import ConnectionProfileConfig, DeepCodeConfig @@ -51,6 +55,10 @@ class ResolvedConnection: # Full per-model declarations (label/capacities/efforts). ``manual_models`` # stays the plain id tuple existing consumers read; these carry the rest. manual_model_entries: tuple[ManualModelConfig, ...] = () + account_id: str | None = None + protocol: str = "auto" + auth: str = "api_key" + compat: ProviderCompat = field(default_factory=ProviderCompat) def public_view(self) -> dict[str, Any]: return { @@ -58,6 +66,10 @@ def public_view(self) -> dict[str, Any]: "label": self.label, "providerName": self.provider_name, "adapter": self.adapter, + "protocol": self.protocol, + "auth": self.auth, + "accountId": self.account_id, + "compat": self.compat.model_dump(by_alias=True, exclude_none=True), "apiBase": self.api_base, "apiKeyEnv": None, "modelCatalog": self.model_catalog_setting, @@ -76,9 +88,11 @@ def public_view(self) -> dict[str, Any]: def is_configured(self) -> bool: credential_ready = ( bool(self.api_key) - or self.local - or self.spec.is_direct - or self.spec.is_oauth + or self.auth == "none" + or ( + self.protocol == "auto" + and (self.local or self.spec.is_direct or self.spec.is_oauth) + ) ) endpoint_ready = not self.spec.requires_api_base or bool(self.api_base) return credential_ready and endpoint_ready @@ -95,9 +109,17 @@ def __init__( self, config: "DeepCodeConfig", credentials: CredentialStore | None = None, + *, + config_loader: Callable[[], "DeepCodeConfig"] | None = None, + credential_overrides: dict[str, str | None] | None = None, ) -> None: self.config = config + self.config_loader = config_loader + self._credential_overrides = dict(credential_overrides or {}) self.credentials = credentials or CredentialStore() + self.revisions = ProviderRevisionStore( + self.credentials.path.parent / "provider_revisions" + ) def list_connections( self, *, include_unconfigured: bool = True @@ -176,15 +198,22 @@ def execution_profile( phase: str = "implementation", model_limits: tuple[int, int] | None = None, reasoning_capabilities: ModelReasoningCapabilities | None = None, + persist_revision: bool = True, ) -> ExecutionProfile: normalized = (selection or ExecutionSelection()).normalized() connection, model = self.resolve_selection(normalized, phase=phase) settings = self.config.resolve_phase(phase) + entry = next( + (item for item in connection.manual_model_entries if item.id == model), None + ) info = resolve_model_info(model) published_context_window, max_output_tokens = model_limits or ( info.context_window, info.max_output_tokens, ) + if entry is not None: + published_context_window = entry.context_window or published_context_window + max_output_tokens = entry.max_output_tokens or max_output_tokens if published_context_window < 1 or max_output_tokens < 1: raise ConfigError(f"Invalid model limits for '{model}'") max_tokens = min(settings.max_tokens, max_output_tokens) @@ -205,14 +234,61 @@ def execution_profile( model, provider_name=connection.provider_name, ) + if entry is not None: + capabilities = declared_reasoning_capabilities( + entry.reasoning_efforts, capabilities + ) + reasoning_supported = ( + None + if entry is None or entry.reasoning_efforts is None + else entry.reasoning_efforts is not False + ) + if reasoning_supported is False and normalized.reasoning_effort not in { + None, + "auto", + "none", + "off", + }: + raise ConfigError("This model is explicitly declared non-reasoning") try: reasoning_effort = resolve_reasoning_effort( requested=normalized.reasoning_effort, - configured=settings.reasoning_effort, + configured=settings.reasoning_effort + if reasoning_supported is not False + else None, capabilities=capabilities, ) except ValueError as exc: raise ConfigError(str(exc)) from exc + compat = ProviderCompat.model_validate( + { + **connection.compat.model_dump(exclude_none=True), + **( + entry.compat.model_dump(exclude_none=True) + if entry is not None + else {} + ), + } + ) + revision = ( + self.revisions.put( + { + "schemaVersion": 1, + "connectionId": connection.id, + "providerName": connection.provider_name, + "adapter": connection.adapter, + "protocol": connection.protocol, + "auth": connection.auth, + "apiBase": connection.api_base, + "extraHeaders": connection.extra_headers, + "compat": compat.model_dump(by_alias=True, exclude_none=True), + "credentialDigest": credential_digest(connection.api_key), + "credentialAccount": connection.account_id, + } + ) + if persist_revision + else None + ) return ExecutionProfile( connection_id=connection.id, provider_name=connection.provider_name, @@ -224,6 +300,13 @@ def execution_profile( temperature=settings.temperature, reasoning_effort=reasoning_effort, config_revision=self.connection_revision(connection), + protocol=connection.protocol, + provider_revision=revision, + input_modalities=tuple(entry.input_modalities) + if entry is not None and entry.input_modalities is not None + else None, + tool_calling=entry.tool_calling if entry is not None else None, + reasoning_supported=reasoning_supported, ) def build_provider(self, profile: ExecutionProfile) -> LLMProvider: @@ -236,6 +319,7 @@ def build_provider(self, profile: ExecutionProfile) -> LLMProvider: api_base=connection.api_base, default_model=profile.model_id, extra_headers=connection.extra_headers, + compat=connection.compat, ) elif connection.adapter == "openai_compat": from core.providers.openai_compat import OpenAICompatProvider @@ -246,12 +330,28 @@ def build_provider(self, profile: ExecutionProfile) -> LLMProvider: default_model=profile.model_id, extra_headers=connection.extra_headers, spec=connection.spec, + protocol=connection.protocol, + compat=connection.compat, + auth_mode=connection.auth, ) else: raise ConfigError( f"Unsupported adapter '{connection.adapter}' " f"for connection '{connection.id}'" ) + + def current_credential(): + resolver = ConnectionResolver( + self.config_loader() if self.config_loader else self.config, + self.credentials, + credential_overrides=self._credential_overrides, + ) + return resolver.connection_for_profile(profile).api_key + + provider.request_guard = current_credential + provider.input_modalities = profile.input_modalities + provider.tool_calling = profile.tool_calling + provider.reasoning_supported = profile.reasoning_supported provider.generation = GenerationSettings( temperature=profile.temperature, max_tokens=profile.max_tokens, @@ -259,27 +359,81 @@ def build_provider(self, profile: ExecutionProfile) -> LLMProvider: ) return provider - def connection_for_profile( - self, - profile: ExecutionProfile, - ) -> ResolvedConnection: - """Resolve credentials while rejecting non-secret config drift.""" - + def connection_for_profile(self, profile: ExecutionProfile) -> ResolvedConnection: + """Resolve live credentials against a frozen private route, or legacy revision.""" connection = self.resolve_connection( - profile.connection_id, - model=profile.model_id, + profile.connection_id, model=profile.model_id ) - if self.connection_revision(connection) != profile.config_revision: - raise ConfigError( - f"LLM connection '{connection.id}' changed after this Turn was " - "accepted. Retry with the current Session selection." - ) if not connection.is_usable: raise ConfigError( - f"LLM connection '{connection.id}' has no credential. " - "Configure it in Desktop Settings or with `deepcode provider set`." + f"LLM connection '{connection.id}' is disabled or has no credential" ) - return connection + if profile.provider_revision is None: + if self.connection_revision(connection) != profile.config_revision: + raise ConfigError( + f"LLM connection '{connection.id}' changed after this Turn was accepted. Retry with the current Session selection." + ) + entry = next( + ( + item + for item in connection.manual_model_entries + if item.id == profile.model_id + ), + None, + ) + if entry is not None: + connection = replace( + connection, + compat=ProviderCompat.model_validate( + { + **connection.compat.model_dump(exclude_none=True), + **entry.compat.model_dump(exclude_none=True), + } + ), + ) + return connection + try: + snapshot = self.revisions.get(profile.provider_revision) + if ( + snapshot.get("schemaVersion") != 1 + or snapshot.get("connectionId") != connection.id + or snapshot.get("providerName") != connection.provider_name + or snapshot.get("auth") != connection.auth + or snapshot.get("protocol") != profile.protocol + or snapshot.get("adapter") != profile.adapter + or snapshot.get("credentialAccount") != connection.account_id + ): + raise ValueError("Provider identity changed after admission") + if connection.extra_headers != snapshot["extraHeaders"]: + raise ValueError( + "Header credentials or configuration changed after this Turn was accepted; resubmit with current settings" + ) + if connection.api_key is None and snapshot[ + "credentialDigest" + ] != credential_digest(None): + raise ValueError("The credential used by this Turn has been removed") + if ( + credential_digest(connection.api_key) != snapshot["credentialDigest"] + and connection.api_base != snapshot["apiBase"] + ): + raise ValueError( + "Credential and endpoint changed after this Turn was accepted" + ) + return replace( + connection, + adapter=snapshot["adapter"], + protocol=snapshot["protocol"], + api_base=snapshot["apiBase"], + extra_headers=dict(snapshot["extraHeaders"]), + compat=ProviderCompat.model_validate(snapshot["compat"]), + manual_model_entries=(), + ) + except (ValueError, KeyError, TypeError) as exc: + raise ConfigError( + str(exc) + if isinstance(exc, ValueError) + else "Invalid private provider revision; resubmit with current settings" + ) from exc @staticmethod def connection_revision(connection: ResolvedConnection) -> str: @@ -293,6 +447,17 @@ def connection_revision(connection: ResolvedConnection) -> str: "apiBase": connection.api_base, "extraHeaders": connection.extra_headers, "enabled": connection.enabled, + **( + {"protocol": connection.protocol} + if connection.protocol != "auto" + else {} + ), + **({"auth": connection.auth} if connection.auth != "api_key" else {}), + **( + {"compat": connection.compat.model_dump(exclude_none=True)} + if connection.compat.model_dump(exclude_none=True) + else {} + ), }, sort_keys=True, separators=(",", ":"), @@ -334,20 +499,28 @@ def _profile_connection( legacy_key=legacy_key, key_optional=spec.is_local or spec.is_direct or spec.is_oauth, ) + account_id = None + if profile.auth == "oauth": + api_key, account_id = self.credentials.oauth_credential(connection_id) + source = "oauth" if api_key else "missing" model_entries = _clean_model_entries(profile.manual_models) return ResolvedConnection( id=connection_id, label=profile.label.strip() or connection_id, provider_name=spec.name, - adapter=profile.adapter or spec.backend, - api_key=api_key, + adapter=protocol_adapter(profile.protocol, profile.adapter or spec.backend), + protocol=profile.protocol, + account_id=account_id, + auth=profile.auth, + compat=profile.compat, + api_key=api_key if profile.auth != "none" else None, api_base=profile.api_base or spec.default_api_base or None, extra_headers=dict(profile.extra_headers), - model_catalog=_catalog_kind(profile.model_catalog, spec), + model_catalog=_catalog_kind(profile.model_catalog, spec, profile.protocol), model_catalog_setting=profile.model_catalog, manual_models=tuple(entry.id for entry in model_entries), manual_model_entries=model_entries, - credential_source=source, + credential_source=source if profile.auth != "none" else "not_required", local=spec.is_local, enabled=profile.enabled, spec=spec, @@ -367,6 +540,11 @@ def template_connection(self, template: str) -> ResolvedConnection: def _legacy_connection(self, spec: ProviderSpec) -> ResolvedConnection: provider = getattr(self.config.providers, spec.name) + if ( + provider.auth == "none" + and protocol_adapter(provider.protocol, spec.backend) == "anthropic" + ): + raise ConfigError("Anthropic Messages requires an API key") api_key, source = self._credential( spec.name, env_name=spec.env_key, @@ -377,14 +555,17 @@ def _legacy_connection(self, spec: ProviderSpec) -> ResolvedConnection: id=spec.name, label=spec.label, provider_name=spec.name, - adapter=spec.backend, - api_key=api_key, + adapter=protocol_adapter(provider.protocol, spec.backend), + protocol=provider.protocol, + auth=provider.auth, + compat=provider.compat, + api_key=api_key if provider.auth != "none" else None, api_base=provider.api_base or spec.default_api_base or None, extra_headers=dict(provider.extra_headers or {}), - model_catalog=_catalog_kind("auto", spec), + model_catalog=_catalog_kind("auto", spec, provider.protocol), model_catalog_setting="auto", manual_models=(), - credential_source=source, + credential_source=source if provider.auth != "none" else "not_required", local=spec.is_local, enabled=True, spec=spec, @@ -398,6 +579,9 @@ def _credential( legacy_key: str | None, key_optional: bool, ) -> tuple[str | None, str]: + if connection_id in self._credential_overrides: + value = self._credential_overrides[connection_id] + return value, "request" if value else "missing" if env_name and os.environ.get(env_name): return os.environ[env_name], "environment" stored = self.credentials.get(connection_id) @@ -433,9 +617,13 @@ def _clean_model_entries( return tuple(entries.values()) -def _catalog_kind(configured: str, spec: ProviderSpec) -> str: +def _catalog_kind(configured: str, spec: ProviderSpec, protocol: str = "auto") -> str: if configured != "auto": return configured + if protocol == "anthropic_messages": + return "anthropic" + if protocol in {"openai_chat", "openai_responses"} and spec.backend == "anthropic": + return "openai" if spec.name == "openrouter": return "openrouter" if spec.name == "anthropic": diff --git a/core/providers/protocol_config.py b/core/providers/protocol_config.py new file mode 100644 index 000000000..a66df6031 --- /dev/null +++ b/core/providers/protocol_config.py @@ -0,0 +1,127 @@ +"""Explicit provider wire choices; every compatibility field has an encoder.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict +from pydantic.alias_generators import to_camel + +ProviderProtocol = Literal[ + "auto", "openai_chat", "openai_responses", "anthropic_messages" +] + + +class ProviderCompat(BaseModel): + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + extra="forbid", + frozen=True, + hide_input_in_errors=True, + ) + + token_limit_field: Literal["max_tokens", "max_completion_tokens"] | None = None + temperature: bool | None = None + system_role: Literal["system", "developer", "user"] | None = None + reasoning_field: Literal["reasoning_effort", "reasoning", "omit"] | None = None + reasoning_content: Literal["preserve", "empty", "omit"] | None = None + tool_message_name: bool | None = None + parallel_tool_calls: bool | None = None + + def validate_protocol(self, protocol: str) -> None: + supplied = self.model_dump(exclude_none=True) + if not supplied: + return + if protocol == "openai_chat": + return + if protocol == "anthropic_messages" and set(supplied) <= {"temperature"}: + return + if protocol == "openai_responses" and set(supplied) <= { + "temperature", + "reasoning_field", + "parallel_tool_calls", + }: + if self.reasoning_field != "reasoning_effort": + return + raise ValueError( + "Compatibility overrides require an explicit matching protocol; the selected fields are unsupported for this protocol" + ) + + +def protocol_adapter(protocol: str, legacy_adapter: str) -> str: + if protocol == "auto": + return legacy_adapter + return "anthropic" if protocol == "anthropic_messages" else "openai_compat" + + +def apply_chat_compat( + body: dict, + compat: ProviderCompat, + *, + max_tokens: int, + temperature: float, + reasoning_effort: str | None, +) -> dict: + """Apply explicit overrides after the built-in model defaults, without user JSON merging.""" + if compat.token_limit_field is not None: + body.pop("max_tokens", None) + body.pop("max_completion_tokens", None) + body[compat.token_limit_field] = max(1, max_tokens) + if compat.temperature is False: + body.pop("temperature", None) + elif compat.temperature is True: + body["temperature"] = temperature + messages = [dict(message) for message in body["messages"]] + if compat.system_role is not None: + for message in messages: + if message.get("role") in {"system", "developer"}: + message["role"] = compat.system_role + if compat.reasoning_field is not None: + body.pop("reasoning_effort", None) + extra = dict(body.get("extra_body", {})) + for key in ("reasoning", "thinking", "enable_thinking", "reasoning_split"): + extra.pop(key, None) + effort = reasoning_effort.lower() if reasoning_effort else None + if compat.reasoning_field == "reasoning_effort" and effort not in { + None, + "auto", + }: + body["reasoning_effort"] = effort + elif compat.reasoning_field == "reasoning" and effort not in {None, "auto"}: + extra["reasoning"] = ( + {"enabled": False} if effort == "none" else {"effort": effort} + ) + if extra: + body["extra_body"] = extra + else: + body.pop("extra_body", None) + if compat.reasoning_content is not None: + for message in messages: + if message.get("role") == "assistant": + if compat.reasoning_content == "empty": + message.setdefault("reasoning_content", "") + elif compat.reasoning_content == "omit": + message.pop("reasoning_content", None) + if compat.tool_message_name is not None: + calls = { + call["id"]: call.get("function", {}).get("name") + for message in messages + for call in message.get("tool_calls", []) + if isinstance(call, dict) and "id" in call + } + for message in messages: + if message.get("role") == "tool": + if not compat.tool_message_name: + message.pop("name", None) + elif not message.get("name"): + name = calls.get(message.get("tool_call_id")) + if not name: + raise ValueError( + "The configured protocol requires a tool result name, but its matching tool call is missing" + ) + message["name"] = name + if compat.parallel_tool_calls is not None and body.get("tools"): + body["parallel_tool_calls"] = compat.parallel_tool_calls + body["messages"] = messages + return body diff --git a/core/providers/reasoning.py b/core/providers/reasoning.py index c7ad114c3..e181915b8 100644 --- a/core/providers/reasoning.py +++ b/core/providers/reasoning.py @@ -95,6 +95,25 @@ def from_dict(cls, value: Any) -> "ModelReasoningCapabilities | None": return None +def declared_reasoning_capabilities( + efforts: list[str] | bool | None, + fallback: ModelReasoningCapabilities | None, +) -> ModelReasoningCapabilities | None: + """Resolve the same manual capability declaration for catalog and execution.""" + if efforts is None: + return fallback + if efforts is False: + return ModelReasoningCapabilities() + levels = tuple( + dict.fromkeys(level.strip().lower() for level in efforts if level.strip()) + ) + return ModelReasoningCapabilities( + supported_efforts=tuple(level for level in levels if level != "off"), + default_enabled=True, + mandatory="off" not in levels, + ) + + def resolve_reasoning_effort( *, requested: str | None, diff --git a/core/providers/revisions.py b/core/providers/revisions.py new file mode 100644 index 000000000..da63f53f7 --- /dev/null +++ b/core/providers/revisions.py @@ -0,0 +1,73 @@ +"""Private, content-addressed provider routes referenced by admitted Turns.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from pathlib import Path + +from core.file_lock import exclusive_file_lock +from core.private_storage import atomic_write_private_json, open_existing_private_file + + +def credential_digest(value: str | None) -> str: + return hashlib.sha256((value or "").encode()).hexdigest() + + +class ProviderRevisionStore: + """Keep route/header snapshots private; API key bodies are never stored here.""" + + def __init__(self, directory: Path): + self.directory = directory + + @staticmethod + def _identity(value: dict) -> str: + return hashlib.sha256( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode() + ).hexdigest() + + def put(self, value: dict) -> str: + if len(json.dumps(value).encode()) > 65536: + raise ValueError("Resolved provider configuration exceeds 64 KiB") + identity = self._identity(value) + path = self.directory / f"{identity}.json" + if path.exists(): + self.get(identity) + return identity + with exclusive_file_lock(self.directory / "write.lock"): + if path.exists(): + self.get(identity) + else: + atomic_write_private_json(path, value) + return identity + + def get(self, identity: str) -> dict: + if not re.fullmatch(r"[a-f0-9]{64}", identity): + raise ValueError("Invalid provider revision identity") + try: + with os.fdopen( + open_existing_private_file(self.directory / f"{identity}.json"), + "r", + encoding="utf-8", + ) as stream: + raw = stream.read(65537) + value = json.loads(raw) + if ( + len(raw) > 65536 + or not isinstance(value, dict) + or self._identity(value) != identity + ): + raise ValueError("Invalid provider revision") + return value + except (OSError, ValueError) as exc: + raise ValueError( + "The private provider revision is missing or invalid; resubmit with the current configuration" + ) from exc diff --git a/core/sessions/store.py b/core/sessions/store.py index 6b12a89e5..f500b1aa8 100644 --- a/core/sessions/store.py +++ b/core/sessions/store.py @@ -80,6 +80,12 @@ def __init__( self._disk_signatures: dict[str, tuple[int, int, int, int]] | None = None self._deletions = SessionDeletionJournal(self.root) + def close(self) -> None: + """Release this store's index connection before offline file operations.""" + with self._lock: + if self._index is not None: + self._index.close() + # ------------------------------------------------------------------ # Path helpers # ------------------------------------------------------------------ diff --git a/deepcode.py b/deepcode.py index 4e8816361..b3eae5cf7 100755 --- a/deepcode.py +++ b/deepcode.py @@ -162,6 +162,22 @@ def main(): from cli.provider_cli import run as provider_run raise SystemExit(provider_run(sys.argv[2:])) + elif sys.argv[1] == "service": + from cli.service_cli import run as service_run + + raise SystemExit(service_run(sys.argv[2:])) + elif sys.argv[1] == "desktop": + from cli.desktop_cli import run as desktop_run + + raise SystemExit(desktop_run(sys.argv[2:])) + elif sys.argv[1] == "web": + from cli.web_cli import run as web_run + + raise SystemExit(web_run(sys.argv[2:])) + elif sys.argv[1] == "serve": + from app_server.service import main as serve_main + + raise SystemExit(serve_main(sys.argv[2:])) elif sys.argv[1] in {"session", "sessions"}: from cli.session_cli import run as session_run @@ -197,7 +213,9 @@ def row(cmd, desc): [ "", "Usage:", - row("deepcode", "Interactive coding agent (TUI, default)"), + row("deepcode", "Open the terminal coding agent (TUI)"), + row("deepcode desktop", "Open the native desktop client"), + row("deepcode web", "Open the local browser client"), row( "deepcode init", "Set up ~/.deepcode so deepcode runs anywhere", @@ -205,7 +223,15 @@ def row(cmd, desc): row("deepcode test ", "Test paper reproduction"), row("deepcode test --fast", "Test paper (fast mode)"), row("deepcode mcp serve", "Expose DeepCode as an MCP server"), - row("deepcode mcp list|add|remove", "Manage MCP clients"), + row( + "deepcode service ", + "Manage the background service", + ), + row( + "deepcode serve --foreground", + "Run the service in this terminal", + ), + row("deepcode mcp ", "Connect and manage MCP tools"), row( "deepcode skill ", "List, inspect, import, and manage Agent Skills", @@ -243,29 +269,20 @@ def row(cmd, desc): "Manage durable Agent Automations", ), "", - " More agent entry points:", - row( - 'python -m cli.exec_cli "" -w .', - "Headless one-shot run", - ), - row( - 'python -m cli.loop_cli ""', - "Headless durable Goal", - ), - row( - "python -m cli.loop_cli --resume ", - "Resume a durable Goal without replacing history", - ), + "Examples:", + row("deepcode --trust", "Start in a project you trust"), row( - "python -m cli.schedule_cli ...", - "Scheduled / keepalive runs", + "deepcode --resume ", + "Continue a saved conversation", ), - "", - "Examples:", - row("deepcode", "Drop into the interactive agent"), + row("deepcode provider list", "Show model connections"), + row("deepcode service status", "Check the background service"), row("deepcode test rice", "Test RICE paper reproduction"), row("deepcode test rice --fast", "Test RICE paper (fast mode)"), "", + "Run 'deepcode --help' for command options.", + "Run 'deepcode chat --help' for TUI options.", + "", "Available papers:", ] ) diff --git a/desktop/README.md b/desktop/README.md index a24bdc9b1..772584f01 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -10,6 +10,20 @@ approvals, Git review, files, terminals, tests, Artifacts, and provider settings. Work started in one interface can be resumed in the other without converting or copying its Session. +## Start Desktop + +After installing the CLI, run: + +```console +deepcode desktop +``` + +A local-source CLI installation launches its recorded checkout and prepares +missing development resources. A published CLI opens an installed Desktop app. +Use `deepcode desktop --source /path/to/DeepCode` to choose a checkout explicitly, +or `deepcode desktop --app /path/to/application` for a custom app location. +See the [main installation guide](../README.md#install-the-runtime) for setup. + ## Run from source ### Requirements @@ -61,65 +75,51 @@ rustc --version cargo --version ``` -#### 3. Prepare the repository +#### 3. Install the source checkout -Run these commands from the repository root: +From the repository root, build Web assets and register the global CLI: ```powershell -uv venv --python 3.12 -uv pip install --python .venv\Scripts\python.exe -e . -Set-Location desktop -npm ci -$env:DEEPCODE_PYTHON = (Resolve-Path ..\.venv\Scripts\python.exe) -npm run setup:sidecar -npm run build:sidecar +npm --prefix desktop ci +npm --prefix desktop run build:web +uv tool install --python 3.12 --force . +deepcode init ``` -The explicit interpreter path prevents an active Conda or other environment -from receiving the editable DeepCode installation. +If `deepcode` is not on PATH, run `uv tool update-shell` and reopen PowerShell. #### 4. Start DeepCode Desktop -From the same `desktop` directory and PowerShell window: - ```powershell -npm run tauri -- dev +deepcode desktop ``` -Keep the terminal open while Desktop is running. Press `Ctrl+C` in that terminal -to stop the development application. On later launches, set `DEEPCODE_PYTHON` -again if using a new PowerShell window: - -```powershell -Set-Location desktop -$env:DEEPCODE_PYTHON = (Resolve-Path ..\.venv\Scripts\python.exe) -npm run tauri -- dev -``` +The first launch prepares missing Desktop resources. Later launches reuse them. +Keep this development terminal open while Desktop is running. Closing the +window or pressing Ctrl+C closes the client; the shared service remains running. ### macOS and Linux -After installing the platform dependencies from the Tauri prerequisite guide, -run the following commands from the repository root: +Install the platform dependencies from the Tauri prerequisite guide. From the +repository root, install the current checkout once: ```bash -uv venv --python 3.12 -uv pip install --python .venv/bin/python -e . -cd desktop -npm ci -npm run setup:sidecar -npm run build:sidecar -cd .. -./scripts/deepcode-desktop +npm --prefix desktop ci +npm --prefix desktop run build:web +uv tool install --python 3.12 --force . +deepcode init ``` -To use `deepcode-desktop` from any working directory, link the launcher into a -directory on `PATH` once: +Then start the application from any directory: ```bash -mkdir -p ~/.local/bin -ln -sf "$(pwd)/scripts/deepcode-desktop" ~/.local/bin/deepcode-desktop +deepcode desktop ``` +Use `deepcode desktop --setup` to rebuild source dependencies and Desktop +resources. To select a different checkout explicitly, use +`deepcode desktop --source /path/to/DeepCode`. + ### Development sidecar The first `build:sidecar` creates the resource directory declared in @@ -153,6 +153,10 @@ runtime dependencies. 6. Open a Session to inherit the default, or use its composer picker to switch the connection/model for future Turns. +The model check above is minimal inference. To also check streaming and a local +tool round trip, run `deepcode provider test CONNECTION_ID --model MODEL_ID --agent` +with the configured IDs. See [Models and providers](../docs/guide/models.md). + API keys are written to `~/.deepcode/credentials.json` in user-private storage. Desktop receives only configured/missing status and never reads a stored key back. The same named connections and defaults are used by CLI and Desktop. @@ -257,12 +261,13 @@ the next explicit page with stable ID deduplication. Live notifications refresh the first page; an overflow warning also resets the visible pages safely rather than presenting a partial cache as complete. -Interval schedules execute while a scheduler-enabled Desktop or App Server is -active. If several compatible processes share the database, one holds the -scheduler leader lease and the others remain available for takeover. Agent and -Workflow Turns still use the shared cross-process capacity and workspace -fences, so another Session cannot mutate the same canonical checkout at the -same time. +The shared background service owns interval scheduling and its scheduler +leadership lease. Schedules continue after Desktop, TUI, and Web close, provided +the service and computer remain running. Agent and Workflow Turns still obey +execution capacity and workspace fences, preventing concurrent mutations of the +same canonical checkout. Inspect the service with `deepcode service status`; +see [Automations](../docs/guide/goals-and-headless.md#persistent-interval-automations) +for manual runs and pausing future submissions. Automation instructions never grant trust or elevated permissions. Each Turn captures the workspace's explicit permission setting or the safe default, and diff --git a/desktop/e2e/web.spec.ts b/desktop/e2e/web.spec.ts new file mode 100644 index 000000000..b521cba54 --- /dev/null +++ b/desktop/e2e/web.spec.ts @@ -0,0 +1,400 @@ +import { test, expect } from "@playwright/test"; +import { + execFileSync, + spawn, + spawnSync, + type ChildProcess, +} from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repository = fileURLToPath(new URL("../../", import.meta.url)); +const python = + process.env.DEEPCODE_TEST_PYTHON ?? + join( + repository, + ".venv", + process.platform === "win32" ? "Scripts/python.exe" : "bin/python", + ); +const live = process.env.DEEPCODE_WEB_LIVE === "1"; +const installed = process.env.DEEPCODE_WEB_PACKAGE_DIR; +const standalone = process.env.DEEPCODE_WEB_BINARY; +let root: string, + workspace: string, + environment: NodeJS.ProcessEnv, + processHandle: ChildProcess; +const cwd = () => (installed ? root : repository); +const command = (...args: string[]) => + execFileSync(python, args, { + cwd: cwd(), + env: environment, + encoding: "utf8", + timeout: 40000, + }); +function link() { + if (standalone) + return JSON.parse( + execFileSync( + standalone, + [ + "--web", + "--database", + join(root, "state.sqlite3"), + "--port", + "0", + "--no-open", + "--json", + ], + { cwd: root, env: environment, encoding: "utf8", timeout: 40000 }, + ), + ).url as string; + return JSON.parse( + command( + "-m", + "deepcode", + "web", + "--database", + join(root, "state.sqlite3"), + "--no-open", + "--json", + ), + ).url as string; +} + +test.beforeAll(async () => { + root = mkdtempSync(join(tmpdir(), "deepcode-web-e2e-")); + workspace = join(root, "workspace"); + mkdirSync(workspace); + mkdirSync(join(root, "home")); + writeFileSync(join(workspace, "example.py"), "answer = 41\n"); + for (const args of [ + ["init", "-q"], + ["config", "user.name", "Web test"], + ["config", "user.email", "web-test@example.invalid"], + ["add", "."], + ["commit", "-qm", "fixture"], + ]) + execFileSync("git", args, { cwd: workspace }); + writeFileSync(join(workspace, "example.py"), "answer = 42\n"); + if (live) + execFileSync( + python, + [ + "-m", + "tests.app_server.web_worker", + join(root, "home", "deepcode_config.json"), + "--prepare-live-config", + ], + { cwd: repository }, + ); + environment = { + ...process.env, + DEEPCODE_HOME: join(root, "home"), + DEEPCODE_SESSIONS_DIR: join(root, "home", "sessions"), + ...(installed ? { PYTHONPATH: installed } : {}), + }; + for (const key of Object.keys(environment)) + if (/_API_KEY$/.test(key)) delete environment[key]; + if (standalone) link(); + else + processHandle = spawn( + python, + installed + ? [ + "-m", + "app_server.service", + "--database", + join(root, "state.sqlite3"), + "--port", + "0", + ] + : [ + "-m", + "tests.app_server.web_worker", + root, + ...(live ? ["--live"] : []), + ], + { cwd: cwd(), env: environment, stdio: "ignore" }, + ); + await expect + .poll( + () => existsSync(join(root, "state.sqlite3.service", "instance.json")), + { timeout: 15000 }, + ) + .toBe(true); +}); + +test.afterAll(() => { + try { + command( + "-m", + "deepcode", + "service", + "stop", + "--database", + join(root, "state.sqlite3"), + "--cancel-running", + "--timeout", + "2", + "--json", + ); + } finally { + if (processHandle && processHandle.exitCode === null) + processHandle.kill("SIGTERM"); + if (root) rmSync(root, { recursive: true, force: true }); + } +}); + +test("packaged browser: project, approval, reconnect, upload, diff and terminal", async ({ + page, + context, +}, testInfo) => { + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + page.on("dialog", (dialog) => dialog.accept()); + await page.goto(link()); + await expect( + page + .getByRole("button", { name: "Open project folder", exact: true }) + .first(), + ).toBeEnabled(); + expect(page.url()).not.toContain("ticket="); + await page + .getByRole("button", { name: "Open project folder", exact: true }) + .first() + .click(); + await page.getByLabel("Project folder", { exact: true }).fill(workspace); + await page.getByRole("button", { name: "Open project", exact: true }).click(); + await page.getByRole("button", { name: "Trust folder", exact: true }).click(); + await expect( + page.getByText("Trusted", { exact: true }).first(), + ).toBeVisible(); + // Source-controlled labels make this independent of generated CSS names. + await page + .getByRole("button", { name: /New (thread|session|task)/i }) + .first() + .click(); + const composer = page + .getByRole("textbox", { name: /message|prompt|instruction/i }) + .first(); + await expect(composer).toBeEnabled(); + + const attachment = join(root, "context.txt"); + writeFileSync( + attachment, + "Browser upload acceptance: preserve the public API.\n", + ); + const chooser = page.waitForEvent("filechooser"); + await page.getByRole("button", { name: "Attach workspace files" }).click(); + await (await chooser).setFiles(attachment); + await expect(page.getByLabel("Attached context files")).toContainText( + "context.txt", + ); + + if (!installed && !standalone) { + await composer.fill( + live + ? "Work only in this workspace. Create arithmetic.py with triangular(n), sum 1 through n, rejecting negative n. Create test_arithmetic.py with standard-library unittest for 0, 1, 10, 100 and negative input. Use write for these two files, then run exactly python3 -m unittest -v. No dependencies, network, git changes or subagents. Finish after tests pass." + : "Verify the browser approval flow and finish.", + ); + await composer.press("Enter"); + if (live) { + await expect(page.getByLabel("Thread conversation")).toContainText( + "Create arithmetic.py", + ); + await page.reload(); + const deadline = Date.now() + 240000; + while (Date.now() < deadline) { + const approve = page.getByRole("button", { + name: "Allow once", + exact: true, + }); + if (await approve.count()) { + const card = approve.first().locator("..").locator(".."); + await card.getByText("Review arguments", { exact: true }).click(); + const args = JSON.parse(await card.locator("pre").innerText()); + const tool = await card.locator("strong").first().innerText(); + if (tool === "write") { + expect([ + join(workspace, "arithmetic.py"), + join(workspace, "test_arithmetic.py"), + ]).toContain(resolve(workspace, args.file_path)); + } else { + expect(tool).toBe("bash"); + expect(["python3 -m unittest -v", "pwd", "ls"]).toContain( + args.command.trim(), + ); + } + await approve.first().click(); + } + if ( + existsSync(join(workspace, "test_arithmetic.py")) && + (await page + .getByRole("button", { name: /Stop (turn|task|generation)/i }) + .count()) === 0 && + (await approve.count()) === 0 + ) + break; + await page.waitForTimeout(500); + } + expect(existsSync(join(workspace, "arithmetic.py"))).toBe(true); + await expect( + page.getByText(/^Worked for|^Work completed/).first(), + ).toBeVisible(); + const result = spawnSync(python, ["-m", "unittest", "-v"], { + cwd: workspace, + encoding: "utf8", + }); + expect(result.status).toBe(0); + await testInfo.attach("generated-code", { + body: readFileSync(join(workspace, "arithmetic.py")), + contentType: "text/plain", + }); + await testInfo.attach("independent-tests", { + body: result.stdout + result.stderr, + contentType: "text/plain", + }); + } else { + await expect( + page.getByRole("button", { name: "Allow once", exact: true }), + ).toBeVisible(); + await page.reload(); + await expect( + page.getByRole("button", { name: "Allow once", exact: true }), + ).toBeVisible(); + await context.setOffline(true); + await context.setOffline(false); + await page + .getByRole("button", { name: "Allow once", exact: true }) + .click(); + await expect( + page.getByText("done", { exact: true }).first(), + ).toBeVisible(); + } + const admissions = JSON.parse( + command( + "-c", + "import sqlite3,json,sys; c=sqlite3.connect(sys.argv[1]); print(json.dumps(c.execute(\"select count(*) from event_log where type in ('turn.started','turn.queued')\").fetchone()[0]))", + join(root, "state.sqlite3"), + ), + ); + expect(admissions).toBe(1); + await testInfo.attach("task-admissions", { + body: JSON.stringify({ admissions }), + contentType: "application/json", + }); + } + + await page.reload(); + await expect( + page.getByText("Trusted", { exact: true }).first(), + ).toBeVisible(); + await page.getByRole("button", { name: "Settings", exact: true }).click(); + await expect( + page.getByRole("heading", { name: "Service updates" }), + ).toBeVisible(); + await expect( + page.getByRole("button", { name: "Copy configuration path" }), + ).toBeVisible(); + await page + .getByRole("combobox", { name: "Default Session access", exact: true }) + .selectOption("read_only"); + await page + .getByRole("button", { name: "Save safety settings", exact: true }) + .click(); + await expect(page.getByText(/Effective default: Read only/)).toBeVisible(); + await page + .getByRole("combobox", { name: "Default Session access", exact: true }) + .selectOption("ask"); + await page + .getByRole("button", { name: "Save safety settings", exact: true }) + .click(); + await expect(page.getByText(/Effective default: Ask/)).toBeVisible(); + await page + .getByRole("button", { name: "Close settings", exact: true }) + .click(); + await page + .getByRole("button", { name: /Review/ }) + .first() + .click(); + await expect( + page.getByText("example.py", { exact: true }).first(), + ).toBeVisible(); + await page + .getByRole("button", { name: "Open in editor", exact: true }) + .first() + .click(); + const downloaded = page.waitForEvent("download"); + await page.getByRole("button", { name: "Download", exact: true }).click(); + const file = await downloaded; + expect(readFileSync((await file.path())!, "utf8")).toBe("answer = 42\n"); + await page.getByRole("tab", { name: "terminal", exact: true }).click(); + await page + .getByRole("button", { name: "Start terminal", exact: true }) + .click(); + await expect(page.getByText(/^PID \d+$/)).toBeVisible(); + const pid = await page.getByText(/^PID \d+$/).innerText(); + const terminal = page.locator(".xterm-helper-textarea"); + await terminal.pressSequentially("printf 'once\\n' >> browser-pty.txt"); + await terminal.press("Enter"); + await expect + .poll(() => existsSync(join(workspace, "browser-pty.txt"))) + .toBe(true); + expect(readFileSync(join(workspace, "browser-pty.txt"), "utf8")).toBe( + "once\n", + ); + await page.reload(); + await page.getByRole("button", { name: "Review", exact: true }).click(); + await page.getByRole("tab", { name: "terminal", exact: true }).click(); + await expect(page.getByText(pid, { exact: true })).toBeVisible(); + expect(readFileSync(join(workspace, "browser-pty.txt"), "utf8")).toBe( + "once\n", + ); + await page.screenshot({ + path: testInfo.outputPath("web-ready.png"), + fullPage: true, + }); + expect(errors).toEqual([]); + await page.getByRole("button", { name: "Sign out", exact: true }).click(); + await expect(page.getByText(/Signed out/).first()).toBeVisible(); +}); + + +test("browser access: missing and used links, sign-out, and fresh-link recovery", async ({ page, context }) => { + const accessLink = link(); + const base = new URL(accessLink).origin; + await page.goto(base); + const notice = page.getByRole("alert"); + await expect(notice).toContainText("Browser access required"); + await expect(notice).toContainText("deepcode web"); + await expect(notice).not.toContainText("APP_SERVER_OFFLINE"); + await expect(notice.getByRole("button", { name: "Reconnect" })).toHaveCount(0); + await expect(notice.locator("span")).toHaveCSS("white-space", "normal"); + + await page.goto(accessLink); + await expect(page.getByRole("button", { name: "Open project folder", exact: true }).first()).toBeEnabled(); + await expect(notice).toHaveCount(0); + + await context.clearCookies(); + await page.goto(accessLink); + await expect(notice).toContainText("Browser access required"); + await page.goto(link()); + await expect(page.getByRole("button", { name: "Open project folder", exact: true }).first()).toBeEnabled(); + await expect(notice).toHaveCount(0); + + await page.getByRole("button", { name: "Sign out", exact: true }).click(); + await expect(notice).toContainText("Browser access required"); + await expect(notice.getByRole("button", { name: "Reconnect" })).toHaveCount(0); + await page.goto(link()); + await expect(page.getByRole("button", { name: "Open project folder", exact: true }).first()).toBeEnabled(); + await expect(notice).toHaveCount(0); +}); diff --git a/desktop/package-lock.json b/desktop/package-lock.json index e9403f985..723cbe9e7 100644 --- a/desktop/package-lock.json +++ b/desktop/package-lock.json @@ -27,6 +27,7 @@ }, "devDependencies": { "@eslint/js": "^10.0.1", + "@playwright/test": "^1.58.2", "@tauri-apps/cli": "^2.11.4", "@testing-library/react": "^16.3.2", "@types/node": "^24.10.1", @@ -42,7 +43,7 @@ "typescript": "^6.0.3", "typescript-eslint": "^8.64.0", "vite": "^8.1.4", - "vitest": "^4.1.10" + "vitest": "^4.1.11" } }, "node_modules/@apidevtools/json-schema-ref-parser": { @@ -871,6 +872,22 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@playwright/test": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.2.tgz", + "integrity": "sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.58.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", @@ -1874,16 +1891,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", - "integrity": "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.10", - "@vitest/utils": "4.1.10", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -1892,13 +1909,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", - "integrity": "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.10", + "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -1919,9 +1936,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.10.tgz", - "integrity": "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", "dev": true, "license": "MIT", "dependencies": { @@ -1932,13 +1949,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.10.tgz", - "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.10", + "@vitest/utils": "4.1.11", "pathe": "^2.0.3" }, "funding": { @@ -1946,14 +1963,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.10.tgz", - "integrity": "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.10", - "@vitest/utils": "4.1.10", + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -1962,9 +1979,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.10.tgz", - "integrity": "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", "dev": true, "license": "MIT", "funding": { @@ -1972,13 +1989,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.10.tgz", - "integrity": "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.10", + "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -3080,9 +3097,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { @@ -4670,6 +4687,53 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", + "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.58.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", + "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.25", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", @@ -5140,9 +5204,9 @@ } }, "node_modules/tinyrainbow": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", - "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", "dev": true, "license": "MIT", "engines": { @@ -5548,19 +5612,19 @@ } }, "node_modules/vitest": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.10.tgz", - "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.10", - "@vitest/mocker": "4.1.10", - "@vitest/pretty-format": "4.1.10", - "@vitest/runner": "4.1.10", - "@vitest/snapshot": "4.1.10", - "@vitest/spy": "4.1.10", - "@vitest/utils": "4.1.10", + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -5588,12 +5652,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.10", - "@vitest/browser-preview": "4.1.10", - "@vitest/browser-webdriverio": "4.1.10", - "@vitest/coverage-istanbul": "4.1.10", - "@vitest/coverage-v8": "4.1.10", - "@vitest/ui": "4.1.10", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/desktop/package.json b/desktop/package.json index fdc8d8a09..8901516db 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -6,8 +6,10 @@ "scripts": { "dev": "vite", "build": "npm run check:version && npm run check:tauri && npm run check:protocol && tsc --noEmit && vite build", + "build:web": "npm run check:protocol && tsc --noEmit && vite build --mode web", + "test:web": "npm run build:web && playwright test", "setup:sidecar": "node scripts/run-python.mjs scripts/setup-sidecar-env.py", - "build:sidecar": "node scripts/run-python.mjs scripts/build-sidecar.py", + "build:sidecar": "npm run build:web && node scripts/run-python.mjs scripts/build-sidecar.py", "audit:licenses": "node scripts/run-python.mjs scripts/audit-licenses.py --output build/licenses/dependencies.json", "check:version": "node scripts/check-version.mjs", "check:tauri": "node scripts/check-tauri-config.mjs", @@ -40,6 +42,7 @@ }, "devDependencies": { "@eslint/js": "^10.0.1", + "@playwright/test": "^1.58.2", "@tauri-apps/cli": "^2.11.4", "@testing-library/react": "^16.3.2", "@types/node": "^24.10.1", @@ -55,7 +58,7 @@ "typescript": "^6.0.3", "typescript-eslint": "^8.64.0", "vite": "^8.1.4", - "vitest": "^4.1.10" + "vitest": "^4.1.11" }, "overrides": { "nanoid": "3.3.18" diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts new file mode 100644 index 000000000..085f38ace --- /dev/null +++ b/desktop/playwright.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: "./e2e", + workers: 1, + reporter: [["list"], ["json", { outputFile: "test-results/results.json" }]], + timeout: process.env.DEEPCODE_WEB_LIVE === "1" ? 360000 : 90000, + use: { + actionTimeout: 10000, + headless: true, + viewport: { width: 1440, height: 1000 }, + screenshot: "only-on-failure", + trace: "retain-on-failure", + launchOptions: process.env.DEEPCODE_CHROME_PATH + ? { executablePath: process.env.DEEPCODE_CHROME_PATH } + : {}, + }, +}); diff --git a/desktop/scripts/build-sidecar.py b/desktop/scripts/build-sidecar.py index e08d10e8c..2eb35c0dd 100644 --- a/desktop/scripts/build-sidecar.py +++ b/desktop/scripts/build-sidecar.py @@ -15,7 +15,6 @@ import sys from pathlib import Path - DESKTOP_ROOT = Path(__file__).resolve().parents[1] REPOSITORY_ROOT = DESKTOP_ROOT.parent BUILD_ROOT = DESKTOP_ROOT / "build" / "sidecar" @@ -23,6 +22,7 @@ APP_SERVER_ROOT = DIST_ROOT / "deepcode-app-server" SIDECAR_ENV_ROOT = BUILD_ROOT / ".venv" BUNDLED_DATA = ( + (REPOSITORY_ROOT / "app_server" / "web_assets", "app_server/web_assets"), ( REPOSITORY_ROOT / "core" / "application" / "goal_prompts", "core/application/goal_prompts", @@ -201,6 +201,7 @@ def _verify_bundle(binary: Path) -> None: result.get("ok") is not True or result.get("skillCreator") is not True or not result.get("bundledMcpPresets") + or result.get("webAssets") is not True ): raise RuntimeError("packaged runtime import probe did not report success") @@ -210,14 +211,7 @@ def _verify_bundle(binary: Path) -> None: database = smoke_root / "state.sqlite3" environment = dict(os.environ) environment["DEEPCODE_HOME"] = str(home) - process = subprocess.Popen( - [str(binary), "--database", str(database)], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=environment, - ) + environment["DEEPCODE_SESSIONS_DIR"] = str(home / "sessions") initialize = { "jsonrpc": "2.0", "id": 1, @@ -228,30 +222,99 @@ def _verify_bundle(binary: Path) -> None: }, } shutdown = {"jsonrpc": "2.0", "id": 2, "method": "shutdown", "params": {}} + process = None try: + subprocess.run( + [ + str(binary), + "--service", + "start", + "--database", + str(database), + "--port", + "0", + "--json", + ], + env=environment, + check=True, + capture_output=True, + text=True, + timeout=45, + ) + process = subprocess.Popen( + [str(binary), "--database", str(database)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=environment, + ) stdout, stderr = process.communicate( f"{json.dumps(initialize)}\n{json.dumps(shutdown)}\n", timeout=30, ) + if process.returncode != 0: + raise RuntimeError( + f"packaged App Server smoke failed ({process.returncode}): " + f"{stderr[-2_000:]}" + ) + responses = [json.loads(line) for line in stdout.splitlines() if line.strip()] + if ( + len(responses) != 2 + or responses[0].get("result", {}).get("protocolVersion") != "1.0" + ): + raise RuntimeError( + "packaged App Server smoke returned invalid RPC responses" + ) + except subprocess.CalledProcessError as exc: + service_log = database.with_name(database.name + ".service") / "service.log" + detail = ( + service_log.read_text(encoding="utf-8", errors="replace") + if service_log.exists() + else "No service log was created." + ) + raise RuntimeError( + f"packaged service startup failed ({exc.returncode}):\n" + f"{exc.stdout[-2_000:]}\n{exc.stderr[-2_000:]}\n{detail[-4_000:]}" + ) from exc except subprocess.TimeoutExpired: - process.kill() - stdout, stderr = process.communicate() + if process is not None: + process.kill() + stdout, stderr = process.communicate() + else: + stderr = "Timed out starting the packaged service" raise RuntimeError( f"packaged App Server smoke timed out: {stderr[-2_000:]}" ) from None finally: - shutil.rmtree(smoke_root, ignore_errors=True) - if process.returncode != 0: - raise RuntimeError( - f"packaged App Server smoke failed ({process.returncode}): " - f"{stderr[-2_000:]}" - ) - responses = [json.loads(line) for line in stdout.splitlines() if line.strip()] - if ( - len(responses) != 2 - or responses[0].get("result", {}).get("protocolVersion") != "1.0" - ): - raise RuntimeError("packaged App Server smoke returned invalid RPC responses") + failure = sys.exception() + try: + subprocess.run( + [ + str(binary), + "--service", + "stop", + "--database", + str(database), + "--cancel-running", + "--timeout", + "3", + "--json", + ], + env=environment, + check=True, + capture_output=True, + text=True, + timeout=35, + ) + except (OSError, subprocess.SubprocessError) as exc: + detail = f"packaged service cleanup failed: {exc}" + if failure is None: + raise RuntimeError(detail) from exc + # Keep the original startup/protocol failure as the primary error. + failure.add_note(detail) + finally: + shutil.rmtree(smoke_root, ignore_errors=True) if __name__ == "__main__": diff --git a/desktop/src-tauri/binaries/.gitkeep b/desktop/src-tauri/binaries/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/desktop/src-tauri/build.rs b/desktop/src-tauri/build.rs index ebeb1cbf3..d860e1e6a 100644 --- a/desktop/src-tauri/build.rs +++ b/desktop/src-tauri/build.rs @@ -1,6 +1,3 @@ fn main() { - if let Ok(target) = std::env::var("TARGET") { - println!("cargo:rustc-env=DEEPCODE_TARGET_TRIPLE={target}"); - } tauri_build::build() } diff --git a/desktop/src-tauri/src/sidecar.rs b/desktop/src-tauri/src/sidecar.rs index c39f73f7c..48ebf24ce 100644 --- a/desktop/src-tauri/src/sidecar.rs +++ b/desktop/src-tauri/src/sidecar.rs @@ -225,7 +225,12 @@ impl RpcBridge { } pub fn request(&self, method: &str, params: Value) -> PendingResult { - self.request_with_timeout(method, params, REQUEST_TIMEOUT) + let timeout = if method == "provider/test" { + Duration::from_secs(125) + } else { + REQUEST_TIMEOUT + }; + self.request_with_timeout(method, params, timeout) } pub fn restart(self: &Arc) -> Result { @@ -241,11 +246,9 @@ impl RpcBridge { self.update_phase(SidecarPhase::Stopping, None); let _ = self.request_with_timeout("shutdown", json!({}), SHUTDOWN_REQUEST_TIMEOUT); - // The App Server acknowledges shutdown before its application-level - // cleanup finishes. Give that cleanup one independent grace period: - // coordinator quiescing, scheduler release, live Turn cancellation, - // terminal process groups, and the lifetime lease all complete before - // the Python process exits. + // The child is a native RPC attachment. Shutdown closes its connection; + // allow pipe and transport cleanup to finish before reaping the relay. + // The shared service retains the application and accepted tasks. let deadline = Instant::now() + SHUTDOWN_EXIT_GRACE; loop { let exited = { @@ -510,7 +513,7 @@ impl RpcBridge { phase: SidecarPhase::Crashed, message: Some(message), launch_source: self.status().launch_source, - server_info: None, + server_info: self.status().server_info, }); } @@ -630,22 +633,6 @@ fn resolve_launch_spec(app: &AppHandle) -> Result { if source_bundle.is_file() { return Ok(binary_launch(source_bundle, "source App Server bundle")); } - - let target = env!("DEEPCODE_TARGET_TRIPLE"); - let source_name = if cfg!(windows) { - format!("deepcode-app-server-{target}.exe") - } else { - format!("deepcode-app-server-{target}") - }; - let source_binary = repository - .join("desktop/src-tauri/binaries") - .join(source_name); - if source_binary.is_file() { - return Ok(binary_launch( - source_binary, - "legacy source external binary", - )); - } } #[cfg(debug_assertions)] diff --git a/desktop/src/App.module.css b/desktop/src/App.module.css index aad9ca3f4..c137ea143 100644 --- a/desktop/src/App.module.css +++ b/desktop/src/App.module.css @@ -15,7 +15,7 @@ position: relative; display: grid; min-width: 0; - height: calc(100vh - 16px); + height: calc(100% - 16px); grid-template-rows: auto minmax(0, 1fr) auto; margin: 8px 8px 8px 4px; overflow: hidden; @@ -123,7 +123,7 @@ .reviewPane { position: relative; min-width: 0; - height: calc(100vh - 16px); + height: calc(100% - 16px); margin: 8px 8px 8px 0; overflow: hidden; border: 1px solid var(--border-subtle); @@ -143,7 +143,7 @@ top: 8px; right: 8px; width: min(430px, calc(100vw - 72px)); - height: calc(100vh - 16px); + height: calc(100% - 16px); margin: 0; box-shadow: var(--shadow-float); } diff --git a/desktop/src/App.test.tsx b/desktop/src/App.test.tsx index 58575aa0f..02bf73ca1 100644 --- a/desktop/src/App.test.tsx +++ b/desktop/src/App.test.tsx @@ -1,7 +1,9 @@ import { cleanup, + act, fireEvent, render, + renderHook, screen, waitFor, within, @@ -28,12 +30,13 @@ import type { WorkflowRun, } from "./generated/app-server"; import { App } from "./App"; +import { useWorkspaceController } from "./app/useWorkspaceController"; import { __resetComposerBehaviorForTests } from "./app/composerBehavior"; import { __resetEscapeLayersForTests } from "./app/escapeLayer"; import { __setLocaleForTests } from "./app/i18n"; import type { AnyRpcNotification, - DesktopRuntime, + ClientRuntime, DesktopUpdateInfo, DesktopUpdateProgress, RpcMethod, @@ -227,7 +230,9 @@ const diagnostics: DiagnosticsSnapshot = { ], }; -class TestRuntime implements DesktopRuntime { +class TestRuntime implements ClientRuntime { + readonly notifications = new Set<(notification: AnyRpcNotification) => void>(); + readonly statuses = new Set<(status: SidecarStatus) => void>(); readonly calls: string[] = []; readonly requests: Array<{ method: string; params: unknown }> = []; readonly diagnosticsExports: DiagnosticsSnapshot[] = []; @@ -1004,12 +1009,19 @@ class TestRuntime implements DesktopRuntime { }, } as unknown as MethodResults[M]; } - case "event/replay": + case "event/replay": { + const { threadId, after = 0, through, limit = 500 } = params as MethodParams["event/replay"]; + const history = this.events.filter((event) => event.threadId === threadId); + const headSequence = Math.min(through ?? Infinity, history.at(-1)?.sequence ?? 0); + const remaining = history.filter((event) => event.sequence > after && event.sequence <= headSequence); + const events = remaining.slice(0, limit); return { - events: this.events, - nextAfter: null, - hasMore: false, + events, + nextAfter: remaining.length > limit ? events.at(-1)!.sequence : null, + hasMore: remaining.length > limit, + headSequence, } as MethodResults[M]; + } case "file/list": return { entries: [], truncated: false } as unknown as MethodResults[M]; case "git/status": @@ -1076,13 +1088,13 @@ class TestRuntime implements DesktopRuntime { } async onNotification(listener: (notification: AnyRpcNotification) => void) { - void listener; - return () => undefined; + this.notifications.add(listener); + return () => { this.notifications.delete(listener); }; } async onStatus(listener: (status: SidecarStatus) => void) { - void listener; - return () => undefined; + this.statuses.add(listener); + return () => { this.statuses.delete(listener); }; } async onLog(listener: (message: string) => void) { @@ -1275,6 +1287,160 @@ const recoveryEvents: Event[] = [ }, ]; +function liveDelta(sequence: number, delta: string): Event { + return { + ...recoveryEvents[1], + eventId: `event-${sequence}`, + sequence, + type: "item.delta", + payload: { delta }, + }; +} + +describe("workspace event recovery", () => { + it("repairs skipped deltas and a dropped approval without resetting the selected item", async () => { + const events = [...recoveryEvents]; + const runtime = new TestRuntime([project], [thread], events); + const { result } = renderHook(() => useWorkspaceController(runtime)); + await waitFor(() => expect(result.current.state.items).toHaveLength(1)); + act(() => result.current.selectItem("item-1")); + events.push(liveDelta(3, " A"), liveDelta(4, "B")); + act(() => + runtime.notifications.forEach((receive) => + receive({ jsonrpc: "2.0", method: "item.delta", params: events[3] }), + ), + ); + await waitFor(() => + expect(result.current.state.items[0].payload.text).toBe( + "Recovered final answer AB", + ), + ); + expect(result.current.state.selectedItemId).toBe("item-1"); + const approval = { + ...recoveryEvents[0], + eventId: "event-5", + sequence: 5, + type: "approval.requested", + payload: { approval: pendingApproval as unknown as JsonValue }, + }; + events.push(approval); + act(() => + runtime.notifications.forEach((receive) => + receive({ + jsonrpc: "2.0", + method: "server.warning", + params: { + code: "EVENT_QUEUE_OVERFLOW", + dropped: 1, + replayRequired: true, + }, + }), + ), + ); + await waitFor(() => expect(result.current.state.approvals).toHaveLength(1)); + expect(result.current.state.selectedItemId).toBe("item-1"); + const replayRequests = runtime.requests.filter( + (request) => request.method === "event/replay", + ); + expect( + replayRequests.map( + (request) => (request.params as MethodParams["event/replay"]).after, + ), + ).toEqual([0, 2, 4]); + }); + + it("holds a newer live delta until its replayed base exists", async () => { + const events = [...recoveryEvents]; + const runtime = new TestRuntime([project], [thread], events); + const original = runtime.request.bind(runtime); + let resolve!: (value: MethodResults["event/replay"]) => void; + const first = new Promise((yes) => { + resolve = yes; + }); + let paused = false; + vi.spyOn(runtime, "request").mockImplementation(async (method, params) => { + if (method === "event/replay" && !paused) { + paused = true; + return first as Promise; + } + return original(method, params); + }); + const { result } = renderHook(() => useWorkspaceController(runtime)); + await waitFor(() => expect(paused).toBe(true)); + const delta = liveDelta(3, " appended once"); + events.push(delta); + act(() => + runtime.notifications.forEach((receive) => + receive({ jsonrpc: "2.0", method: "item.delta", params: delta }), + ), + ); + expect(result.current.state.items).toHaveLength(0); + await act(async () => + resolve({ + events: recoveryEvents, + nextAfter: null, + hasMore: false, + headSequence: 2, + }), + ); + await waitFor(() => + expect(result.current.state.items[0]?.payload.text).toBe( + "Recovered final answer appended once", + ), + ); + act(() => + runtime.notifications.forEach((receive) => + receive({ jsonrpc: "2.0", method: "item.delta", params: delta }), + ), + ); + expect(result.current.state.items[0].payload.text).toBe( + "Recovered final answer appended once", + ); + }); + + it.each(["stopped", "starting"] as const)( + "replays missed events when a %s runtime becomes ready again", + async (phase) => { + const events = [...recoveryEvents]; + const runtime = new TestRuntime([project], [thread], events); + const { result } = renderHook(() => useWorkspaceController(runtime)); + await waitFor(() => expect(runtime.calls).toContain("settings/read")); + act(() => + runtime.statuses.forEach((receive) => + receive({ ...readyStatus, phase }), + ), + ); + events.push(liveDelta(3, " after reconnect")); + act(() => runtime.statuses.forEach((receive) => receive(readyStatus))); + await waitFor(() => + expect(result.current.state.items[0]?.payload.text).toBe( + "Recovered final answer after reconnect", + ), + ); + expect( + runtime.calls.filter((method) => method === "project/list"), + ).toHaveLength(2); + }, + ); + + it("cleans up a notification subscription that resolves after unmount", async () => { + const runtime = new TestRuntime(); + let resolve!: (cleanup: () => void) => void; + vi.spyOn(runtime, "onNotification").mockImplementation( + () => + new Promise((yes) => { + resolve = yes; + }), + ); + const cleanup = vi.fn(); + const { unmount } = renderHook(() => useWorkspaceController(runtime)); + unmount(); + await act(async () => resolve(cleanup)); + expect(cleanup).toHaveBeenCalledOnce(); + expect(runtime.calls).toEqual([]); + }); +}); + const presentationItems: Item[] = [ { id: "item-presentation-user", @@ -2429,7 +2595,7 @@ describe("desktop command center", () => { const discoverParams = runtime.requests.find( (request) => request.method === "provider/discover", )?.params as MethodParams["provider/discover"]; - expect(discoverParams.connectionId).toBe("openai"); + expect(discoverParams.connection).toMatchObject({ id: "openai", protocol: "auto", auth: "api_key" }); fireEvent.click(screen.getByRole("button", { name: "Save and check" })); await waitFor(() => { @@ -2479,7 +2645,7 @@ describe("desktop command center", () => { model: "gpt-5", }); }); - expect(await screen.findByText("Ready for agent work")).toBeTruthy(); + expect(await screen.findByText("Model request verified")).toBeTruthy(); const methods = runtime.requests.map((request) => request.method); expect(methods.indexOf("settings/update")).toBeLessThan( methods.indexOf("provider/test"), diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index 700f4d251..637f4acc6 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -15,8 +15,7 @@ import { import { DesktopSidebar } from "./features/navigation/DesktopSidebar"; import { ThreadHeader } from "./features/thread/ThreadHeader"; import { useTranscriptMode } from "./features/thread/transcriptMode"; -import type { DesktopRuntime } from "./rpc/contracts"; -import { tauriRuntime } from "./rpc/tauriRuntime"; +import type { ClientRuntime } from "./rpc/contracts"; import { initI18n } from "./app/i18n"; import styles from "./App.module.css"; @@ -67,7 +66,7 @@ function LoadingSurface({ ); } -export function App({ runtime = tauriRuntime }: { runtime?: DesktopRuntime }) { +export function App({ runtime }: { runtime: ClientRuntime }) { const controller = useWorkspaceController(runtime); // Mounted for its effect: paints saved appearance preferences at startup. useAppearance(); @@ -190,6 +189,7 @@ export function App({ runtime = tauriRuntime }: { runtime?: DesktopRuntime }) {
(null); + const [path, setPath] = useState(""); + const [notice, setNotice] = useState(null); + const [runtime] = useState( + () => + new BrowserRuntime({ + buildId: __WEB_BUILD_ID__, + chooseDirectory: () => + new Promise((resolve) => { + setPath(""); + setPicker({ resolve }); + }), + }), + ); + useEffect(() => { + const dispose = () => runtime.dispose(); + // Opening a fresh access link in this tab only changes its fragment. + // Reload so the new document exchanges the ticket before connecting. + const openAccessLink = () => { + if (new URLSearchParams(location.hash.slice(1)).has("ticket")) + location.reload(); + }; + window.addEventListener("pagehide", dispose); + window.addEventListener("hashchange", openAccessLink); + return () => { + window.removeEventListener("pagehide", dispose); + window.removeEventListener("hashchange", openAccessLink); + dispose(); + }; + }, [runtime]); + const closePicker = (value: string | null) => { + picker?.resolve(value); + setPicker(null); + }; + return ( +
+
+ DeepCode · Local service + {notice} + +
+
+ +
+ {picker && ( +
+
{ + event.preventDefault(); + if (path.trim()) closePicker(path.trim()); + }} + > +

Open a project on the service machine

+

+ Enter the folder path on the machine running DeepCode. It will be + opened as untrusted until you choose to trust it. +

+ + setPath(event.target.value)} + placeholder="/path/to/project" + /> +
+ + +
+
+
+ )} +
+ ); +} diff --git a/desktop/src/app/i18n.ts b/desktop/src/app/i18n.ts index a3e6efe0d..6e46df3da 100644 --- a/desktop/src/app/i18n.ts +++ b/desktop/src/app/i18n.ts @@ -22,6 +22,53 @@ export const LOCALES = [ export type Locale = (typeof LOCALES)[number]["value"]; const ZH_CN: Record = { + "provider.signInAuth": "使用 OpenRouter 登录", + "provider.account": "账户", + "provider.loginExplanation": "OpenRouter 登录会返回由用户管理的 API key，不提供刷新令牌。请在运行 DeepCode 的机器上登录。", + "provider.cancelLogin": "取消登录", + "provider.disconnectConfirm": "在本机断开此账户？之后的模型请求将停止。远端密钥请在 OpenRouter 设置页撤销。", + "provider.disconnect": "断开账户", + "provider.manageKeys": "管理远端密钥", + "provider.openLogin": "打开登录页面", + "provider.login.starting": "正在启动登录", + "provider.login.pending": "等待授权", + "provider.login.exchanging": "正在完成授权", + "provider.login.authenticated": "已登录", + "provider.login.cancelled": "已取消", + "provider.login.expired": "登录已过期", + "provider.login.failed": "登录失败", + + "provider.protocol": "API 协议", + "provider.auto": "自动 · 保持现有路由", + "provider.auth": "认证方式", + "provider.apiKey": "API 密钥", + "provider.noAuth": "无需认证", + "provider.compat": "协议兼容性", + "provider.compatExplicit": "请先选择明确的 API 协议，再设置兼容选项。", + "provider.inherit": "继承默认值", + "provider.yes": "是", + "provider.no": "否", + "provider.resetCompat": "清除兼容选项", + "provider.verifyDraft": "验证当前表单", + "provider.verifyModel": "待验证模型", + "provider.probeBudget": "使用当前表单，不保存配置。Agent 验证最多调用模型 3 次，总预算 90 秒，仅使用本地验证工具；Provider 的推理模式可能增加 token 用量。", + "provider.quick": "快速测试", + "provider.agentTest": "验证 Agent 兼容性", + "provider.testing": "正在验证…", + "provider.probeStale": "设置已变化，请重新验证当前配置。", + "provider.capabilities": "模型能力", + "provider.inputModalities": "输入类型", + "provider.textOnly": "仅文本", + "provider.textImage": "文本与图像", + "provider.toolCalling": "工具调用", + "provider.compat.tokenLimitField": "Token 上限字段", + "provider.compat.temperature": "发送 temperature", + "provider.compat.systemRole": "指令消息角色", + "provider.compat.reasoningField": "推理参数字段", + "provider.compat.reasoningContent": "推理历史回传", + "provider.compat.toolMessageName": "发送工具结果名称", + "provider.compat.parallelToolCalls": "并行工具调用", + // Settings dialog shell "settings.title": "设置", "settings.section.general": "通用", @@ -159,6 +206,14 @@ const ZH_CN: Record = { // Runtime notice "runtime.offline": "本地应用服务器不可用。", "runtime.restart": "重启服务", + "runtime.reconnect": "重新连接", + "runtime.browserAuthRequired": "需要授权浏览器访问", + "runtime.browserAuthHelp": "请在终端运行 deepcode web，打开新生成的浏览器访问链接。无需 DeepCode 账号。", + "service.title": "后台服务", + "service.stop": "停止后台服务", + "service.detach": "关闭 Desktop 只断开当前窗口；任务和定时工作会继续在共享后台运行。", + "service.activity": "{{phase}} · {{active}} 个执行中任务 · {{queued}} 个排队任务 · {{terminals}} 个终端", + "service.stopConfirm": "停止共享后台？当前有 {{active}} 个执行中任务、{{queued}} 个排队任务、{{terminals}} 个终端。最多等待 10 秒；仍有活动工作时会保留服务运行。", // Goal rail "goal.setGoal": "设定目标", "goal.setGoalHint": "在普通 Turn 之间保持一个持久的目标", diff --git a/desktop/src/app/interactiveTurnRouter.test.ts b/desktop/src/app/interactiveTurnRouter.test.ts index 5eb7bcd86..0ff900269 100644 --- a/desktop/src/app/interactiveTurnRouter.test.ts +++ b/desktop/src/app/interactiveTurnRouter.test.ts @@ -7,7 +7,7 @@ import type { } from "../generated/app-server"; import type { BridgeError, - DesktopRuntime, + ClientRuntime, RpcMethod, } from "../rpc/contracts"; import { @@ -24,8 +24,8 @@ type Step = | { method: RpcMethod; result: unknown } | { method: RpcMethod; error: BridgeError }; -function scriptedRuntime(steps: Step[], calls: RecordedCall[]): DesktopRuntime { - const transport: Pick = { +function scriptedRuntime(steps: Step[], calls: RecordedCall[]): ClientRuntime { + const transport: Pick = { async request( method: M, params: MethodParams[M], @@ -38,7 +38,7 @@ function scriptedRuntime(steps: Step[], calls: RecordedCall[]): DesktopRuntime { return step.result as MethodResults[M]; }, }; - return transport as DesktopRuntime; + return transport as ClientRuntime; } function turn(id: string, status: Turn["status"], ordinal = 1): Turn { diff --git a/desktop/src/app/interactiveTurnRouter.ts b/desktop/src/app/interactiveTurnRouter.ts index 3c03c8a48..101227c57 100644 --- a/desktop/src/app/interactiveTurnRouter.ts +++ b/desktop/src/app/interactiveTurnRouter.ts @@ -3,7 +3,7 @@ import type { Turn, TurnStartParams, } from "../generated/app-server"; -import type { BridgeError, DesktopRuntime } from "../rpc/contracts"; +import type { BridgeError, ClientRuntime } from "../rpc/contracts"; export type InteractiveDelivery = "started" | "steered" | "queued"; @@ -43,7 +43,7 @@ export interface InteractiveTurnInput { * the user's input. */ export async function sendInteractiveTurn( - runtime: DesktopRuntime, + runtime: ClientRuntime, input: InteractiveTurnInput, ): Promise { const messageId = input.messageId ?? `desktop-${crypto.randomUUID()}`; @@ -86,7 +86,7 @@ export function latestExecutingTurn( } async function start( - runtime: DesktopRuntime, + runtime: ClientRuntime, input: InteractiveTurnInput, messageId: string, ): Promise { @@ -108,7 +108,7 @@ async function start( } async function startOrSteerOnce( - runtime: DesktopRuntime, + runtime: ClientRuntime, input: InteractiveTurnInput, messageId: string, ): Promise { @@ -126,7 +126,7 @@ async function startOrSteerOnce( } async function enqueue( - runtime: DesktopRuntime, + runtime: ClientRuntime, input: InteractiveTurnInput, messageId: string, ): Promise { @@ -148,7 +148,7 @@ async function enqueue( } async function steerOrEnqueue( - runtime: DesktopRuntime, + runtime: ClientRuntime, input: InteractiveTurnInput, expectedTurnId: string, messageId: string, @@ -164,7 +164,7 @@ async function steerOrEnqueue( } async function steer( - runtime: DesktopRuntime, + runtime: ClientRuntime, input: InteractiveTurnInput, expectedTurnId: string, messageId: string, diff --git a/desktop/src/app/replayThreadHistory.test.ts b/desktop/src/app/replayThreadHistory.test.ts index 30a889cb6..005069c96 100644 --- a/desktop/src/app/replayThreadHistory.test.ts +++ b/desktop/src/app/replayThreadHistory.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { Event, @@ -7,6 +7,7 @@ import type { } from "../generated/app-server"; import type { RpcMethod, RpcTransport } from "../rpc/contracts"; import { replayThreadHistory } from "./replayThreadHistory"; +import { ThreadEventStream } from "./threadEventStream"; function event(sequence: number): Event { return { @@ -46,6 +47,57 @@ class ReplayRuntime implements RpcTransport { } describe("replayThreadHistory", () => { + it("pins the first page's head while later events continue to arrive", async () => { + const requests: MethodParams["event/replay"][] = []; + const runtime = new ReplayRuntime(async (params) => { + requests.push(params); + return params.after === 0 + ? { events: [event(1)], nextAfter: 1, hasMore: true, headSequence: 2 } + : { + events: [event(2)], + nextAfter: null, + hasMore: false, + headSequence: 2, + }; + }); + const received: number[] = []; + expect( + await replayThreadHistory(runtime, "thread-1", (value) => + received.push(value.sequence), + ), + ).toBe(2); + expect(requests[0].through).toBeUndefined(); + expect(requests[1].through).toBe(2); + expect(received).toEqual([1, 2]); + }); + + it.each([ + { events: [event(1), event(3)], headSequence: 3 }, + { events: [{ ...event(1), threadId: "another" }], headSequence: 1 }, + { events: [event(1)], headSequence: 2 }, + { events: [event(1)], headSequence: 0 }, + ])("rejects missing or mismatched replay evidence", async (page) => { + const runtime = new ReplayRuntime(async () => ({ + ...page, + hasMore: false, + nextAfter: null, + })); + await expect( + replayThreadHistory(runtime, "thread-1", () => undefined), + ).rejects.toThrow("event/replay"); + }); + + it("does not silently accept a cursor ahead of replaced history", async () => { + const runtime = new ReplayRuntime(async () => ({ + events: [], + hasMore: false, + nextAfter: null, + headSequence: 1, + })); + await expect( + replayThreadHistory(runtime, "thread-1", () => undefined, { after: 5 }), + ).rejects.toThrow("history changed"); + }); it("continues from the server cursor even when a byte-bounded page is short", async () => { const runtime = new ReplayRuntime(async ({ after }) => after === 0 @@ -96,3 +148,172 @@ describe("replayThreadHistory", () => { ).rejects.toThrow("event/replay did not advance its cursor"); }); }); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +function page(...sequences: number[]): MethodResults["event/replay"] { + return { events: sequences.map(event), nextAfter: null, hasMore: false }; +} + +describe("ThreadEventStream", () => { + it("orders replay before overlapping live deltas and drops duplicates", async () => { + const first = deferred(); + const runtime = new ReplayRuntime(async ({ after }) => + after === 0 ? first.promise : page(3, 4), + ); + const accepted: number[] = []; + const stream = new ThreadEventStream( + runtime, + "thread-1", + (value) => accepted.push(value.sequence), + vi.fn(), + ); + const recovering = stream.recover(); + stream.receive(event(4)); + stream.receive(event(3)); + stream.receive(event(2)); + expect(accepted).toEqual([]); + first.resolve(page(1, 2)); + await recovering; + stream.receive(event(3)); + stream.receive(event(5)); + expect(accepted).toEqual([1, 2, 3, 4, 5]); + expect(runtime.afters).toEqual([0, 2]); + }); + + it("repairs an out-of-order live gap from its last contiguous cursor", async () => { + const runtime = new ReplayRuntime(async () => page(2, 3)); + const accepted: number[] = []; + const stream = new ThreadEventStream( + runtime, + "thread-1", + (value) => accepted.push(value.sequence), + vi.fn(), + ); + stream.receive(event(1)); + stream.receive(event(3)); + await vi.waitFor(() => expect(accepted).toEqual([1, 2, 3])); + stream.receive(event(2)); + expect(accepted).toEqual([1, 2, 3]); + expect(runtime.afters).toEqual([1]); + }); + + it("coalesces overflow warnings and recovers a missing tail during replay", async () => { + const first = deferred(); + const runtime = new ReplayRuntime(async ({ after }) => + after === 0 ? first.promise : page(2), + ); + const accepted: number[] = []; + const stream = new ThreadEventStream( + runtime, + "thread-1", + (value) => accepted.push(value.sequence), + vi.fn(), + ); + const initial = stream.recover(); + for (let index = 0; index < 100; index++) + expect(stream.recover()).toBe(initial); + expect(runtime.afters).toEqual([0]); + first.resolve(page(1)); + await initial; + expect(accepted).toEqual([1, 2]); + expect(runtime.afters).toEqual([0, 1]); + }); + + it("recovers a large live burst with one in-flight replay instead of retaining live payloads", async () => { + const first = deferred(); + const runtime = new ReplayRuntime(async ({ after = 0, limit = 1000 }) => { + if (!after) return first.promise; + const last = Math.min(after + limit, 3000); + return { + events: Array.from({ length: last - after }, (_, index) => + event(after + index + 1), + ), + headSequence: 3000, + nextAfter: last < 3000 ? last : null, + hasMore: last < 3000, + }; + }); + const accepted: number[] = []; + const stream = new ThreadEventStream( + runtime, + "thread-1", + (value) => accepted.push(value.sequence), + vi.fn(), + ); + const done = stream.recover(); + for (let sequence = 3000; sequence > 0; sequence--) + stream.receive(event(sequence)); + expect(runtime.afters).toEqual([0]); + first.resolve(page(1)); + await done; + expect(accepted).toEqual( + Array.from({ length: 3000 }, (_, index) => index + 1), + ); + expect(runtime.afters).toEqual([0, 1, 1001, 2001]); + }); + + it.each([false, true])( + "ignores late results or errors after selection changes (%s)", + async (fails) => { + const pending = deferred(); + const runtime = new ReplayRuntime(async () => pending.promise); + const accept = vi.fn(); + const onError = vi.fn(); + const stream = new ThreadEventStream( + runtime, + "thread-1", + accept, + onError, + ); + const done = stream.recover(); + stream.stop(); + stream.receive(event(1)); + if (fails) pending.reject(new Error("connection closed")); + else pending.resolve(page(1)); + await done; + expect(accept).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + }, + ); + + it("retains only applied progress after failure and can repair on a later warning", async () => { + let calls = 0; + const runtime = new ReplayRuntime(async () => { + calls++; + if (calls === 1) return { ...page(1), nextAfter: 1, hasMore: true }; + if (calls === 2) throw new Error("connection closed"); + return page(2, 3); + }); + const accepted: number[] = []; + const stream = new ThreadEventStream( + runtime, + "thread-1", + (value) => accepted.push(value.sequence), + vi.fn(), + ); + await expect(stream.recover()).rejects.toThrow("connection closed"); + await stream.recover(); + expect(accepted).toEqual([1, 2, 3]); + expect(runtime.afters).toEqual([0, 1, 1]); + }); + + it("surfaces irrecoverable history instead of skipping a gap or spinning", async () => { + const runtime = new ReplayRuntime(async () => page()); + const accept = vi.fn(); + const onError = vi.fn(); + const stream = new ThreadEventStream(runtime, "thread-1", accept, onError); + stream.receive(event(3)); + await vi.waitFor(() => expect(onError).toHaveBeenCalledOnce()); + expect(runtime.afters).toEqual([0]); + expect(accept).not.toHaveBeenCalled(); + }); +}); diff --git a/desktop/src/app/replayThreadHistory.ts b/desktop/src/app/replayThreadHistory.ts index 9c6285721..17be83140 100644 --- a/desktop/src/app/replayThreadHistory.ts +++ b/desktop/src/app/replayThreadHistory.ts @@ -15,19 +15,24 @@ export async function replayThreadHistory( runtime: Pick, threadId: string, accept: (event: Event) => void, -): Promise { - let after = 0; + options: { after?: number; isCurrent?: () => boolean } = {}, +): Promise { + let after = options.after ?? 0; + let through: number | undefined; let limit = DEFAULT_REPLAY_LIMIT; + const isCurrent = options.isCurrent ?? (() => true); - for (;;) { + while (isCurrent()) { let result: MethodResults["event/replay"]; try { result = await runtime.request("event/replay", { threadId, after, limit, + ...(through !== undefined ? { through } : {}), }); } catch (error) { + if (!isCurrent()) return after; if (errorCode(error) === "RESPONSE_TOO_LARGE" && limit > 1) { limit = Math.max(1, Math.floor(limit / 2)); continue; @@ -35,8 +40,31 @@ export async function replayThreadHistory( throw error; } + if (!isCurrent()) return after; + if (result.headSequence !== undefined) { + if ( + !Number.isSafeInteger(result.headSequence) || + result.headSequence < after || + (through !== undefined && through !== result.headSequence) + ) { + throw new Error("event/replay history changed; reload the Thread"); + } + through = result.headSequence; + } + for (const event of result.events) { + if (!isCurrent()) return after; + if ( + event.threadId !== threadId || + event.sequence !== after + 1 || + (through !== undefined && event.sequence > through) + ) { + throw new Error( + "event/replay returned a non-contiguous Thread history", + ); + } accept(event); + after = event.sequence; } const compatible = result as MethodResults["event/replay"] & { @@ -48,14 +76,17 @@ export async function replayThreadHistory( ? compatible.hasMore : result.events.length === limit; if (!hasMore) { - return; + if (through !== undefined && after !== through) { + throw new Error("event/replay ended before its captured head"); + } + return after; } const last = result.events.at(-1); const nextAfter = compatible.nextAfter ?? last?.sequence ?? null; - if (nextAfter === null || nextAfter <= after) { + if (nextAfter === null || !result.events.length || nextAfter !== after) { throw new Error("event/replay did not advance its cursor"); } - after = nextAfter; } + return after; } diff --git a/desktop/src/app/threadEventStream.ts b/desktop/src/app/threadEventStream.ts new file mode 100644 index 000000000..4d99440b5 --- /dev/null +++ b/desktop/src/app/threadEventStream.ts @@ -0,0 +1,78 @@ +import type { Event } from "../generated/app-server"; +import type { RpcTransport } from "../rpc/contracts"; +import { replayThreadHistory } from "./replayThreadHistory"; + +/** One selected Thread's contiguous cursor, shared by replay and live delivery. + * Gapped live payloads are recovered from the durable log instead of retained + * in an unbounded client queue. Stopping the stream fences late RPC results. + */ +export class ThreadEventStream { + private cursor = 0; + private observedHead = 0; + private revision = 0; + private active = true; + private pending: Promise | null = null; + + constructor( + private readonly runtime: RpcTransport, + readonly threadId: string, + private readonly accept: (event: Event) => void, + private readonly onError: (error: unknown) => void, + ) {} + + stop(): void { + this.active = false; + } + + receive(event: Event): void { + if ( + !this.active || + event.threadId !== this.threadId || + event.sequence <= this.cursor + ) + return; + this.observedHead = Math.max(this.observedHead, event.sequence); + if (!this.pending && event.sequence === this.cursor + 1) { + this.accept(event); + this.cursor = event.sequence; + } else if (!this.pending) { + void this.recover().catch(this.onError); + } + } + + /** Also used for an overflow warning, including a lost final notification. */ + recover(): Promise { + if (!this.active) return Promise.resolve(); + this.revision += 1; + if (!this.pending) this.pending = this.replay(); + return this.pending; + } + + private async replay(): Promise { + try { + while (this.active) { + const revision = this.revision; + const before = this.cursor; + await replayThreadHistory( + this.runtime, + this.threadId, + (event) => { + this.accept(event); + this.cursor = event.sequence; + }, + { after: this.cursor, isCurrent: () => this.active }, + ); + if (!this.active) return; + if (this.observedHead > this.cursor && before === this.cursor) { + throw new Error( + "event/replay cannot recover an observed event; reload the Thread", + ); + } + if (revision === this.revision && this.observedHead <= this.cursor) + return; + } + } finally { + this.pending = null; + } + } +} diff --git a/desktop/src/app/useWorkspaceController.ts b/desktop/src/app/useWorkspaceController.ts index 203eb2b97..d0d8fc3c8 100644 --- a/desktop/src/app/useWorkspaceController.ts +++ b/desktop/src/app/useWorkspaceController.ts @@ -17,7 +17,7 @@ import { confirmAction } from "../platform/confirmAction"; import type { AnyRpcNotification, BridgeError, - DesktopRuntime, + ClientRuntime, SidecarStatus, } from "../rpc/contracts"; import { @@ -25,7 +25,7 @@ import { sendInteractiveTurn, type InteractiveDelivery, } from "./interactiveTurnRouter"; -import { replayThreadHistory } from "./replayThreadHistory"; +import { ThreadEventStream } from "./threadEventStream"; import { initialWorkspaceState, workspaceReducer, @@ -57,7 +57,8 @@ function isEvent(value: unknown): value is Event { typeof value === "object" && value !== null && "sequence" in value && - typeof (value as { sequence?: unknown }).sequence === "number" + Number.isSafeInteger((value as { sequence?: unknown }).sequence) && + (value as { sequence: number }).sequence > 0 ); } @@ -133,10 +134,11 @@ export interface WorkspaceController { dismissError(): void; } -export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceController { +export function useWorkspaceController(runtime: ClientRuntime): WorkspaceController { const [state, dispatch] = useReducer(workspaceReducer, initialWorkspaceState); const selectedThreadRef = useRef(null); const loadedRuntimeRef = useRef(false); + const eventStreamRef = useRef(null); const reportError = useCallback((error: unknown) => { dispatch({ type: "error", error: normalizeError(error) }); @@ -144,12 +146,23 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro const replayThread = useCallback( async (threadId: string) => { + if (selectedThreadRef.current !== threadId) return; + eventStreamRef.current?.stop(); dispatch({ type: "trace-reset" }); - await replayThreadHistory(runtime, threadId, (event) => { - dispatch({ type: "event", event }); - }); + const stream = new ThreadEventStream( + runtime, + threadId, + (event) => { + if (selectedThreadRef.current === threadId) { + dispatch({ type: "event", event }); + } + }, + reportError, + ); + eventStreamRef.current = stream; + await stream.recover(); }, - [runtime], + [reportError, runtime], ); const loadSettings = useCallback( @@ -165,6 +178,7 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro const loadGoal = useCallback( async (threadId: string) => { const result = await runtime.request("thread/goal/get", { threadId }); + if (selectedThreadRef.current !== threadId) return; dispatch({ type: "goal", goal: result.goal, @@ -199,6 +213,7 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro localStorage.setItem(PROJECT_KEY, selectedThread.projectId); } selectedThreadRef.current = selected; + if (!selected) eventStreamRef.current?.stop(); if (selected) { localStorage.setItem(THREAD_KEY, selected); const resumed = await runtime.request("thread/resume", { @@ -253,48 +268,63 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro dispatch({ type: "runtime", status }); if (status.phase === "ready") { void loadProjects(); - } else if (status.phase === "crashed" || status.phase === "stopped") { + } else { loadedRuntimeRef.current = false; + eventStreamRef.current?.stop(); } }; const acceptNotification = (notification: AnyRpcNotification) => { if (disposed) return; if (notification.method === "server.warning") { - const threadId = selectedThreadRef.current; - if (threadId) void replayThread(threadId).catch(reportError); + if (notification.params.replayRequired === true) { + void eventStreamRef.current?.recover().catch(reportError); + } return; } if (isEvent(notification.params)) { - if (notification.method === "thread.updated") { - dispatch({ type: "event", event: notification.params }); - return; - } const threadId = selectedThreadRef.current; if (threadId === notification.params.threadId) { + if (eventStreamRef.current?.threadId === threadId) { + eventStreamRef.current.receive(notification.params); + } + } else if (notification.method === "thread.updated") { dispatch({ type: "event", event: notification.params }); } } }; void (async () => { + const register = async (subscription: Promise<() => void>) => { + const cleanup = await subscription; + if (disposed) cleanup(); + else cleanups.push(cleanup); + }; try { - cleanups.push(await runtime.onStatus(acceptStatus)); - cleanups.push(await runtime.onNotification(acceptNotification)); - cleanups.push( - await runtime.onLog((message) => dispatch({ type: "log", message })), + // Install live delivery before status can trigger the first replay. + await register(runtime.onNotification(acceptNotification)); + if (disposed) return; + await register(runtime.onStatus(acceptStatus)); + if (disposed) return; + await register( + runtime.onLog((message) => { + if (!disposed) dispatch({ type: "log", message }); + }), ); + if (disposed) return; acceptStatus(await runtime.status()); } catch (error) { - reportError(error); + if (!disposed) reportError(error); } })(); return () => { disposed = true; + eventStreamRef.current?.stop(); + loadedRuntimeRef.current = false; for (const cleanup of cleanups) cleanup(); }; - }, [loadProjects, replayThread, reportError, runtime]); + }, [loadProjects, reportError, runtime]); const withBusy = useCallback( async ( @@ -326,6 +356,7 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro dispatch({ type: "project-upsert", project: result.project }); dispatch({ type: "select-project", projectId: result.project.id }); selectedThreadRef.current = null; + eventStreamRef.current?.stop(); localStorage.setItem(PROJECT_KEY, result.project.id); await loadThreads(result.project.id); await loadSettings(result.project.id); @@ -338,6 +369,7 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro withBusy(async () => { dispatch({ type: "select-project", projectId }); selectedThreadRef.current = null; + eventStreamRef.current?.stop(); localStorage.setItem(PROJECT_KEY, projectId); await loadThreads(projectId, localStorage.getItem(THREAD_KEY)); await loadSettings(projectId); @@ -376,9 +408,10 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro dispatch({ type: "select-thread", threadId: result.thread.id }); selectedThreadRef.current = result.thread.id; localStorage.setItem(THREAD_KEY, result.thread.id); + await replayThread(result.thread.id); return result.thread; }), - [runtime, selectedProject, withBusy], + [replayThread, runtime, selectedProject, withBusy], ); const forkThread = useCallback( @@ -396,8 +429,9 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro dispatch({ type: "select-thread", threadId: isolated.thread.id }); selectedThreadRef.current = isolated.thread.id; localStorage.setItem(THREAD_KEY, isolated.thread.id); + await replayThread(isolated.thread.id); }), - [runtime, selectedThread, withBusy], + [replayThread, runtime, selectedThread, withBusy], ); const activateThread = useCallback( @@ -452,6 +486,7 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro if (!wasSelected) return; selectedThreadRef.current = null; + eventStreamRef.current?.stop(); localStorage.removeItem(THREAD_KEY); const replacement = remaining @@ -778,9 +813,9 @@ export function useWorkspaceController(runtime: DesktopRuntime): WorkspaceContro [runtime, selectedThread, withBusy], ); - const pickWorkflowFile = useCallback(() => runtime.pickFile(), [runtime]); + const pickWorkflowFile = useCallback(() => runtime.pickFile(selectedThreadRef.current ?? undefined), [runtime]); const pickContextFiles = useCallback( - () => runtime.pickContextFiles(), + () => runtime.pickContextFiles(selectedThreadRef.current ?? undefined), [runtime], ); diff --git a/desktop/src/app/workspaceState.test.ts b/desktop/src/app/workspaceState.test.ts index 14e1b0e8b..d3f1851ee 100644 --- a/desktop/src/app/workspaceState.test.ts +++ b/desktop/src/app/workspaceState.test.ts @@ -88,7 +88,7 @@ describe("workspace event projection", () => { expect(replayed.items[0].status).toBe("completed"); expect(replayed.items[0].payload.text).toBe("final"); - expect(replayed.lastSequence).toBe(20); + expect(replayed.entitySequences["item:item-1"]).toBe(20); }); it("projects Goal outcome from the same versioned event", () => { @@ -325,7 +325,7 @@ describe("workspace event projection", () => { }); expect(projected.workflows).toEqual([workflow]); - expect(projected.lastSequence).toBe(21); + expect(projected.entitySequences[`workflow:${workflow.id}`]).toBe(21); }); it("replays structured Turn plans and ignores stale updates", () => { @@ -381,7 +381,7 @@ describe("workspace event projection", () => { ], updatedAt: "2026-07-16T00:00:02Z", }); - expect(stale.lastSequence).toBe(22); + expect(stale.entitySequences[`plan:${turn.id}`]).toBe(22); }); it("keeps the global Session index when project context changes", () => { @@ -416,7 +416,7 @@ describe("workspace event projection", () => { updatedAt: "2026-07-16T00:00:00Z", }, }, - lastSequence: 4, + entitySequences: { "item:item-1": 4 }, }, { type: "thread-remove", threadId: thread.id }, ); @@ -426,6 +426,6 @@ describe("workspace event projection", () => { expect(removed.turns).toEqual([]); expect(removed.items).toEqual([]); expect(removed.plansByTurnId).toEqual({}); - expect(removed.lastSequence).toBe(0); + expect(removed.entitySequences).toEqual({}); }); }); diff --git a/desktop/src/app/workspaceState.ts b/desktop/src/app/workspaceState.ts index 7b96ecad5..b48381e7c 100644 --- a/desktop/src/app/workspaceState.ts +++ b/desktop/src/app/workspaceState.ts @@ -38,7 +38,6 @@ export interface WorkspaceState { plansByTurnId: Record; goal: Goal | null; goalOutcome: GoalOutcome | null; - lastSequence: number; entitySequences: Record; selectedItemId: string | null; busy: boolean; @@ -89,7 +88,6 @@ export const initialWorkspaceState: WorkspaceState = { plansByTurnId: {}, goal: null, goalOutcome: null, - lastSequence: 0, entitySequences: {}, selectedItemId: null, busy: false, @@ -154,24 +152,15 @@ function applyItemDelta(state: WorkspaceState, event: Event): WorkspaceState { const itemId = event.itemId; const delta = event.payload.delta; if (!itemId || typeof delta !== "string") { - return { - ...state, - lastSequence: Math.max(state.lastSequence, event.sequence), - }; + return state; } const key = `item:${itemId}`; if ((state.entitySequences[key] ?? 0) >= event.sequence) { - return { - ...state, - lastSequence: Math.max(state.lastSequence, event.sequence), - }; + return state; } const index = state.items.findIndex((candidate) => candidate.id === itemId); if (index === -1) { - return { - ...state, - lastSequence: Math.max(state.lastSequence, event.sequence), - }; + return state; } const current = state.items[index]; @@ -183,7 +172,6 @@ function applyItemDelta(state: WorkspaceState, event: Event): WorkspaceState { if (current.kind === "reasoning_summary" && !isReasoningDelta) { return { ...state, - lastSequence: Math.max(state.lastSequence, event.sequence), entitySequences: { ...state.entitySequences, [key]: event.sequence, @@ -218,7 +206,6 @@ function applyItemDelta(state: WorkspaceState, event: Event): WorkspaceState { return { ...state, items, - lastSequence: Math.max(state.lastSequence, event.sequence), entitySequences: { ...state.entitySequences, [key]: event.sequence, @@ -234,10 +221,7 @@ function applyDomainEvent(state: WorkspaceState, event: Event): WorkspaceState { const plan = eventPlan(event); const key = event.turnId ? `plan:${event.turnId}` : null; if (!plan || !key || (state.entitySequences[key] ?? 0) >= event.sequence) { - return { - ...state, - lastSequence: Math.max(state.lastSequence, event.sequence), - }; + return state; } return { ...state, @@ -245,7 +229,6 @@ function applyDomainEvent(state: WorkspaceState, event: Event): WorkspaceState { ...state.plansByTurnId, [plan.turnId]: plan, }, - lastSequence: Math.max(state.lastSequence, event.sequence), entitySequences: { ...state.entitySequences, [key]: event.sequence, @@ -255,10 +238,7 @@ function applyDomainEvent(state: WorkspaceState, event: Event): WorkspaceState { if (event.type === "goal.updated") { const key = `goal:${event.threadId}`; if ((state.entitySequences[key] ?? 0) > event.sequence) { - return { - ...state, - lastSequence: Math.max(state.lastSequence, event.sequence), - }; + return state; } const rawGoal = event.payload.goal; const rawOutcome = event.payload.outcome; @@ -268,7 +248,6 @@ function applyDomainEvent(state: WorkspaceState, event: Event): WorkspaceState { goalOutcome: isRecord(rawOutcome) ? (rawOutcome as unknown as GoalOutcome) : null, - lastSequence: Math.max(state.lastSequence, event.sequence), entitySequences: { ...state.entitySequences, [key]: event.sequence, @@ -317,7 +296,6 @@ function applyDomainEvent(state: WorkspaceState, event: Event): WorkspaceState { artifact && accepted.get("artifact") ? upsert(state.artifacts, artifact) : state.artifacts, - lastSequence: Math.max(state.lastSequence, event.sequence), entitySequences: nextSequences, }; } @@ -363,7 +341,6 @@ export function workspaceReducer( plansByTurnId: {}, goal: null, goalOutcome: null, - lastSequence: 0, entitySequences: {}, selectedItemId: null, }; @@ -395,7 +372,6 @@ export function workspaceReducer( plansByTurnId: selected ? {} : state.plansByTurnId, goal: selected ? null : state.goal, goalOutcome: selected ? null : state.goalOutcome, - lastSequence: selected ? 0 : state.lastSequence, entitySequences: selected ? {} : state.entitySequences, selectedItemId: selected ? null : state.selectedItemId, }; @@ -412,7 +388,6 @@ export function workspaceReducer( plansByTurnId: {}, goal: null, goalOutcome: null, - lastSequence: 0, entitySequences: {}, selectedItemId: null, }; @@ -427,7 +402,6 @@ export function workspaceReducer( plansByTurnId: {}, goal: null, goalOutcome: null, - lastSequence: 0, entitySequences: {}, selectedItemId: null, }; diff --git a/desktop/src/components/RuntimeNotice.module.css b/desktop/src/components/RuntimeNotice.module.css index 1ef1e4da6..80b396e0a 100644 --- a/desktop/src/components/RuntimeNotice.module.css +++ b/desktop/src/components/RuntimeNotice.module.css @@ -28,12 +28,10 @@ } .copy span { - overflow: hidden; color: var(--text-secondary); font-size: var(--text-xs); line-height: 1.45; - text-overflow: ellipsis; - white-space: nowrap; + overflow-wrap: anywhere; } .actions { @@ -74,8 +72,4 @@ right: 8px; width: calc(100vw - 16px); } - - .copy span { - white-space: normal; - } } diff --git a/desktop/src/components/RuntimeNotice.test.tsx b/desktop/src/components/RuntimeNotice.test.tsx new file mode 100644 index 000000000..71e03dd7d --- /dev/null +++ b/desktop/src/components/RuntimeNotice.test.tsx @@ -0,0 +1,81 @@ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { __setLocaleForTests, initI18n } from "../app/i18n"; +import type { SidecarStatus } from "../rpc/contracts"; +import { RuntimeNotice } from "./RuntimeNotice"; + +const stopped: SidecarStatus = { + phase: "stopped", + message: "The browser needs a new access link", + errorCode: "AUTH_REQUIRED", + launchSource: "browser", + serverInfo: null, +}; + +beforeEach(() => { + initI18n(); + __setLocaleForTests("en"); +}); +afterEach(() => { + cleanup(); + __setLocaleForTests("en"); +}); + +it.each(["en", "zh-CN"] as const)( + "explains browser authorization in %s without offering a futile reconnect", + (locale) => { + __setLocaleForTests(locale); + render( + , + ); + expect(screen.getByRole("alert").textContent).toContain("deepcode web"); + expect(screen.getByRole("alert").textContent).toContain( + locale === "en" ? "Browser access required" : "需要授权浏览器访问", + ); + expect(screen.getByRole("alert").textContent).not.toContain("APP_SERVER_OFFLINE"); + expect(screen.queryByRole("button")).toBeNull(); + }, +); + +it("keeps reconnection available for network failures", () => { + const reconnect = vi.fn(); + render( + , + ); + fireEvent.click(screen.getByRole("button", { name: "Reconnect" })); + expect(reconnect).toHaveBeenCalledOnce(); + expect(screen.getByRole("alert").textContent).toContain("Network unavailable"); +}); + +it("keeps native restart and ordinary application error dismissal", () => { + const restart = vi.fn(), dismiss = vi.fn(); + render( + , + ); + expect(screen.getByRole("alert").textContent).toContain("Trust this project"); + fireEvent.click(screen.getByRole("button", { name: "Restart service" })); + fireEvent.click(screen.getByRole("button", { name: "Dismiss error" })); + expect(restart).toHaveBeenCalledOnce(); + expect(dismiss).toHaveBeenCalledOnce(); +}); diff --git a/desktop/src/components/RuntimeNotice.tsx b/desktop/src/components/RuntimeNotice.tsx index d9578a19b..06d225d19 100644 --- a/desktop/src/components/RuntimeNotice.tsx +++ b/desktop/src/components/RuntimeNotice.tsx @@ -10,6 +10,7 @@ interface RuntimeNoticeProps { busy: boolean; onRestart(): void; onDismissError(): void; + reconnectOnly?: boolean; } export function RuntimeNotice({ @@ -18,20 +19,25 @@ export function RuntimeNotice({ busy, onRestart, onDismissError, + reconnectOnly = false, }: RuntimeNoticeProps) { const { t } = useTranslation(); if (!error && runtime.phase !== "crashed" && runtime.phase !== "stopped") return null; const serviceOffline = runtime.phase === "crashed" || runtime.phase === "stopped"; - const message = error?.message ?? runtime.message ?? t("runtime.offline", "The local App Server is unavailable."); + const code = runtime.errorCode ?? error?.code; + const authRequired = reconnectOnly && code === "AUTH_REQUIRED"; + const message = authRequired + ? t("runtime.browserAuthHelp", "Run deepcode web in your terminal to open a new browser access link. No DeepCode account is needed.") + : (runtime.errorCode ? runtime.message : error?.message) ?? runtime.message ?? t("runtime.offline", "The local App Server is unavailable."); return (
}> runtime.downloadFile!(thread.id, path) : undefined} trusted={trusted} hasActiveTurn={hasActiveTurn} workbench={workbench} diff --git a/desktop/src/features/management/ManagementWorkspace.tsx b/desktop/src/features/management/ManagementWorkspace.tsx index b8505fb1c..91dd085cc 100644 --- a/desktop/src/features/management/ManagementWorkspace.tsx +++ b/desktop/src/features/management/ManagementWorkspace.tsx @@ -1,6 +1,6 @@ import type { Project, SkillInfo, Thread } from "../../generated/app-server"; import type { DesktopDestination } from "../../app/useDesktopUi"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; import { AutomationsPage } from "../automations/AutomationsPage"; import { SkillsPage } from "../extensions/SkillsPage"; import { PluginsPage } from "../plugins/PluginsPage"; @@ -8,7 +8,7 @@ import { McpPage } from "../mcp/McpPage"; interface ManagementWorkspaceProps { destination: DesktopDestination; - runtime: DesktopRuntime; + runtime: ClientRuntime; project: Project | null; onThreadCreated(thread: Thread): void; onOpenThread(threadId: string): void; diff --git a/desktop/src/features/mcp/McpPage.test.tsx b/desktop/src/features/mcp/McpPage.test.tsx index 9e3d0f58d..15bb06753 100644 --- a/desktop/src/features/mcp/McpPage.test.tsx +++ b/desktop/src/features/mcp/McpPage.test.tsx @@ -8,7 +8,7 @@ import type { MethodResults, Project, } from "../../generated/app-server"; -import type { DesktopRuntime, RpcMethod } from "../../rpc/contracts"; +import type { ClientRuntime, RpcMethod } from "../../rpc/contracts"; import { McpPage } from "./McpPage"; const project: Project = { @@ -202,7 +202,7 @@ test("adds a credential-bound stdio MCP server through the shared RPC", async () const runtime = new McpRuntime(); render( , ); @@ -283,7 +283,7 @@ test("shows Plugin MCP servers without exposing native edit actions", async () = render( , ); @@ -298,7 +298,7 @@ test("adds a bundled preset disabled, tests it, then enables agent use", async ( const runtime = new McpRuntime(); render( , ); diff --git a/desktop/src/features/mcp/McpPage.tsx b/desktop/src/features/mcp/McpPage.tsx index 5e00ce93a..d0dca9a08 100644 --- a/desktop/src/features/mcp/McpPage.tsx +++ b/desktop/src/features/mcp/McpPage.tsx @@ -23,7 +23,7 @@ import type { McpServerInfo, Project, } from "../../generated/app-server"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; import styles from "../management/ManagementWorkspace.module.css"; import { useMcpCatalog } from "./useMcpCatalog"; @@ -82,7 +82,7 @@ export function McpPage({ runtime, project, }: { - runtime: DesktopRuntime; + runtime: ClientRuntime; project: Project | null; }) { const catalog = useMcpCatalog(runtime, project); diff --git a/desktop/src/features/mcp/useMcpCatalog.test.tsx b/desktop/src/features/mcp/useMcpCatalog.test.tsx index 70631759d..400f641f3 100644 --- a/desktop/src/features/mcp/useMcpCatalog.test.tsx +++ b/desktop/src/features/mcp/useMcpCatalog.test.tsx @@ -7,7 +7,7 @@ import type { McpProbeResult, Project, } from "../../generated/app-server"; -import type { DesktopRuntime, RpcMethod } from "../../rpc/contracts"; +import type { ClientRuntime, RpcMethod } from "../../rpc/contracts"; import { useMcpCatalog } from "./useMcpCatalog"; function deferred() { @@ -78,7 +78,7 @@ describe("useMcpCatalog project ownership", () => { const runtime = { request, onNotification: async () => () => undefined, - } as unknown as DesktopRuntime; + } as unknown as ClientRuntime; const { result, rerender } = renderHook( ({ selectedProject }) => useMcpCatalog(runtime, selectedProject), diff --git a/desktop/src/features/mcp/useMcpCatalog.ts b/desktop/src/features/mcp/useMcpCatalog.ts index 2ab5d311c..e2407a24f 100644 --- a/desktop/src/features/mcp/useMcpCatalog.ts +++ b/desktop/src/features/mcp/useMcpCatalog.ts @@ -9,7 +9,7 @@ import type { McpProbeResult, Project, } from "../../generated/app-server"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; interface McpCatalogState { key: string; @@ -19,7 +19,7 @@ interface McpCatalogState { error: string | null; } -export function useMcpCatalog(runtime: DesktopRuntime, project: Project | null) { +export function useMcpCatalog(runtime: ClientRuntime, project: Project | null) { const projectId = project?.id; const catalogKey = projectId ?? "__user__"; const generation = useRef(0); diff --git a/desktop/src/features/navigation/DesktopSidebar.module.css b/desktop/src/features/navigation/DesktopSidebar.module.css index 47ed64d88..953b81c64 100644 --- a/desktop/src/features/navigation/DesktopSidebar.module.css +++ b/desktop/src/features/navigation/DesktopSidebar.module.css @@ -1,7 +1,7 @@ .sidebar { display: flex; min-width: 0; - height: calc(100vh - 16px); + height: calc(100% - 16px); flex-direction: column; margin: 8px 4px 8px 8px; padding: 11px 10px 10px; diff --git a/desktop/src/features/plugins/PluginsPage.test.tsx b/desktop/src/features/plugins/PluginsPage.test.tsx index 384c87c1d..cbd00a00b 100644 --- a/desktop/src/features/plugins/PluginsPage.test.tsx +++ b/desktop/src/features/plugins/PluginsPage.test.tsx @@ -8,7 +8,7 @@ import type { } from "../../generated/app-server"; import type { AnyRpcNotification, - DesktopRuntime, + ClientRuntime, RpcMethod, } from "../../rpc/contracts"; import { PluginsPage } from "./PluginsPage"; @@ -96,7 +96,7 @@ afterEach(() => { test("adds, disables, and unregisters a local Plugin", async () => { const runtime = new PluginRuntime(); vi.spyOn(window, "confirm").mockReturnValue(true); - render(); + render(); expect(screen.getByText(/MCP servers join the shared tool runtime/)).toBeTruthy(); await screen.findByText(/No Plugins registered/); @@ -120,7 +120,7 @@ test("adds, disables, and unregisters a local Plugin", async () => { test("reloads the registry after plugins.changed", async () => { const runtime = new PluginRuntime(); - render(); + render(); await screen.findByText(/No Plugins registered/); runtime.catalog = { ...runtime.catalog, plugins: [plugin] }; diff --git a/desktop/src/features/plugins/PluginsPage.tsx b/desktop/src/features/plugins/PluginsPage.tsx index b26442fc1..1b90c25e2 100644 --- a/desktop/src/features/plugins/PluginsPage.tsx +++ b/desktop/src/features/plugins/PluginsPage.tsx @@ -1,10 +1,10 @@ import { FolderInput, Power, RefreshCw, Trash2 } from "lucide-react"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; import styles from "../management/ManagementWorkspace.module.css"; import { usePluginCatalog } from "./usePluginCatalog"; -export function PluginsPage({ runtime }: { runtime: DesktopRuntime }) { +export function PluginsPage({ runtime }: { runtime: ClientRuntime }) { const catalog = usePluginCatalog(runtime); const addPlugin = async () => { diff --git a/desktop/src/features/plugins/usePluginCatalog.ts b/desktop/src/features/plugins/usePluginCatalog.ts index 9380325b9..701c9cede 100644 --- a/desktop/src/features/plugins/usePluginCatalog.ts +++ b/desktop/src/features/plugins/usePluginCatalog.ts @@ -4,7 +4,7 @@ import type { PluginCatalogResult, PluginInfo, } from "../../generated/app-server"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; interface PluginCatalogState extends PluginCatalogResult { loading: boolean; @@ -23,7 +23,7 @@ function message(error: unknown): string { return error instanceof Error ? error.message : String(error); } -export function usePluginCatalog(runtime: DesktopRuntime) { +export function usePluginCatalog(runtime: ClientRuntime) { const [state, setState] = useState(emptyState); const [selectedId, setSelectedId] = useState(null); const generation = useRef(0); diff --git a/desktop/src/features/presets/usePresetCatalog.ts b/desktop/src/features/presets/usePresetCatalog.ts index 319a8c492..b1b1708e7 100644 --- a/desktop/src/features/presets/usePresetCatalog.ts +++ b/desktop/src/features/presets/usePresetCatalog.ts @@ -1,7 +1,7 @@ import { useCallback, useEffect, useState } from "react"; import type { AgentPresetEntry } from "../../generated/app-server"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; export interface PresetCatalogState { entries: AgentPresetEntry[]; @@ -28,7 +28,7 @@ function errorMessage(error: unknown): string { * an effect and a late-landing response for a previous owner is ignored. */ export function usePresetCatalog( - runtime: DesktopRuntime, + runtime: ClientRuntime, projectId: string | null, threadId: string | null, ): PresetCatalogState & { select(presetId: string | null): Promise } { diff --git a/desktop/src/features/settings/ConnectionProbe.test.tsx b/desktop/src/features/settings/ConnectionProbe.test.tsx new file mode 100644 index 000000000..aad9f643f --- /dev/null +++ b/desktop/src/features/settings/ConnectionProbe.test.tsx @@ -0,0 +1,88 @@ +import { + act, + cleanup, + fireEvent, + render, + screen, +} from "@testing-library/react"; +import { afterEach, expect, test, vi } from "vitest"; +import type { ProviderTestResult } from "../../generated/app-server"; +import type { ConnectionCatalogController } from "./useConnectionCatalog"; +import { ConnectionProbe } from "./ConnectionProbe"; + +afterEach(cleanup); + +test("unsaved probe uses current form and hides a result after edits", async () => { + let complete!: (value: ProviderTestResult) => void; + const probe = vi.fn( + () => + new Promise((resolve) => { + complete = resolve; + }), + ); + const controller = { test: probe } as unknown as ConnectionCatalogController; + const connection = { + id: "local", + protocol: "openai_responses" as const, + apiBase: "http://localhost:1234/v1", + }; + const { rerender } = render( + , + ); + fireEvent.change(screen.getByRole("textbox"), { target: { value: "model" } }); + fireEvent.click( + screen.getByRole("button", { name: "Verify Agent compatibility" }), + ); + expect(probe).toHaveBeenCalledWith("local", "model", { + connection, + mode: "agent", + }); + rerender( + , + ); + await act(async () => + complete({ + connectionId: "local", + status: "ready", + ok: true, + latencyMs: 1, + modelCount: 1, + error: null, + stages: [ + { + id: "credential", + status: "passed", + detail: "Credential", + latencyMs: 0, + modelCount: null, + modelId: null, + }, + { + id: "catalog", + status: "passed", + detail: "Catalog", + latencyMs: 0, + modelCount: 1, + modelId: null, + }, + { + id: "model", + status: "passed", + detail: "Model", + latencyMs: 1, + modelCount: null, + modelId: "model", + }, + ], + }), + ); + expect(screen.queryByText("Model request verified")).toBeNull(); + expect( + screen.getByText( + "Settings changed. Verify again to check this configuration.", + ), + ).toBeTruthy(); +}); diff --git a/desktop/src/features/settings/ConnectionProbe.tsx b/desktop/src/features/settings/ConnectionProbe.tsx new file mode 100644 index 000000000..8bf5ae2ad --- /dev/null +++ b/desktop/src/features/settings/ConnectionProbe.tsx @@ -0,0 +1,100 @@ +import { useRef, useState } from "react"; +import { useTranslation } from "react-i18next"; +import type { + ProviderTestResult, + ProviderUpsertParams, +} from "../../generated/app-server"; +import type { ConnectionCatalogController } from "./useConnectionCatalog"; +import { ConnectionVerification } from "./ConnectionVerification"; +import styles from "./ConnectionSettings.module.css"; + +export function ConnectionProbe({ + connection, + controller, +}: { + connection: ProviderUpsertParams["connection"]; + controller: ConnectionCatalogController; +}) { + const { t } = useTranslation(); + const [model, setModel] = useState(""); + const [busy, setBusy] = useState(false); + const inFlight = useRef(false); + const [outcome, setOutcome] = useState<{ + fingerprint: string; + result?: ProviderTestResult; + error?: string; + } | null>(null); + const fingerprint = JSON.stringify([connection, model.trim()]); + const test = async (mode: "quick" | "agent") => { + if (inFlight.current) return; + inFlight.current = true; + setBusy(true); + setOutcome(null); + try { + const result = await controller.test(connection.id, model.trim(), { + connection, + mode, + }); + setOutcome({ fingerprint, result }); + } catch (cause) { + setOutcome({ + fingerprint, + error: cause instanceof Error ? cause.message : String(cause), + }); + } finally { + inFlight.current = false; + setBusy(false); + } + }; + const current = outcome?.fingerprint === fingerprint ? outcome : null; + return ( +
+ {t("provider.verifyDraft", "Verify current settings")} + + + {t( + "provider.probeBudget", + "Uses the current form without saving. Agent verification makes up to 3 model requests within 90 seconds, using only a local verification tool. Provider reasoning can increase the token budget.", + )} + +
+ + +
+ {busy ? ( +

{t("provider.testing", "Verification running…")}

+ ) : null} + {current?.error ?

{current.error}

: null} + {current?.result ? ( + + ) : null} + {outcome && !current ? ( +

+ {t( + "provider.probeStale", + "Settings changed. Verify again to check this configuration.", + )} +

+ ) : null} +
+ ); +} diff --git a/desktop/src/features/settings/ConnectionSettings.tsx b/desktop/src/features/settings/ConnectionSettings.tsx index 7fa9a0b89..f0ec4ef1f 100644 --- a/desktop/src/features/settings/ConnectionSettings.tsx +++ b/desktop/src/features/settings/ConnectionSettings.tsx @@ -15,13 +15,16 @@ import type { ConnectionInfo, ManualModelEntry, ProviderTestResult, - ProviderUpsertParams, } from "../../generated/app-server"; import { useEscapeLayer } from "../../app/escapeLayer"; import { confirmAction } from "../../platform/confirmAction"; import type { ConnectionCatalogController } from "./useConnectionCatalog"; import { ConnectionVerification } from "./ConnectionVerification"; import { ModelListEditor } from "./ModelListEditor"; +import { type Draft, emptyDraft, connectionMutation } from "./connectionDraft"; +import { ProtocolSettings } from "./ProtocolSettings"; +import { ConnectionProbe } from "./ConnectionProbe"; +import { ProviderLogin } from "./ProviderLogin"; import styles from "./ConnectionSettings.module.css"; interface ConnectionSettingsProps { @@ -33,40 +36,6 @@ interface ConnectionSettingsProps { scope: ConfigScope; } -interface Draft { - id: string; - label: string; - template: string; - adapter: "openai_compat" | "anthropic"; - apiBase: string; - /** Environment-variable reference — the advanced alternative to a - * stored key (the write-only input is the primary path, dsh style). */ - apiKeyEnv: string; - apiKey: string; - clearApiKey: boolean; - modelCatalog: "auto" | "openrouter" | "openai" | "anthropic" | "manual"; - manualModels: ManualModelEntry[]; - /** True when the launch environment currently provides this key: it - * outranks a pasted key, so the form must say so instead of letting a - * paste silently lose. */ - environmentShadows: boolean; - shadowingEnvName: string; -} - -const emptyDraft: Draft = { - id: "", - label: "", - template: "", - adapter: "openai_compat", - apiBase: "", - apiKeyEnv: "", - apiKey: "", - clearApiKey: false, - modelCatalog: "auto", - manualModels: [], - environmentShadows: false, - shadowingEnvName: "", -}; export function ConnectionSettings({ controller, @@ -126,6 +95,9 @@ export function ConnectionSettings({ label: connection.label, template: connection.providerName, adapter: connection.adapter, + protocol: connection.protocol ?? "auto", + auth: connection.auth ?? "api_key", + compat: connection.compat ?? {}, apiBase: connection.apiBase ?? "", apiKeyEnv: connection.apiKeyEnv ?? "", apiKey: "", @@ -155,6 +127,14 @@ export function ConnectionSettings({ template: template.name, adapter: template.adapter === "anthropic" ? "anthropic" : "openai_compat", apiBase: template.defaultApiBase ?? "", + protocol: "auto", + auth: template.local ? "none" : "api_key", + compat: {}, + apiKey: "", + apiKeyEnv: "", + clearApiKey: false, + environmentShadows: false, + shadowingEnvName: "", modelCatalog: "auto", manualModels: [], })); @@ -165,24 +145,7 @@ export function ConnectionSettings({ setSaving(true); try { const connectionId = editing.id.trim().toLocaleLowerCase(); - const connection: ProviderUpsertParams["connection"] = { - id: connectionId, - label: editing.label.trim() || editing.id.trim(), - template: editing.template, - adapter: editing.adapter, - apiBase: editing.apiBase.trim() || null, - apiKeyEnv: editing.apiKeyEnv.trim() || null, - modelCatalog: editing.modelCatalog, - manualModels: editing.manualModels - .map((entry) => ({ ...entry, id: entry.id.trim() })) - .filter((entry) => entry.id) - .map((entry) => (hasDeclarations(entry) ? entry : entry.id)), - enabled: true, - }; - if (editing.apiKey.trim()) { - connection.apiKey = editing.apiKey.trim(); - } - if (editing.clearApiKey) connection.clearApiKey = true; + const connection = connectionMutation(editing); await controller.upsert(connection); setEditing(null); setTestingId(connectionId); @@ -231,13 +194,7 @@ export function ConnectionSettings({ try { // Probe THE FORM AS SHOWN (dsh's rule): an unsaved base URL or a key // typed but not yet stored takes part; nothing is written. - const result = await controller.discover({ - ...(editingExisting - ? { connectionId: editing.id.trim().toLocaleLowerCase() } - : { template: editing.template }), - ...(editing.apiBase.trim() ? { apiBase: editing.apiBase.trim() } : {}), - ...(editing.apiKey.trim() ? { apiKey: editing.apiKey.trim() } : {}), - }); + const result = await controller.discover({ connection: connectionMutation(editing) }); setModelFetchState({ editorId: editingId, loading: false, @@ -412,6 +369,7 @@ export function ConnectionSettings({ {modelFaceLabel(connection, result)}

+ {connection.auth === "oauth" ? : null} {result ? ( ) : testingId === connection.id ? ( @@ -530,7 +488,8 @@ export function ConnectionSettings({ ) : null} - {!selectedTemplate?.local ? ( + + {editing.auth === "api_key" ? (
Credential {editing.environmentShadows ? ( @@ -646,6 +605,7 @@ export function ConnectionSettings({ ) : null} setEditing({ ...editing, manualModels }) @@ -658,6 +618,7 @@ export function ConnectionSettings({
) : null} +
Advanced connection settings @@ -809,14 +770,6 @@ export function ConnectionSettings({ ); } -function hasDeclarations(entry: ManualModelEntry): boolean { - return ( - entry.label != null || - entry.contextWindow != null || - entry.maxOutputTokens != null || - entry.reasoningEfforts != null - ); -} function modelFaceLabel( connection: ConnectionInfo, @@ -843,6 +796,8 @@ function credentialLabel(connection: ConnectionInfo): string { return "legacy config"; case "not_required": return "no key required"; + case "oauth": + return "signed-in account"; default: return "no key"; } @@ -856,6 +811,8 @@ function connectionStatus( if (result?.status === "connected") return "Catalog connected"; if (result?.status === "limited") return "Model check needed"; if (result?.status === "error") return "Needs attention"; + if (connection.auth === "none") return "No authentication required"; + if (connection.auth === "oauth") return connection.configured ? "Signed in" : "Sign-in required"; return connection.configured ? "Credential saved" : "Needs credential"; } diff --git a/desktop/src/features/settings/ConnectionVerification.tsx b/desktop/src/features/settings/ConnectionVerification.tsx index a1c07831b..2f58d6d32 100644 --- a/desktop/src/features/settings/ConnectionVerification.tsx +++ b/desktop/src/features/settings/ConnectionVerification.tsx @@ -20,21 +20,33 @@ const STAGE_LABELS: Record = { credential: "Credential", catalog: "Model catalog", model: "Model request", + stream: "Streaming", + tool: "Tool call", + continuation: "Tool result continuation", + reasoning: "Reasoning", + image: "Image input", }; export function ConnectionVerification({ result, compact = false, }: ConnectionVerificationProps) { + const label = + result.status === "ready" && + result.stages.some( + (stage) => stage.id === "continuation" && stage.status === "passed", + ) + ? "Agent tool round trip verified" + : statusLabel(result.status); return (
- {statusLabel(result.status)} + {label} {result.latencyMs} ms total
@@ -62,7 +74,7 @@ function StageIcon({ stage }: { stage: ProviderVerificationStage }) { function statusLabel(status: ProviderTestResult["status"]): string { switch (status) { case "ready": - return "Ready for agent work"; + return "Model request verified"; case "connected": return "Catalog connected"; case "limited": diff --git a/desktop/src/features/settings/ModelListEditor.tsx b/desktop/src/features/settings/ModelListEditor.tsx index 492cea6db..69a54f134 100644 --- a/desktop/src/features/settings/ModelListEditor.tsx +++ b/desktop/src/features/settings/ModelListEditor.tsx @@ -4,8 +4,8 @@ * row's own disclosure. Capacity fields accept K/M suffixes (1M = 1000K) * and store plain counts; text that does not parse stays on screen so the * save-time rejection names a row that is still visible. Rows are the - * config file's own entries — unknown future fields survive because plain - * ids stay plain and objects are edited field-wise, never rebuilt. + * config file's own entries; supported declarations are edited field-wise + * while an entry with only an id stays a plain id when saved. * * Effort declarations (`reasoningEfforts`) are config-file-only, exactly * as in dsh: a per-model ladder is a capability statement, not a form @@ -21,19 +21,32 @@ import { Plus, Trash2 } from "lucide-react"; import { useState } from "react"; -import type { ManualModelEntry } from "../../generated/app-server"; +import { useTranslation } from "react-i18next"; +import { CompatEditor } from "./ProtocolSettings"; +import type { + ManualModelEntry, + ProviderProtocol, +} from "../../generated/app-server"; import { capacityText, parseCapacity } from "./modelCapacity"; import styles from "./ConnectionSettings.module.css"; interface ModelListEditorProps { + protocol?: ProviderProtocol; entries: ManualModelEntry[]; onChange(entries: ManualModelEntry[]): void; } -export function ModelListEditor({ entries, onChange }: ModelListEditorProps) { +export function ModelListEditor({ + entries, + onChange, + protocol = "auto", +}: ModelListEditorProps) { + const { t } = useTranslation(); const update = (index: number, patch: Partial) => { onChange( - entries.map((entry, at) => (at === index ? { ...entry, ...patch } : entry)), + entries.map((entry, at) => + at === index ? { ...entry, ...patch } : entry, + ), ); }; @@ -64,9 +77,7 @@ export function ModelListEditor({ entries, onChange }: ModelListEditorProps) { type="button" className={styles.modelRowRemove} aria-label={`Remove model ${entry.id || index + 1}`} - onClick={() => - onChange(entries.filter((_, at) => at !== index)) - } + onClick={() => onChange(entries.filter((_, at) => at !== index))} > @@ -88,6 +99,68 @@ export function ModelListEditor({ entries, onChange }: ModelListEditorProps) { />
+
+ + {t("provider.capabilities", "Model capabilities")} + + + + update(index, { compat })} + /> +
))} + ) : null} + + ); +} diff --git a/desktop/src/features/settings/ProviderLogin.tsx b/desktop/src/features/settings/ProviderLogin.tsx new file mode 100644 index 000000000..bf144d0f1 --- /dev/null +++ b/desktop/src/features/settings/ProviderLogin.tsx @@ -0,0 +1,136 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import type { + ConnectionInfo, + ProviderLoginFlow, +} from "../../generated/app-server"; +import type { ConnectionCatalogController } from "./useConnectionCatalog"; +import { confirmAction } from "../../platform/confirmAction"; +import styles from "./ConnectionSettings.module.css"; + +const pending = (flow: ProviderLoginFlow | null) => + flow && ["starting", "pending", "exchanging"].includes(flow.status); + +export function ProviderLogin({ + connection, + controller, +}: { + connection: ConnectionInfo; + controller: ConnectionCatalogController; +}) { + const { t } = useTranslation(); + const [flow, setFlow] = useState(null); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + const { pollLogin, reload } = controller; + useEffect(() => { + if (!pending(flow) || !flow) return; + let cancelled = false; + const timer = setTimeout(() => { + void pollLogin(flow.flowId) + .then(async (next) => { + if (cancelled) return; + setFlow(next); + if (next.status === "authenticated") await reload(); + }) + .catch((cause) => { + if (!cancelled) setError(String(cause)); + }); + }, 1000); + return () => { + cancelled = true; + clearTimeout(timer); + }; + }, [flow, pollLogin, reload]); + const run = async (operation: () => Promise) => { + setBusy(true); + setError(null); + try { + await operation(); + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setBusy(false); + } + }; + return ( +
+ {connection.accountId ? ( +

+ {t("provider.account", "Account")}: {connection.accountId} +

+ ) : null} + + {t( + "provider.loginExplanation", + "OpenRouter login supplies a user-controlled API key. No refresh token is issued. Sign in on the machine running DeepCode.", + )} + +
+ + {pending(flow) && flow ? ( + + ) : null} + {connection.accountId ? ( + + ) : null} + + {t("provider.manageKeys", "Manage remote keys")} + +
+ {flow?.authorizationUrl ? ( + + {t("provider.openLogin", "Open sign-in page")} + + ) : null} + {flow ? ( +

{t(`provider.login.${flow.status}`, flow.status)}

+ ) : null} + {error || flow?.error ?

{error || flow?.error}

: null} +
+ ); +} diff --git a/desktop/src/features/settings/SettingsDialog.tsx b/desktop/src/features/settings/SettingsDialog.tsx index 766a28c52..92d2c5033 100644 --- a/desktop/src/features/settings/SettingsDialog.tsx +++ b/desktop/src/features/settings/SettingsDialog.tsx @@ -17,7 +17,7 @@ import type { SettingsSnapshot, } from "../../generated/app-server"; import { useEscapeLayer } from "../../app/escapeLayer"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; import { SETTINGS_SECTIONS, type SettingsSectionId, @@ -25,7 +25,7 @@ import { import styles from "./SettingsDialog.module.css"; interface SettingsDialogProps { - runtime: DesktopRuntime; + runtime: ClientRuntime; project: Project | null; settings: SettingsSnapshot | null; busy: boolean; @@ -135,7 +135,7 @@ export function SettingsDialog({ onClick={() => void openConfigFile()} > - {t("settings.openConfig", "Open configuration file")} + {runtime.host?.nativeOpen === false ? "Copy configuration path" : t("settings.openConfig", "Open configuration file")} + +

+ {t( + "service.detach", + "Closing Desktop disconnects this window. Tasks and scheduled work continue in the shared service.", + )} +

+ {activity && ( +

+ {t( + "service.activity", + "{{phase}} · {{active}} active tasks · {{queued}} queued · {{terminals}} terminals", + { + phase: activity.phase, + active: activity.activeTurns, + queued: activity.queuedTurns, + terminals: activity.terminals, + }, + )} +

+ )} + {error && ( +

+ {error} +

+ )} +
+ ); +} diff --git a/desktop/src/features/settings/sections/UpdatesCard.tsx b/desktop/src/features/settings/sections/UpdatesCard.tsx index 0aad2db6f..b2eb1bf5e 100644 --- a/desktop/src/features/settings/sections/UpdatesCard.tsx +++ b/desktop/src/features/settings/sections/UpdatesCard.tsx @@ -5,7 +5,7 @@ import { useState } from "react"; import { useTranslation } from "react-i18next"; import type { - DesktopRuntime, + ClientRuntime, DesktopUpdateInfo, DesktopUpdateProgress, } from "../../../rpc/contracts"; @@ -13,7 +13,7 @@ import styles from "../../management/ManagementWorkspace.module.css"; type UpdateState = "idle" | "checking" | "current" | "available" | "installing"; -export function UpdatesCard({ runtime }: { runtime: DesktopRuntime }) { +export function UpdatesCard({ runtime }: { runtime: ClientRuntime }) { const { t } = useTranslation(); const [updateInfo, setUpdateInfo] = useState(null); const [updateState, setUpdateState] = useState("idle"); @@ -21,6 +21,10 @@ export function UpdatesCard({ runtime }: { runtime: DesktopRuntime }) { useState(null); const [updateError, setUpdateError] = useState(null); + if (runtime.host?.updates === false) return
+

Service updates

Update DeepCode on the service machine, then open a fresh link with deepcode web. Reload this page after updating.

+
; + const checkForUpdate = async () => { setUpdateState("checking"); setUpdateInfo(null); diff --git a/desktop/src/features/settings/settingsSections.tsx b/desktop/src/features/settings/settingsSections.tsx index 3ed4f58ad..68fb9113e 100644 --- a/desktop/src/features/settings/settingsSections.tsx +++ b/desktop/src/features/settings/settingsSections.tsx @@ -15,7 +15,7 @@ import type { Project, SettingsSnapshot, } from "../../generated/app-server"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; import { GeneralSection } from "./sections/GeneralSection"; import { ModelsSection } from "./sections/ModelsSection"; import { PluginsSection } from "./sections/PluginsSection"; @@ -25,7 +25,7 @@ export type SettingsSectionId = "general" | "models" | "plugins" | "agent-preset /** Everything a section may need; each uses the subset it cares about. */ export interface SettingsSectionProps { - runtime: DesktopRuntime; + runtime: ClientRuntime; project: Project | null; settings: SettingsSnapshot | null; busy: boolean; diff --git a/desktop/src/features/settings/useConnectionCatalog.ts b/desktop/src/features/settings/useConnectionCatalog.ts index 3eaa7da8a..822b61dd3 100644 --- a/desktop/src/features/settings/useConnectionCatalog.ts +++ b/desktop/src/features/settings/useConnectionCatalog.ts @@ -6,18 +6,29 @@ import type { ConnectionCatalogResult, ModelCatalogResult, ProviderTestResult, + ProviderTestParams, + ProviderLoginFlow, + ProviderLogoutResult, ProviderUpsertParams, } from "../../generated/app-server"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; export interface ConnectionCatalogController { catalog: ConnectionCatalogResult | null; loading: boolean; error: string | null; reload(): Promise; + login(connectionId: string): Promise; + pollLogin(flowId: string): Promise; + cancelLogin(flowId: string): Promise; + logout(connectionId: string): Promise; upsert(connection: ProviderUpsertParams["connection"]): Promise; remove(connectionId: string): Promise; - test(connectionId: string, model?: string): Promise; + test( + connectionId: string, + model?: string, + options?: Pick, + ): Promise; models(connectionId: string, refresh?: boolean): Promise; /** Probe an endpoint AS SHOWN in an editor form; nothing is stored. */ discover( @@ -26,7 +37,7 @@ export interface ConnectionCatalogController { } export function useConnectionCatalog( - runtime: DesktopRuntime, + runtime: ClientRuntime, projectId: string | null, ): ConnectionCatalogController { const [state, setState] = useState<{ @@ -138,10 +149,15 @@ export function useConnectionCatalog( ); const test = useCallback( - async (connectionId: string, model?: string) => { + async ( + connectionId: string, + model?: string, + options?: Pick, + ) => { try { const result = await runtime.request("provider/test", { connectionId, + ...options, ...(projectId ? { projectId } : {}), ...(model ? { model } : {}), }); @@ -174,8 +190,33 @@ export function useConnectionCatalog( [projectId, runtime], ); + const login = useCallback( + (connectionId: string) => + runtime.request("provider/login/start", { + connectionId, + openBrowser: true, + }), + [runtime], + ); + const pollLogin = useCallback( + (flowId: string) => runtime.request("provider/login/poll", { flowId }), + [runtime], + ); + const cancelLogin = useCallback( + (flowId: string) => runtime.request("provider/login/cancel", { flowId }), + [runtime], + ); + const logout = useCallback( + (connectionId: string) => + runtime.request("provider/logout", { connectionId }), + [runtime], + ); const currentProject = state.projectId === projectId; return { + login, + pollLogin, + cancelLogin, + logout, catalog: currentProject ? state.catalog : null, loading: currentProject ? state.loading : true, error: currentProject ? state.error : null, diff --git a/desktop/src/features/settings/useDiagnostics.ts b/desktop/src/features/settings/useDiagnostics.ts index 6471ee250..4db1ec286 100644 --- a/desktop/src/features/settings/useDiagnostics.ts +++ b/desktop/src/features/settings/useDiagnostics.ts @@ -1,10 +1,10 @@ import { useCallback, useEffect, useRef, useState } from "react"; import type { DiagnosticsSnapshot } from "../../generated/app-server"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; export function useDiagnostics( - runtime: DesktopRuntime, + runtime: ClientRuntime, projectId: string | null, ) { const [diagnostics, setDiagnostics] = useState(null); diff --git a/desktop/src/features/skills/useSkillCatalog.test.tsx b/desktop/src/features/skills/useSkillCatalog.test.tsx index 9a5563053..442fa49e6 100644 --- a/desktop/src/features/skills/useSkillCatalog.test.tsx +++ b/desktop/src/features/skills/useSkillCatalog.test.tsx @@ -9,7 +9,7 @@ import type { } from "../../generated/app-server"; import type { AnyRpcNotification, - DesktopRuntime, + ClientRuntime, RpcMethod, } from "../../rpc/contracts"; import { useSkillCatalog } from "./useSkillCatalog"; @@ -96,7 +96,7 @@ function Consumer({ runtime, label, }: { - runtime: DesktopRuntime; + runtime: ClientRuntime; label: string; }) { const state = useSkillCatalog(runtime, projectId); @@ -105,7 +105,7 @@ function Consumer({ test("shares one catalog request across Desktop consumers", async () => { const backend = new CatalogRuntime(); - const runtime = backend as unknown as DesktopRuntime; + const runtime = backend as unknown as ClientRuntime; render( <> @@ -120,7 +120,7 @@ test("shares one catalog request across Desktop consumers", async () => { test("refreshes every consumer after skills.changed", async () => { const backend = new CatalogRuntime(); - const runtime = backend as unknown as DesktopRuntime; + const runtime = backend as unknown as ClientRuntime; const view = render( <> diff --git a/desktop/src/features/skills/useSkillCatalog.ts b/desktop/src/features/skills/useSkillCatalog.ts index 7c0bdeafe..547cd8aca 100644 --- a/desktop/src/features/skills/useSkillCatalog.ts +++ b/desktop/src/features/skills/useSkillCatalog.ts @@ -6,7 +6,7 @@ import type { } from "../../generated/app-server"; import type { AnyRpcNotification, - DesktopRuntime, + ClientRuntime, } from "../../rpc/contracts"; interface SkillCatalogState extends SkillCatalogResult { @@ -33,7 +33,7 @@ const emptyCatalog: SkillCatalogState = { error: null, }; -const stores = new WeakMap(); +const stores = new WeakMap(); function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); @@ -49,7 +49,7 @@ class SkillCatalogStore { private notificationGeneration = 0; private unsubscribeNotifications: (() => void) | null = null; - constructor(private readonly runtime: DesktopRuntime) {} + constructor(private readonly runtime: ClientRuntime) {} snapshot(projectId: string | null): SkillCatalogState { return projectId ? this.entry(projectId).state : emptyCatalog; @@ -179,7 +179,7 @@ class SkillCatalogStore { } } -function storeFor(runtime: DesktopRuntime): SkillCatalogStore { +function storeFor(runtime: ClientRuntime): SkillCatalogStore { let store = stores.get(runtime); if (!store) { store = new SkillCatalogStore(runtime); @@ -189,7 +189,7 @@ function storeFor(runtime: DesktopRuntime): SkillCatalogStore { } export function useSkillCatalog( - runtime: DesktopRuntime, + runtime: ClientRuntime, projectId: string | null, ) { const store = useMemo(() => storeFor(runtime), [runtime]); diff --git a/desktop/src/features/workbench/TerminalPanel.test.tsx b/desktop/src/features/workbench/TerminalPanel.test.tsx new file mode 100644 index 000000000..f260fc8b5 --- /dev/null +++ b/desktop/src/features/workbench/TerminalPanel.test.tsx @@ -0,0 +1,247 @@ +import { + act, + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { + AnyRpcNotification, + ClientRuntime, + SidecarStatus, +} from "../../rpc/contracts"; +import { TerminalPanel } from "./TerminalPanel"; + +const state = vi.hoisted(() => ({ + instances: [] as Array<{ + text: string; + input: (data: string) => void; + disposed: boolean; + }>, +})); +vi.mock("@xterm/xterm", () => ({ + Terminal: class { + text = ""; + cols = 80; + rows = 24; + disposed = false; + input = (data: string) => { + void data; + }; + constructor() { + state.instances.push(this); + } + loadAddon() {} + open() {} + focus() {} + clear() { + this.text = ""; + } + reset() { + this.text = ""; + } + write(data: string, done?: () => void) { + this.text += data; + done?.(); + } + onData(callback: (data: string) => void) { + this.input = callback; + return { dispose() {} }; + } + dispose() { + this.disposed = true; + } + }, +})); +vi.mock("@xterm/addon-fit", () => ({ + FitAddon: class { + fit() {} + }, +})); + +function harness(initial = true) { + const info = { + terminalId: "term_test", + threadId: "thread-1", + pid: 42, + columns: 80, + rows: 24, + workspacePath: "/tmp", + }; + const backend = { + exists: initial, + text: "startup\r\n", + exited: false, + phase: "ready" as SidecarStatus["phase"], + }; + const notifications = new Set<(value: AnyRpcNotification) => void>(); + const statuses = new Set<(value: SidecarStatus) => void>(); + const status = () => + ({ + phase: backend.phase, + serverInfo: { + capabilities: { methods: ["terminal/list", "terminal/read"] }, + }, + }) as SidecarStatus; + const request = vi.fn( + async (method: string, params: { offset?: number; data?: string }) => { + if (method === "terminal/list") + return { + terminals: backend.exists + ? [ + { + terminal: info, + exited: backend.exited, + exitCode: backend.exited ? 0 : null, + }, + ] + : [], + }; + if (method === "terminal/create") { + backend.exists = true; + return { terminal: info }; + } + if (method === "terminal/close") { + backend.exited = true; + return { accepted: true }; + } + if (method === "terminal/write") + return { written: params.data?.length ?? 0 }; + if (method === "terminal/read") { + const offset = params.offset ?? 0; + return { + terminalId: info.terminalId, + threadId: info.threadId, + data: backend.text.slice(offset), + offset, + nextOffset: backend.text.length, + availableFrom: 0, + headOffset: backend.text.length, + hasMore: false, + truncated: false, + exited: backend.exited, + exitCode: backend.exited ? 0 : null, + }; + } + throw new Error(method); + }, + ); + const runtime = { + request, + status: async () => status(), + onNotification: async (listener: (value: AnyRpcNotification) => void) => { + notifications.add(listener); + return () => notifications.delete(listener); + }, + onStatus: async (listener: (value: SidecarStatus) => void) => { + statuses.add(listener); + return () => statuses.delete(listener); + }, + } as unknown as ClientRuntime; + const props = { + runtime, + threadId: info.threadId, + enabled: true, + active: true, + }; + return { props, request, backend, statuses, notifications, status }; +} + +beforeEach(() => { + state.instances.length = 0; + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + disconnect() {} + }, + ); +}); +afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); +}); + +describe("TerminalPanel recovery", () => { + it("reattaches an existing terminal and detaches without terminating it", async () => { + const test = harness(); + const view = render(); + await screen.findByText("PID 42"); + await waitFor(() => expect(state.instances[0].text).toBe("startup\r\n")); + view.unmount(); + expect(state.instances[0].disposed).toBe(true); + render(); + await waitFor(() => expect(state.instances[1]?.text).toBe("startup\r\n")); + expect( + test.request.mock.calls.some( + ([method]) => + method === "terminal/create" || method === "terminal/close", + ), + ).toBe(false); + }); + + it("recovers startup and reconnect output without resending terminal input", async () => { + const test = harness(false); + const view = render(); + await waitFor(() => + expect( + (screen.getByText("Start terminal") as HTMLButtonElement).disabled, + ).toBe(false), + ); + fireEvent.click(screen.getByText("Start terminal")); + await waitFor(() => expect(state.instances[0].text).toBe("startup\r\n")); + act(() => state.instances[0].input("echo test\r")); + test.backend.phase = "stopped"; + act(() => test.statuses.forEach((receive) => receive(test.status()))); + act(() => state.instances[0].input("must not send\r")); + test.backend.text += "after reconnect\r\n"; + test.backend.phase = "ready"; + act(() => test.statuses.forEach((receive) => receive(test.status()))); + await waitFor(() => + expect(state.instances[0].text).toBe(test.backend.text), + ); + expect( + test.request.mock.calls.filter(([method]) => method === "terminal/write"), + ).toHaveLength(1); + expect( + test.request.mock.calls.filter( + ([method]) => method === "terminal/create", + ), + ).toHaveLength(1); + fireEvent.click(screen.getByText("Close")); + await screen.findByText("Exited"); + view.unmount(); + expect( + test.request.mock.calls.filter(([method]) => method === "terminal/close"), + ).toHaveLength(1); + }); + + it("recovers a dropped final output and exit on overflow warning", async () => { + const test = harness(); + render(); + await waitFor(() => + expect(state.instances[0]?.text).toBe(test.backend.text), + ); + test.backend.text += "final output\r\n"; + test.backend.exited = true; + const warning: AnyRpcNotification = { + jsonrpc: "2.0", + method: "server.warning", + params: { + replayRequired: true, + code: "NOTIFICATION_QUEUE_OVERFLOW", + dropped: 1, + }, + }; + act(() => test.notifications.forEach((receive) => receive(warning))); + await screen.findByText("Exited"); + expect(state.instances[0].text).toBe( + test.backend.text + "\r\n[process exited 0]\r\n", + ); + act(() => test.notifications.forEach((receive) => receive(warning))); + await act(async () => {}); + expect(state.instances[0].text.match(/process exited/g)).toHaveLength(1); + }); +}); diff --git a/desktop/src/features/workbench/TerminalPanel.tsx b/desktop/src/features/workbench/TerminalPanel.tsx index ef66acb87..0a21b93b1 100644 --- a/desktop/src/features/workbench/TerminalPanel.tsx +++ b/desktop/src/features/workbench/TerminalPanel.tsx @@ -1,13 +1,14 @@ import type { FitAddon } from "@xterm/addon-fit"; import type { Terminal } from "@xterm/xterm"; -import { useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import type { TerminalInfo } from "../../generated/app-server"; -import type { DesktopRuntime } from "../../rpc/contracts"; +import type { ClientRuntime } from "../../rpc/contracts"; import styles from "./TerminalPanel.module.css"; +import { TerminalOutputReader } from "./terminalOutputReader"; interface TerminalPanelProps { - runtime: DesktopRuntime; + runtime: ClientRuntime; threadId: string | null; enabled: boolean; active: boolean; @@ -28,21 +29,76 @@ export function TerminalPanel({ const inputDisposable = useRef<{ dispose(): void } | null>(null); const initializing = useRef(false); const disposed = useRef(false); + const recovery = useRef(null); + const recoverable = useRef(false); + const generation = useRef(0); const [terminalInfo, setTerminalInfo] = useState(null); const [ended, setEnded] = useState(false); const [error, setError] = useState(null); const [rendererReady, setRendererReady] = useState(false); + const [connected, updateConnected] = useState(false); + const connectionReady = useRef(false); + const setConnected = useCallback((ready: boolean) => { + connectionReady.current = ready; + updateConnected(ready); + }, []); + const [busy, setBusy] = useState(false); + + const reportError = useCallback((cause: unknown) => { + setError(cause instanceof Error ? cause.message : String(cause)); + }, []); + + const finish = useCallback((code: number | null) => { + terminal.current?.write(`\r\n[process exited ${code ?? "unknown"}]\r\n`); + setEnded(true); + info.current = null; + setTerminalInfo(null); + }, []); + + const attach = useCallback( + async (current: TerminalInfo) => { + recovery.current?.stop(); + recovery.current = null; + terminal.current?.reset(); + info.current = current; + setTerminalInfo(current); + setEnded(false); + if (!recoverable.current) return; + const reader = new TerminalOutputReader( + runtime, + current, + (data) => + new Promise((resolve) => { + if (terminal.current) terminal.current.write(data, resolve); + else resolve(); + }), + () => { + terminal.current?.reset(); + terminal.current?.write( + "[Earlier terminal output is no longer available]\r\n", + ); + }, + finish, + reportError, + ); + recovery.current = reader; + await reader.recover(); + }, + [finish, reportError, runtime], + ); useEffect(() => { - if (!active || !host.current || terminal.current || initializing.current) return; + if (!active || !host.current || terminal.current || initializing.current) + return; initializing.current = true; const hostElement = host.current; void (async () => { - const [{ Terminal: XTerm }, { FitAddon: XTermFitAddon }] = await Promise.all([ - import("@xterm/xterm"), - import("@xterm/addon-fit"), - import("@xterm/xterm/css/xterm.css"), - ]); + const [{ Terminal: XTerm }, { FitAddon: XTermFitAddon }] = + await Promise.all([ + import("@xterm/xterm"), + import("@xterm/addon-fit"), + import("@xterm/xterm/css/xterm.css"), + ]); if (disposed.current) return; const instance = new XTerm({ cursorBlink: true, @@ -70,23 +126,28 @@ export function TerminalPanel({ fit.current = fitAddon; inputDisposable.current = instance.onData((data) => { const current = info.current; - if (current) { - void runtime.request("terminal/write", { - threadId: current.threadId, - terminalId: current.terminalId, - data, - }); + if (current && connectionReady.current) { + void runtime + .request("terminal/write", { + threadId: current.threadId, + terminalId: current.terminalId, + data, + }) + .catch(reportError); } }); observer.current = new ResizeObserver(() => { - if (!activeRef.current || !info.current) return; + if (!activeRef.current || !info.current || !connectionReady.current) + return; fitAddon.fit(); - void runtime.request("terminal/resize", { - threadId: info.current.threadId, - terminalId: info.current.terminalId, - columns: Math.max(20, instance.cols), - rows: Math.max(5, instance.rows), - }); + void runtime + .request("terminal/resize", { + threadId: info.current.threadId, + terminalId: info.current.terminalId, + columns: Math.max(20, instance.cols), + rows: Math.max(5, instance.rows), + }) + .catch(reportError); }); observer.current.observe(hostElement); setRendererReady(true); @@ -94,12 +155,11 @@ export function TerminalPanel({ initializing.current = false; setError(cause instanceof Error ? cause.message : String(cause)); }); - }, [active, runtime]); + }, [active, reportError, runtime]); - useEffect( - () => { - disposed.current = false; - return () => { + useEffect(() => { + disposed.current = false; + return () => { disposed.current = true; observer.current?.disconnect(); inputDisposable.current?.dispose(); @@ -108,10 +168,8 @@ export function TerminalPanel({ inputDisposable.current = null; terminal.current = null; fit.current = null; - }; - }, - [], - ); + }; + }, []); useEffect(() => { activeRef.current = active; @@ -119,57 +177,135 @@ export function TerminalPanel({ }, [active]); useEffect(() => { - let disposed = false; - let cleanup: () => void = () => undefined; - void runtime.onNotification((notification) => { - if (disposed || !threadId) return; - if (notification.method === "terminal.output") { - const current = info.current; - if ( - current && - notification.params.threadId === threadId && - notification.params.terminalId === current.terminalId - ) { - terminal.current?.write(notification.params.data); - } + if (!rendererReady || !threadId) return; + terminal.current?.reset(); + let stopped = false; + let refreshVersion = 0; + const cleanups: Array<() => void> = []; + const currentGeneration = ++generation.current; + + const refresh = async () => { + const version = ++refreshVersion; + const status = await runtime.status(); + if (stopped || version !== refreshVersion) return; + if (status.phase !== "ready") { + setConnected(false); + return; } - if ( - notification.method === "terminal.exit" && - notification.params.terminalId === info.current?.terminalId - ) { - terminal.current?.write( - `\r\n\x1b[90m[process exited ${notification.params.exitCode ?? "unknown"}]\x1b[0m\r\n`, - ); - setEnded(true); - info.current = null; - setTerminalInfo(null); + const methods = status.serverInfo?.capabilities.methods ?? []; + recoverable.current = + methods.includes("terminal/read") && methods.includes("terminal/list"); + if (recoverable.current) { + const { terminals } = await runtime.request("terminal/list", { + threadId, + }); + if (stopped || version !== refreshVersion) return; + const previous = + recovery.current?.terminal.terminalId ?? info.current?.terminalId; + const chosen = + terminals.find((entry) => entry.terminal.terminalId === previous) ?? + terminals.filter((entry) => !entry.exited).at(-1) ?? + terminals.at(-1); + if (chosen) { + if ( + recovery.current?.terminal.terminalId === chosen.terminal.terminalId + ) { + await recovery.current.recover(); + } else { + await attach(chosen.terminal); + } + } else if (previous) { + recovery.current?.stop(); + recovery.current = null; + info.current = null; + setTerminalInfo(null); + setEnded(true); + terminal.current?.write( + "\r\n[Terminal session is no longer available]\r\n", + ); + } } - }).then((unsubscribe) => { - if (disposed) unsubscribe(); - else cleanup = unsubscribe; + if (!stopped && version === refreshVersion) setConnected(true); + }; + + const register = async (subscription: Promise<() => void>) => { + const cleanup = await subscription; + if (stopped) cleanup(); + else cleanups.push(cleanup); + }; + void (async () => { + await register( + runtime.onNotification((notification) => { + if (stopped) return; + const current = recovery.current?.terminal ?? info.current; + if (notification.method === "server.warning") { + if (notification.params.replayRequired) + void recovery.current?.recover().catch(reportError); + return; + } + if ( + !current || + (notification.method !== "terminal.output" && + notification.method !== "terminal.exit") || + notification.params.threadId !== threadId || + notification.params.terminalId !== current.terminalId + ) + return; + if (notification.method === "terminal.output") { + if (recovery.current) + recovery.current.receive(notification.params.nextOffset ?? 0); + else terminal.current?.write(notification.params.data); + } else if (recovery.current) { + void recovery.current.recover().catch(reportError); + } else { + finish(notification.params.exitCode); + } + }), + ); + if (stopped) return; + setEnded(false); + setError(null); + await register( + runtime.onStatus((status) => { + if (stopped) return; + if (status.phase === "ready") void refresh().catch(reportError); + else { + refreshVersion++; + setConnected(false); + } + }), + ); + if (!stopped) await refresh(); + })().catch((cause) => { + if (!stopped) reportError(cause); }); + return () => { - disposed = true; - cleanup(); + stopped = true; + generation.current = currentGeneration + 1; + for (const cleanup of cleanups) cleanup(); + recovery.current?.stop(); + recovery.current = null; + info.current = null; + setTerminalInfo(null); + setConnected(false); + setBusy(false); + // Detaching a view does not terminate the PTY. Only Close does that. }; - }, [runtime, threadId]); - - useEffect( - () => () => { - const current = info.current; - if (current) { - void runtime.request("terminal/close", { - threadId: current.threadId, - terminalId: current.terminalId, - }); - info.current = null; - } - }, - [runtime, threadId], - ); + }, [ + attach, + finish, + rendererReady, + reportError, + runtime, + setConnected, + threadId, + ]); const start = async () => { - if (!threadId || !enabled) return; + if (!threadId || !enabled || busy || !connected) return; + const currentGeneration = generation.current; + setBusy(true); setError(null); setEnded(false); terminal.current?.clear(); @@ -180,40 +316,59 @@ export function TerminalPanel({ columns: Math.max(20, terminal.current?.cols ?? 80), rows: Math.max(5, terminal.current?.rows ?? 24), }); - info.current = result.terminal; - setTerminalInfo(result.terminal); + if (currentGeneration !== generation.current) return; + await attach(result.terminal); terminal.current?.focus(); } catch (cause) { - setError(cause instanceof Error ? cause.message : String(cause)); + if (currentGeneration === generation.current) reportError(cause); + } finally { + if (currentGeneration === generation.current) setBusy(false); } }; const close = async () => { const current = info.current; - if (!current) return; - await runtime.request("terminal/close", { - threadId: current.threadId, - terminalId: current.terminalId, - }); - info.current = null; - setTerminalInfo(null); - setEnded(true); - terminal.current?.write("\r\n\x1b[90m[terminal closed]\x1b[0m\r\n"); + if (!current || busy || !connected) return; + const currentGeneration = generation.current; + setBusy(true); + try { + await runtime.request("terminal/close", { + threadId: current.threadId, + terminalId: current.terminalId, + }); + if (currentGeneration !== generation.current) return; + if (recovery.current) await recovery.current.recover(); + else finish(null); + } catch (cause) { + if (currentGeneration === generation.current) reportError(cause); + } finally { + if (currentGeneration === generation.current) setBusy(false); + } }; return (
- {terminalInfo ? `PID ${terminalInfo.pid}` : ended ? "Exited" : "No session"} + + {terminalInfo + ? `PID ${terminalInfo.pid}` + : ended + ? "Exited" + : "No session"} + {terminalInfo ? ( - ) : ( diff --git a/desktop/src/features/workbench/terminalOutputReader.test.ts b/desktop/src/features/workbench/terminalOutputReader.test.ts new file mode 100644 index 000000000..21b7801be --- /dev/null +++ b/desktop/src/features/workbench/terminalOutputReader.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it, vi } from "vitest"; +import type { + MethodParams, + MethodResults, + TerminalInfo, +} from "../../generated/app-server"; +import type { RpcMethod, RpcTransport } from "../../rpc/contracts"; +import { TerminalOutputReader } from "./terminalOutputReader"; + +const terminal: TerminalInfo = { + terminalId: "term_test", + threadId: "thread-1", + pid: 1, + rows: 24, + columns: 80, + workspacePath: "/tmp", +}; +type Page = MethodResults["terminal/read"]; +function page(offset: number, data: string, other: Partial = {}): Page { + const next = offset + new TextEncoder().encode(data).length; + return { + threadId: terminal.threadId, + terminalId: terminal.terminalId, + offset, + data, + nextOffset: next, + headOffset: next, + availableFrom: 0, + hasMore: false, + truncated: false, + exited: false, + exitCode: null, + ...other, + }; +} +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((yes) => { + resolve = yes; + }); + return { resolve, promise }; +} +class Runtime implements RpcTransport { + calls: MethodParams["terminal/read"][] = []; + constructor( + private read: (params: MethodParams["terminal/read"]) => Promise, + ) {} + async request( + method: M, + params: MethodParams[M], + ): Promise { + // Any create/write/close during recovery fails the test. + expect(method).toBe("terminal/read"); + const input = params as MethodParams["terminal/read"]; + this.calls.push(input); + return this.read(input) as Promise; + } +} +function reader( + runtime: Runtime, + write = vi.fn(async (data: string): Promise => { + void data; + }), +) { + const truncated = vi.fn(), + exit = vi.fn(), + error = vi.fn(); + return { + stream: new TerminalOutputReader( + runtime, + terminal, + write, + truncated, + exit, + error, + ), + write, + truncated, + exit, + error, + }; +} + +describe("terminal output recovery", () => { + it("uses UTF-8 byte cursors and ignores overlapping notifications", async () => { + const first = deferred(); + const runtime = new Runtime(async ({ offset }) => + offset === 0 + ? first.promise + : page(7, "!", { exited: true, exitCode: 0 }), + ); + const test = reader(runtime); + const done = test.stream.recover(); + test.stream.receive(8); + first.resolve(page(0, "汉🙂")); + await done; + test.stream.receive(7); + test.stream.receive(8); + expect(test.write.mock.calls.map(([data]) => data).join("")).toBe("汉🙂!"); + expect(runtime.calls.map((call) => call.offset)).toEqual([0, 7]); + expect(test.exit).toHaveBeenCalledExactlyOnceWith(0); + }); + + it("finishes captured windows before catching up, then reports exit after the final byte", async () => { + const runtime = new Runtime(async ({ offset }) => + offset === 0 + ? page(0, "a", { headOffset: 4, hasMore: true }) + : offset === 1 + ? page(1, "bcd", { headOffset: 8, exited: true, exitCode: 0 }) + : page(4, "efgh", { exited: true, exitCode: 0 }), + ); + const test = reader(runtime); + await test.stream.recover(); + expect(runtime.calls.map((call) => call.through)).toEqual([ + undefined, + 4, + undefined, + ]); + expect(test.write.mock.calls.map(([data]) => data).join("")).toBe( + "abcdefgh", + ); + expect(test.exit).toHaveBeenCalledOnce(); + expect(test.exit.mock.invocationCallOrder[0]).toBeGreaterThan( + test.write.mock.invocationCallOrder.at(-1)!, + ); + }); + + it("marks an evicted window once and resumes from the retained byte boundary", async () => { + const runtime = new Runtime(async ({ offset }) => + offset === 0 + ? page(100, "汉", { + truncated: true, + availableFrom: 100, + exited: true, + exitCode: 0, + }) + : page(103, "", { availableFrom: 100, exited: true, exitCode: 0 }), + ); + const test = reader(runtime); + await test.stream.recover(); + expect(test.truncated).toHaveBeenCalledOnce(); + expect(test.write).toHaveBeenCalledExactlyOnceWith("汉"); + expect(test.exit).toHaveBeenCalledOnce(); + expect(runtime.calls.map((call) => call.offset)).toEqual([0, 103]); + }); + + it("coalesces warnings and checks for a lost exit notification", async () => { + const pending = deferred(); + const runtime = new Runtime(async ({ offset }) => + offset === 0 + ? pending.promise + : page(1, "", { exited: true, exitCode: 0 }), + ); + const test = reader(runtime); + const done = test.stream.recover(); + for (let index = 0; index < 50; index++) + expect(test.stream.recover()).toBe(done); + expect(runtime.calls).toHaveLength(1); + pending.resolve(page(0, "x")); + await done; + expect(runtime.calls).toHaveLength(2); + expect(test.exit).toHaveBeenCalledOnce(); + }); + + it("waits for the renderer and cancels a blocked write on detach", async () => { + const render = deferred(); + const runtime = new Runtime(async () => + page(0, "a", { hasMore: true, headOffset: 2 }), + ); + const write = vi.fn(async (text: string) => { + void text; + return render.promise; + }); + const test = reader(runtime, write); + const done = test.stream.recover(); + await vi.waitFor(() => expect(write).toHaveBeenCalledOnce()); + expect(runtime.calls).toHaveLength(1); + test.stream.stop(); + await done; + expect(runtime.calls).toHaveLength(1); + render.resolve(); + }); + + it("retries only the unread output after a read failure", async () => { + let calls = 0; + const runtime = new Runtime(async () => { + calls++; + if (calls === 1) return page(0, "a", { headOffset: 2, hasMore: true }); + if (calls === 2) throw new Error("disconnected"); + return page(1, "b"); + }); + const test = reader(runtime); + await expect(test.stream.recover()).rejects.toThrow("disconnected"); + await test.stream.recover(); + expect(test.write.mock.calls.map(([data]) => data)).toEqual(["a", "b"]); + expect(runtime.calls.map((call) => call.offset)).toEqual([0, 1, 1]); + }); + + it("reduces pages to fit a small transport frame", async () => { + const runtime = new Runtime(async ({ limit = 0 }) => { + if (limit > 128) throw { code: "RESPONSE_TOO_LARGE" }; + return page(0, "🙂"); + }); + const test = reader(runtime); + await test.stream.recover(); + expect(test.write).toHaveBeenCalledExactlyOnceWith("🙂"); + expect(runtime.calls.at(-1)?.limit).toBe(128); + }); + + it.each([ + page(0, "", { headOffset: 1 }), + page(0, "a", { nextOffset: 3, headOffset: 3 }), + page(0, "", { hasMore: true }), + ])("rejects broken cursors instead of spinning", async (broken) => { + const runtime = new Runtime(async () => broken); + await expect(reader(runtime).stream.recover()).rejects.toThrow( + "Terminal output", + ); + expect(runtime.calls).toHaveLength(1); + }); +}); diff --git a/desktop/src/features/workbench/terminalOutputReader.ts b/desktop/src/features/workbench/terminalOutputReader.ts new file mode 100644 index 000000000..a7f60912d --- /dev/null +++ b/desktop/src/features/workbench/terminalOutputReader.ts @@ -0,0 +1,137 @@ +import type { TerminalInfo, MethodResults } from "../../generated/app-server"; +import type { RpcTransport } from "../../rpc/contracts"; + +/** Output-only recovery. Input writes are deliberately outside this reader. */ +export class TerminalOutputReader { + private offset = 0; + private observedHead = 0; + private revision = 0; + private active = true; + private ended = false; + private pending: Promise | null = null; + private cancelWrite: (() => void) | null = null; + + constructor( + private readonly runtime: RpcTransport, + readonly terminal: TerminalInfo, + private readonly write: (data: string) => Promise, + private readonly truncate: () => void, + private readonly exit: (code: number | null) => void, + private readonly onError: (error: unknown) => void, + ) {} + + stop(): void { + this.active = false; + this.cancelWrite?.(); + } + + receive(nextOffset: number): void { + if (!this.active || nextOffset <= this.offset) return; + this.observedHead = Math.max(this.observedHead, nextOffset); + if (!this.pending) void this.recover().catch(this.onError); + } + + recover(): Promise { + if (!this.active) return Promise.resolve(); + this.revision++; + if (!this.pending) this.pending = this.read(); + return this.pending; + } + + private async read(): Promise { + let limit = 16 * 1024; + try { + while (this.active) { + const revision = this.revision; + const before = this.offset; + let through: number | undefined; + let page: MethodResults["terminal/read"]; + for (;;) { + try { + page = await this.runtime.request("terminal/read", { + threadId: this.terminal.threadId, + terminalId: this.terminal.terminalId, + offset: this.offset, + limit, + ...(through === undefined ? {} : { through }), + }); + } catch (error) { + if (!this.active) return; + if ( + (error as { code?: string } | null)?.code === + "RESPONSE_TOO_LARGE" && + limit > 4 + ) { + limit = Math.max(4, Math.floor(limit / 2)); + continue; + } + throw error; + } + if (!this.active) return; + const bytes = new TextEncoder().encode(page.data).length; + if ( + page.terminalId !== this.terminal.terminalId || + page.threadId !== this.terminal.threadId || + page.offset < this.offset || + (!page.truncated && page.offset !== this.offset) || + page.nextOffset !== page.offset + bytes || + page.headOffset < page.nextOffset || + (page.hasMore && bytes === 0) + ) { + throw new Error( + "Terminal output cursor is inconsistent; reopen the terminal view", + ); + } + if (page.truncated) this.truncate(); + if (page.data) await this.render(page.data); + if (!this.active) return; + this.offset = page.nextOffset; + this.observedHead = Math.max(this.observedHead, page.headOffset); + through ??= page.headOffset; + // Eviction can pass a previously captured cutoff; start a new window. + if (page.truncated) through = undefined; + if (page.truncated && !page.hasMore) { + this.revision++; + break; + } + if (!page.hasMore) break; + } + if (this.observedHead > this.offset && this.offset === before) { + throw new Error("Terminal output recovery made no progress"); + } + if ( + page.exited && + this.offset === page.headOffset && + !page.truncated && + !this.ended + ) { + this.ended = true; + this.exit(page.exitCode); + } + if (revision === this.revision && this.observedHead <= this.offset) + return; + } + } finally { + this.pending = null; + } + } + + private render(data: string): Promise { + // One cancellable renderer write, without accumulating handlers on a + // never-settled cancellation promise during a long-lived terminal session. + return new Promise((resolve, reject) => { + let settled = false; + const finish = (error?: unknown) => { + if (settled) return; + settled = true; + this.cancelWrite = null; + if (error === undefined) resolve(); + else reject(error); + }; + this.cancelWrite = () => finish(); + Promise.resolve() + .then(() => (this.active ? this.write(data) : undefined)) + .then(() => finish(), finish); + }); + } +} diff --git a/desktop/src/features/workbench/useCodeWorkbench.test.tsx b/desktop/src/features/workbench/useCodeWorkbench.test.tsx index 69043b6e7..c1c113e49 100644 --- a/desktop/src/features/workbench/useCodeWorkbench.test.tsx +++ b/desktop/src/features/workbench/useCodeWorkbench.test.tsx @@ -12,7 +12,7 @@ import type { } from "../../generated/app-server"; import type { AnyRpcNotification, - DesktopRuntime, + ClientRuntime, RpcMethod, SidecarStatus, } from "../../rpc/contracts"; @@ -100,7 +100,7 @@ const testItem: Item = { updatedAt: "2026-07-16T00:00:00Z", }; -class WorkbenchRuntime implements DesktopRuntime { +class WorkbenchRuntime implements ClientRuntime { readonly firstFiles = deferred(); async request( @@ -202,7 +202,7 @@ it("does not let a slow previous Thread overwrite the active workbench", async ( expect(result.current.entries[0]?.path).toBe("b.txt"); }); -class MutationRuntime implements DesktopRuntime { +class MutationRuntime implements ClientRuntime { readonly calls: Array<{ method: RpcMethod; params: unknown }> = []; readonly writeResult = deferred(); fileListCount = 0; diff --git a/desktop/src/features/workbench/useCodeWorkbench.ts b/desktop/src/features/workbench/useCodeWorkbench.ts index 7f9d880fc..73fed0f63 100644 --- a/desktop/src/features/workbench/useCodeWorkbench.ts +++ b/desktop/src/features/workbench/useCodeWorkbench.ts @@ -9,7 +9,7 @@ import type { TestCommand, Thread, } from "../../generated/app-server"; -import type { BridgeError, DesktopRuntime } from "../../rpc/contracts"; +import type { BridgeError, ClientRuntime } from "../../rpc/contracts"; function errorMessage(error: unknown): string { if (typeof error === "object" && error !== null && "message" in error) { @@ -60,7 +60,7 @@ const initialState: CodeWorkbenchState = { }; export function useCodeWorkbench( - runtime: DesktopRuntime, + runtime: ClientRuntime, thread: Thread | null, ): CodeWorkbenchController { const [state, setState] = useState(initialState); diff --git a/desktop/src/features/workflows/WorkflowComposer.tsx b/desktop/src/features/workflows/WorkflowComposer.tsx index e4d4a9e67..b2913b8eb 100644 --- a/desktop/src/features/workflows/WorkflowComposer.tsx +++ b/desktop/src/features/workflows/WorkflowComposer.tsx @@ -53,15 +53,22 @@ export function WorkflowComposer({ const [enableIndexing, setEnableIndexing] = useState(false); const [planReview, setPlanReview] = useState(true); const [feedback, setFeedback] = useState(""); - const active = workflow && ["queued", "running", "waiting"].includes(workflow.status); + const [fileError, setFileError] = useState(null); + const active = + workflow && ["queued", "running", "waiting"].includes(workflow.status); const interaction = useMemo(() => workflowInteraction(workflow), [workflow]); const progress = workflow?.progressTotal ? Math.round((workflow.progressCurrent / workflow.progressTotal) * 100) : 0; const chooseFile = async () => { - const path = await onPickFile(); - if (path) setSource(path); + setFileError(null); + try { + const path = await onPickFile(); + if (path) setSource(path); + } catch (error) { + setFileError(error instanceof Error ? error.message : String(error)); + } }; const start = async () => { @@ -78,6 +85,7 @@ export function WorkflowComposer({ return (
+ {fileError &&

{fileError}

} {active ? (
@@ -85,7 +93,9 @@ export function WorkflowComposer({ Paper2Code · attempt {workflow.attempt} - {workflow.currentStage?.replaceAll("_", " ") ?? workflow.status} + + {workflow.currentStage?.replaceAll("_", " ") ?? workflow.status} +
{progress}%
@@ -99,7 +109,9 @@ export function WorkflowComposer({

{interaction.title}

{interaction.description}

- {interaction.planPreview ? 
{interaction.planPreview}
: null} + {interaction.planPreview ? ( + 
{interaction.planPreview}
+ ) : null}