diff --git a/.gitattributes b/.gitattributes index 87de85fe..8dd95a0b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,5 @@ *.jpg binary *.ico binary *.woff2 binary +# Files saved by Word and Excel, kept byte for byte. +crates/interop/tests/data/legacy/* binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6240271c..21ffd7c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,9 @@ on: # caller's event, so a release run is a push here and runs CHECKS on all files. workflow_call: +# Every action is pinned to a full commit SHA, with the version in a comment. The release workflow runs these jobs +# in the same run as its build and signing, so they could upload artifacts to it. workflow.test.ts checks the pins. +# # Every job only reads the repository. A called workflow can narrow the caller's permissions but never widen # them, so the release workflow's own read-only grant for this call still works. Uploading the debug build # as a run artifact uses the runner's own token, not GITHUB_TOKEN, so no job needs more than this. @@ -25,16 +28,21 @@ jobs: # Normally 1 to 3 minutes. A run that passes its limit is stuck, so it ends and says so. timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: npm - name: Install tools run: npm ci + # The PDF tests print Japanese words and check every character reaches the PDF's text layer. Windows has a + # Japanese font; Ubuntu runners have none, so Chrome draws empty boxes that carry no text. + - name: Install a Japanese font (Ubuntu) + if: runner.os == 'Linux' + run: sudo apt-get update -q && sudo apt-get install -y -q --no-install-recommends fonts-noto-cjk - name: Design tokens are up to date run: npm run tokens:check - name: Type-check, lint, and format-check @@ -44,8 +52,8 @@ jobs: - name: CHECKS on changed files (pull requests) if: github.event_name == 'pull_request' run: node checks/cli.ts --base origin/${{ github.base_ref }} - - name: CHECKS on all files (pushes) - if: github.event_name == 'push' + - name: CHECKS on all files (pushes and release dry runs) + if: github.event_name != 'pull_request' run: node checks/cli.ts --all components: @@ -58,10 +66,10 @@ jobs: # Normally under 2 minutes. timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: npm @@ -72,7 +80,7 @@ jobs: - name: Keep the failure screenshots # Vitest saves a screenshot of the page for each failed test. if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: component-failures-${{ matrix.os }} path: app/.vitest/attachments @@ -85,10 +93,10 @@ jobs: # Normally about 5 minutes. timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: npm @@ -119,7 +127,7 @@ jobs: run: npx playwright test --config tests/ui/playwright.config.ts - name: Keep the Playwright report if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: playwright-report path: | @@ -134,10 +142,10 @@ jobs: runs-on: windows-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: npm @@ -147,29 +155,57 @@ jobs: run: npm run perf:typing - name: Keep the numbers if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: perf-typing path: tests/perf/typing/results retention-days: 30 + native-engines: + # whisper.cpp, ONNX Runtime, and the language model program need CMake, a C++ compiler, and libclang, so the other + # jobs leave them out. This one builds and tests them on Windows, where the release builds them. + runs-on: windows-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + components: clippy + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - name: Clippy + env: + LIBCLANG_PATH: C:\Program Files\LLVM\bin + run: | + cargo clippy -p opennote-intel-engines --all-targets --features whisper-cpp,onnx -- -D warnings + cargo clippy -p opennote-llm --all-targets --features llama -- -D warnings + - name: Tests + env: + LIBCLANG_PATH: C:\Program Files\LLVM\bin + run: | + cargo test -p opennote-intel-engines --features whisper-cpp,onnx + cargo test -p opennote-llm --features llama + windows-app: # Rust checks and a test build of the desktop app on Windows, the first target platform. runs-on: windows-latest # Normally about 11 minutes with a cold cache. timeout-minutes: 35 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: npm - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: stable components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Install tools run: npm ci - name: Build the interface (the Rust app embeds it) @@ -178,8 +214,18 @@ jobs: run: cargo fmt --all -- --check - name: Clippy run: cargo clippy --workspace --all-targets -- -D warnings + - name: Clippy ratchet (no new unwrap, expect, or panic in shipped code) + # Existing uses are counted per file in checks/data/clippy-ratchet.json. A count may go down, never up. + run: node checks/clippy-ratchet.ts - name: Rust tests run: cargo test --workspace + - name: The notes contract suite against the app's notes bridge and the core + # The suite runs the notes harness example over stdin and stdout. + shell: bash + run: | + cargo build -p opennote --example notes_harness + export OPENNOTE_NOTES_HARNESS="$PWD/target/debug/examples/notes_harness.exe" + npx vitest run --config app/vitest.config.ts --project unit app/src/services/notes/core - name: Core lints and tests with every feature run: | cargo clippy -p opennote-core --all-targets --all-features -- -D warnings @@ -193,7 +239,7 @@ jobs: - name: Test build of the app (ci profile, with the test endpoints the E2E tests use) run: npx tauri build --no-bundle --features test-endpoints -- --profile ci - name: Keep the test build - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: opennote-e2e-exe path: target/ci/opennote.exe @@ -207,28 +253,30 @@ jobs: # session holds the runner for GitHub's six-hour default. timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: npm - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - name: Install tools run: npm ci - name: Cache tauri-driver - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ~/.cargo/bin/tauri-driver.exe key: tauri-driver-2.1.0-${{ runner.os }}-${{ runner.arch }} - name: Cache msedgedriver - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: tests/e2e/.drivers key: msedgedriver-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }} restore-keys: msedgedriver-${{ runner.os }}-${{ runner.arch }}- - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: opennote-e2e-exe path: target/e2e @@ -252,7 +300,7 @@ jobs: - name: Keep the E2E logs # A step that times out is canceled, not failed, and its logs are the ones worth reading. if: failure() || cancelled() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: e2e-logs path: tests/e2e/logs @@ -264,23 +312,33 @@ jobs: # Normally about 10 minutes with a cold cache. timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: stable components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Clippy for the core, the kill harness, and the benchmarks run: >- cargo clippy -p opennote-core -p opennote-crashtest -p opennote-perf --all-targets --all-features -- -D warnings - name: Core tests with every feature run: cargo test -p opennote-core --all-features + - name: Print crate tests, with stand-in lp and lpstat scripts + run: cargo test -p opennote-print + - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 + with: + tool: cargo-deny + - name: Licenses of every crate the build uses + run: cargo deny check licenses - name: Benchmark harness tests run: cargo test -p opennote-perf - name: Fuzz targets compile - run: cargo check --manifest-path crates/core/fuzz/Cargo.toml + run: | + cargo check --manifest-path crates/core/fuzz/Cargo.toml + cargo check --manifest-path crates/interop/fuzz/Cargo.toml - name: No work package stubs are left (pull requests into main) if: github.event_name == 'pull_request' && github.base_ref == 'main' run: | @@ -295,15 +353,17 @@ jobs: # Normally about 5 minutes with a cold cache. timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: npm - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Install tools run: npm ci - name: Rust fixture tests @@ -317,10 +377,10 @@ jobs: # Normally under 1 minute. timeout-minutes: 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.x' - name: Reference reader tests diff --git a/.github/workflows/crosscheck.yml b/.github/workflows/crosscheck.yml new file mode 100644 index 00000000..e570cae9 --- /dev/null +++ b/.github/workflows/crosscheck.yml @@ -0,0 +1,66 @@ +name: crosscheck + +# A second opinion on OpenNote's exports and table formulas, from independent readers. They are built from pinned +# revisions of open-source projects (tests/crosscheck/pins.json). +# - wordcraft-cli reads DOCX. +# - gridcraft-cli reads XLSX and evaluates formulas. +# - deckcraft-cli reads PPTX. +# - pdfcraft-cli reads PDF, with its accessibility check and PDF/A preflight. +# +# This job never blocks a merge. Those projects are alpha software and not ground truth, so a difference means +# "look" and shows up as a warning. Nothing from them ships with OpenNote. See docs/CHECKS.md, "Cross-checks". +on: + workflow_dispatch: + schedule: + - cron: '41 3 * * 1' + pull_request: + paths: + - 'crates/interop/src/export/**' + - 'crates/interop/src/docx/**' + - 'app/src/features/pages/pdf/**' + - 'app/src/features/tables/engine/**' + - 'tests/crosscheck/**' + - '.github/workflows/crosscheck.yml' + +permissions: + contents: read + +jobs: + crosscheck: + runs-on: ubuntu-latest + # A failure of the job itself (a tool that did not build) must not turn a pull request red. + continue-on-error: true + # The first build of the four tools takes about 25 minutes. A run that passes the limit is stuck. + timeout-minutes: 60 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: npm + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - name: Install tools + run: npm ci + - name: Build the cross-check tools from their pinned revisions + run: npm run crosscheck:install + - name: Make the corpus of exports + run: node tests/crosscheck/corpus.ts + - name: Check the exports with the other readers + run: node tests/crosscheck/run.ts --json crosscheck.json + - name: Compare table formulas with gridcraft-cli + run: npm run crosscheck:formulas + - name: Keep the corpus and the report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: crosscheck + path: | + tests/crosscheck/.corpus + crosscheck.json + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1a4a9979..a1a397a4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,31 +1,278 @@ name: nightly -# The nightly run (ARCHITECTURE.md section 21.10). This skeleton builds the app with the test endpoints and runs -# every E2E spec. WP1 grows it with the other architectures, 200% text, start-up baselines, and the frame probe; -# WP2 adds the updater specs, and WP4 the NVDA canary. A failure opens an issue, as docs/DEVELOPMENT.md section 8 -# requires. - +# The long-running hardening tests of DEVELOPMENT.md section 8: fuzzing, the kill harness, property tests with +# many cases, the performance budgets, a two-hour random-editing soak, and an NVDA run of the keyboard and screen +# reader checklist. The e2e-full job runs every E2E spec in the full app on Windows. It is +# the Phase 3 skeleton, which later phases grow with other architectures, 200% text, start-up baselines, and the +# frame probe. +# +# The jobs run every night on the default branch and on demand. A failure opens an issue, or adds a comment to the +# one that is already open. See docs/HARDENING.md. +# +# Like release.yml, every job gets only the permissions it needs, nothing persists git credentials, and the +# actions are pinned to full commit SHAs. No job uses a secret. Only the last job can write, and it can write +# issues only. on: schedule: - - cron: '0 3 * * *' + - cron: '17 2 * * *' workflow_dispatch: + inputs: + fuzz_minutes: + description: Minutes to fuzz each target + type: number + default: 30 + kills: + description: Kills in the crash-safety test + type: number + default: 1000 + proptest_cases: + description: Cases for each property test + type: number + default: 10000 + pages: + description: Pages in the performance notebook + type: number + default: 1000 + soak_minutes: + description: Minutes of random editing in the soak + type: number + default: 120 + +permissions: {} + +# Two nightly runs never overlap, so the fuzz corpus cache is never written by two runs at once. +concurrency: + group: nightly + cancel-in-progress: false -permissions: - contents: read +env: + CARGO_TERM_COLOR: always jobs: + plan: + # One fuzz job per target, read from the fuzz_targets folder so a new target is never forgotten. + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + targets: ${{ steps.targets.outputs.targets }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - name: List the fuzz targets + id: targets + run: | + # Each entry names the crate and the target: the core formats, and the importers and search indexer. + targets=$(for crate in crates/core crates/interop; do + for file in "$crate"/fuzz/fuzz_targets/*.rs; do + jq -nc --arg crate "$crate" --arg name "$(basename -s .rs "$file")" '{crate: $crate, name: $name}' + done + done | jq -sc .) + if [ "$targets" = "[]" ]; then + echo "::error::No fuzz targets found in crates/core/fuzz or crates/interop/fuzz." + exit 1 + fi + echo "targets=$targets" >> "$GITHUB_OUTPUT" + + fuzz: + # Runs each cargo-fuzz target for the same time, in parallel. The corpus is kept between nights in the + # Actions cache, so each night starts where the last one ended. + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(needs.plan.outputs.targets) }} + env: + FUZZ_TARGET: ${{ matrix.target.name }} + FUZZ_MINUTES: ${{ inputs.fuzz_minutes || '30' }} + FUZZ_CRATE: ${{ matrix.target.crate }} + FUZZ_DIR: ${{ github.workspace }}/${{ matrix.target.crate }}/fuzz + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: nightly + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ${{ matrix.target.crate }}/fuzz + shared-key: nightly-fuzz + - name: Install cargo-fuzz + run: cargo install cargo-fuzz --locked --version 0.13.2 + - name: Restore the corpus + id: corpus + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ matrix.target.crate }}/fuzz/corpus/${{ matrix.target.name }} + key: fuzz-corpus-${{ matrix.target.name }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: fuzz-corpus-${{ matrix.target.name }}- + - name: Fuzz ${{ matrix.target.crate }} ${{ matrix.target.name }} + run: | + if ! [[ "$FUZZ_MINUTES" =~ ^[0-9]+$ ]]; then + echo "::error::fuzz_minutes must be a whole number." + exit 1 + fi + mkdir -p "$FUZZ_DIR/corpus/$FUZZ_TARGET" "$FUZZ_DIR/regressions/$FUZZ_TARGET" + cd "$FUZZ_CRATE" + # The first folder collects new inputs. The second holds the crashes fixed in the past, as seeds. + cargo fuzz run "$FUZZ_TARGET" \ + "$FUZZ_DIR/corpus/$FUZZ_TARGET" "$FUZZ_DIR/regressions/$FUZZ_TARGET" \ + -- -max_total_time=$((FUZZ_MINUTES * 60)) -rss_limit_mb=2048 + - name: Keep the corpus, even when the run found a crash + if: ${{ always() && steps.corpus.outcome == 'success' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ matrix.target.crate }}/fuzz/corpus/${{ matrix.target.name }} + key: fuzz-corpus-${{ matrix.target.name }}-${{ github.run_id }}-${{ github.run_attempt }} + - name: Keep the crashing input + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: fuzz-crash-${{ matrix.target.name }} + path: ${{ matrix.target.crate }}/fuzz/artifacts/${{ matrix.target.name }} + if-no-files-found: ignore + retention-days: 30 + + kill: + # Kills a writer during saves and checks that no page is damaged or lost (DEVELOPMENT.md section 6). + # The seed is the run number, so a failure can be replayed with the same seed. + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + env: + KILLS: ${{ inputs.kills || '1000' }} + KILL_NOTEBOOK: ${{ github.workspace }}/kill-notebook + # Optimized code runs more saves in the same time than debug code, and keeps debug assertions. + CARGO_PROFILE_DEV_OPT_LEVEL: 2 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + shared-key: nightly-kill + - name: Build the kill harness + run: cargo build -p opennote-crashtest + - name: Kill the writer ${{ inputs.kills || '1000' }} times + run: | + mkdir -p "$KILL_NOTEBOOK" + echo "Seed: $GITHUB_RUN_ID" + target/debug/opennote-crashtest run --notebook "$KILL_NOTEBOOK" --iterations "$KILLS" --seed "$GITHUB_RUN_ID" + - name: Keep the notebook the harness checked + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: kill-notebook + path: kill-notebook + if-no-files-found: ignore + retention-days: 30 + + proptest: + # The core tests with far more random cases than a pull request runs. + runs-on: ubuntu-latest + timeout-minutes: 180 + permissions: + contents: read + env: + PROPTEST_CASES: ${{ inputs.proptest_cases || '10000' }} + CARGO_PROFILE_DEV_OPT_LEVEL: 2 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + shared-key: nightly-proptest + - name: Core tests with ${{ inputs.proptest_cases || '10000' }} cases for each property + run: cargo test -p opennote-core --all-features + - name: Keep the failing cases + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: proptest-regressions + path: crates/core/**/proptest-regressions + if-no-files-found: ignore + retention-days: 30 + + perf: + # The full benchmarks with their budget check. A run on a hosted runner catches large regressions. The + # budgets of BRAND.md section 10 still count only on the reference laptop. + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + env: + PAGES: ${{ inputs.pages || '1000' }} + PERF_DIR: ${{ github.workspace }}/perf-scratch + PERF_RESULTS: ${{ github.workspace }}/perf.json + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + shared-key: nightly-perf + - name: Build the benchmarks + run: cargo build --release -p opennote-perf + - name: Run every suite and check the budgets + run: | + mkdir -p "$PERF_DIR" + target/release/opennote-perf bench all --pages "$PAGES" --machine github-ubuntu-latest \ + --out "$PERF_RESULTS" "$PERF_DIR" + - name: Show the results + if: always() + run: | + [ -f "$PERF_RESULTS" ] || exit 0 + { + echo '| Measure | Value | Budget |' + echo '|---|---|---|' + jq -r '.results[] | "| \(.name) | \(.value) \(.unit) | \(.gate // "none") |"' "$PERF_RESULTS" + } >> "$GITHUB_STEP_SUMMARY" + - name: Keep the results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: perf-results + path: perf.json + if-no-files-found: ignore + retention-days: 90 + e2e-full: + # Builds the app with the test endpoints and runs every E2E spec on Windows, where the real window lives. runs-on: windows-latest + timeout-minutes: 120 + permissions: + contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: npm - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + shared-key: nightly-e2e - name: Install tools run: npm ci - name: Test build of the app @@ -37,29 +284,146 @@ jobs: - name: Every E2E spec env: OPENNOTE_E2E_EXE: target/ci/opennote.exe - run: node --test "tests/e2e/**/*.spec.ts" + # One file at a time, because the specs share the machine's windows (tests/e2e/harness.ts). The specs in + # tests/e2e/later cover Phases 6 to 13: export a PDF, import a notebook, record then stop, search, back up + # now, quick capture, and the tool windows. + run: node --test --test-concurrency=1 "tests/e2e/**/*.spec.ts" - name: Start-up baseline for main run: node tests/perf/startup.ts --runs 20 --exe target/ci/opennote.exe --record - name: Keep the E2E logs if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: nightly-e2e-logs path: tests/e2e/logs + if-no-files-found: ignore retention-days: 14 - report-failure: - needs: [e2e-full] - if: failure() + soak: + # A scripted person edits one long note at random for two hours and the job holds memory, nodes, and typing + # time flat from the first quarter to the last (A4-35, tests/soak). The seed is the run number, so a failure + # can be replayed with OPENNOTE_SOAK_SEED. + runs-on: ubuntu-latest + timeout-minutes: 200 + permissions: + contents: read + env: + OPENNOTE_SOAK_MINUTES: ${{ inputs.soak_minutes || '120' }} + OPENNOTE_SOAK_SEED: ${{ github.run_number }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: npm + - name: Install tools + run: npm ci + - name: Edit at random for ${{ inputs.soak_minutes || '120' }} minutes + run: | + if ! [[ "$OPENNOTE_SOAK_MINUTES" =~ ^[0-9]+$ ]]; then + echo "::error::soak_minutes must be a whole number." + exit 1 + fi + npm run soak + - name: Show the samples + if: always() + run: node tests/soak/summary.ts >> "$GITHUB_STEP_SUMMARY" + - name: Keep the samples + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: soak-results + path: tests/soak/results/soak.json + if-no-files-found: ignore + retention-days: 90 + + nvda: + # NVDA, driven by Guidepup, walks through the checklist for the keyboard and the screen reader in tests/nvda in a headed browser and + # records what it said (A4-42). The Guidepup packages are installed here, not in package.json, so no other job + # needs them. A phrase NVDA did not say fails the job, and nvda.json has everything it did say. + runs-on: windows-latest + timeout-minutes: 45 + permissions: + contents: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: npm + - name: Install tools + run: npm ci + - name: Install Guidepup + run: npm install --no-save --no-package-lock @guidepup/guidepup@0.35.0 + - name: Set up NVDA + run: npx --yes @guidepup/setup@0.29.1 setup --ci + - name: Walk the checklist with NVDA + env: + OPENNOTE_NVDA: '1' + run: npm run nvda + - name: Show the results + if: always() + shell: bash + run: '[ -f tests/nvda/results/nvda.md ] && cat tests/nvda/results/nvda.md >> "$GITHUB_STEP_SUMMARY" || true' + - name: Keep the results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: nvda-results + path: tests/nvda/results/nvda.* + if-no-files-found: ignore + retention-days: 90 + + report: + # Opens an issue when any job failed, or adds a comment to the issue that is already open. It is the only + # job that can write, it can only write issues, and it never checks out the code. + needs: [plan, fuzz, kill, proptest, perf, e2e-full, soak, nvda] + if: ${{ always() && github.ref == 'refs/heads/main' && contains(needs.*.result, 'failure') }} runs-on: ubuntu-latest + timeout-minutes: 10 permissions: issues: write steps: - - name: Open an issue + - name: Open or update the issue env: GH_TOKEN: ${{ github.token }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_REPO: ${{ github.repository }} + PLAN_RESULT: ${{ needs.plan.result }} + FUZZ_RESULT: ${{ needs.fuzz.result }} + KILL_RESULT: ${{ needs.kill.result }} + PROPTEST_RESULT: ${{ needs.proptest.result }} + PERF_RESULT: ${{ needs.perf.result }} + E2E_RESULT: ${{ needs.e2e-full.result }} + SOAK_RESULT: ${{ needs.soak.result }} + NVDA_RESULT: ${{ needs.nvda.result }} run: | - gh issue create --repo "$GITHUB_REPOSITORY" \ - --title "Nightly run failed on $(date -u +%Y-%m-%d)" \ - --body "The nightly run failed: $RUN_URL" + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + { + echo "The nightly tests failed on $(date -u +%F). Failed jobs:" + echo + [ "$PLAN_RESULT" = failure ] && echo "- Listing the fuzz targets" + [ "$FUZZ_RESULT" = failure ] && echo "- Fuzzing: a crashing input is in the run's artifacts" + [ "$KILL_RESULT" = failure ] && echo "- Crash safety: replay with seed $GITHUB_RUN_ID" + [ "$PROPTEST_RESULT" = failure ] && echo "- Property tests: the log has the failing case" + [ "$PERF_RESULT" = failure ] && echo "- Performance: the results are in the run's artifacts" + [ "$E2E_RESULT" = failure ] && echo "- E2E specs: the logs are in the run's artifacts" + # checks-disable-next-line length: one shell line that writes one line of the run summary + [ "$SOAK_RESULT" = failure ] && echo "- Random-editing soak: the samples are in the run's artifacts; replay with the run number as the seed" + [ "$NVDA_RESULT" = failure ] && echo "- NVDA checklist: the spoken log is in the run's artifacts" + echo + echo "Run: $run_url" + echo + echo "See docs/HARDENING.md for how to reproduce each failure." + } > body.md + gh label create nightly-failure --color B60205 --description "A nightly test run failed" --force + number=$(gh issue list --label nightly-failure --state open --limit 1 \ + --json number --jq '.[0].number // empty') + if [ -n "$number" ]; then + gh issue comment "$number" --body-file body.md + else + gh issue create --title "The nightly tests are failing" --label nightly-failure --body-file body.md + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 093a03e6..e13ca265 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,34 +1,121 @@ name: release -# Pushing a tag such as v0.1.0 runs CI, builds one exe per Windows architecture, signs the updates, and publishes -# a GitHub Release. See docs/RELEASING.md. Each job gets only the permissions it needs, and only the signing step -# sees the signing secrets. The actions in this file are pinned to full commit SHAs, because they build and sign -# the updates. +# Pushing a tag such as v1.0.0 runs the release checklist and CI, builds one exe per Windows architecture, signs the +# updates, and publishes a GitHub Release. +# +# Running the workflow by hand (workflow_dispatch) is a dry run. It does everything except publish, signs with a +# key made for that run, tests an update from the last beta to the new build, and keeps the results as run +# artifacts. Its dry_run input stays on: a release is published only from a pushed tag. +# +# See docs/RELEASING.md. Each job gets only the permissions it needs. The signing secrets belong to the release +# environment, which only v* tags may use after a maintainer approves. They reach one step of the package job, which +# can only read the repository. The publish job can write releases, but it gets no secrets and runs no project code. +# The actions in this file are pinned to full commit SHAs, because they build and sign updates. on: push: tags: ['v*'] + workflow_dispatch: + inputs: + dry_run: + description: 'Dry run: build, sign with a throwaway key, and test an update from the last beta. Publishes nothing.' + type: boolean + default: true permissions: {} +# One release run per ref, so two runs never publish the same tag at once. A dry run on a branch has its own group. +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + defaults: run: shell: pwsh jobs: version: - # Fails in seconds when the tag doesn't match the version files, before CI and the build run. + # Works out the tag, and fails in seconds when it doesn't match the version files, before CI and the build run. + # A dry run uses the version in the files. The release checklist is enforced for a pushed tag, except a tag + # for a test build, such as v0.3.0-test.1, which only reports it. runs-on: ubuntu-latest permissions: contents: read + outputs: + tag: ${{ steps.tag.outputs.tag }} + enforce: ${{ steps.tag.outputs.enforce }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: + # The whole history and every branch, to check that the tag is on main. + fetch-depth: 0 persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 + - name: Check that the tag is on main + # Only reviewed code is released. Branch protection on main doesn't cover tags, so a tag pushed on any other + # commit stops here. The release environment and the tag ruleset (docs/RELEASING.md) guard the secrets. + if: github.event_name == 'push' + run: | + git merge-base --is-ancestor $env:GITHUB_SHA origin/main + if ($LASTEXITCODE -ne 0) { + "::error::The tagged commit $env:GITHUB_SHA is not on main. Tag a merge commit on main." + exit 1 + } + - name: Refuse a run by hand that is not a dry run + # Publishing needs the release environment, which only a pushed v tag can use. + if: github.event_name == 'workflow_dispatch' + env: + DRY_RUN: ${{ inputs.dry_run }} + run: | + if ($env:DRY_RUN -ne 'true') { + "::error::A release is published only from a pushed v tag. Run by hand with dry_run on, or push the tag." + exit 1 + } + - id: tag + name: Work out the tag + run: | + if ($env:GITHUB_EVENT_NAME -eq 'push') { $tag = $env:GITHUB_REF_NAME } + else { $tag = 'v' + (Get-Content package.json -Raw | ConvertFrom-Json).version } + $enforce = ($env:GITHUB_EVENT_NAME -eq 'push') -and ($tag -notmatch '-test(\.|$)') + "tag=$tag" >> $env:GITHUB_OUTPUT + "enforce=$($enforce.ToString().ToLower())" >> $env:GITHUB_OUTPUT - name: Check that the tag matches the version files - run: node app/scripts/check-version.ts $env:GITHUB_REF_NAME + env: + TAG: ${{ steps.tag.outputs.tag }} + run: node app/scripts/check-version.ts $env:TAG + + checklist: + # The release checklist in docs/DEVELOPMENT.md section 10. For a release it must be complete, so a missing + # sign-off stops the release before any build. For a dry run or a test build it reports what is left. + needs: version + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + issues: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + - name: Install tools without install scripts + run: npm ci --ignore-scripts + - name: Check the release checklist + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.version.outputs.tag }} + ENFORCE: ${{ needs.version.outputs.enforce }} + run: | + $extra = if ($env:ENFORCE -eq 'true') { @() } else { @('--report') } + $version = $env:TAG -replace '^v', '' + node app/scripts/release/checklist.ts --version $version --tag $env:TAG --markdown checklist.md @extra + - name: Show the checklist in the run summary + if: always() + run: if (Test-Path checklist.md) { Get-Content checklist.md >> $env:GITHUB_STEP_SUMMARY } ci: # The same checks a pull request gets, run on the tagged commit. No secrets are passed. @@ -40,48 +127,89 @@ jobs: build: # Builds one exe per Windows architecture with a read-only token and no secrets. The x64 runner # cross-compiles the 32-bit and ARM64 exes with the MSVC tools it has for those architectures. - needs: ci + needs: [ci, checklist] runs-on: windows-latest permissions: contents: read + outputs: + # The SHA-256 of each exe, under a name for its Rust target. The package job signs an exe only if it still has + # this hash. Each leg of the matrix sets only its own output, and GitHub keeps every output that is set. + sha256-x86_64-pc-windows-msvc: ${{ steps.hash.outputs.sha256-x86_64-pc-windows-msvc }} + sha256-i686-pc-windows-msvc: ${{ steps.hash.outputs.sha256-i686-pc-windows-msvc }} + sha256-aarch64-pc-windows-msvc: ${{ steps.hash.outputs.sha256-aarch64-pc-windows-msvc }} strategy: - # Let every architecture finish, so one run shows every failure. The publish job needs all three. + # Let every architecture finish, so one run shows every failure. The package job needs all three. fail-fast: false matrix: # The same targets and file names as app/release-files.json. The write-manifest.ts tests check this. include: + # Only the 64-bit exe carries the native engines (whisper.cpp, ONNX Runtime, and the language model + # program), because ONNX Runtime has no build for the 32-bit target. The others use the pure Rust speech + # reader and the rule-based writing tools, and say so where a model would be offered. - target: x86_64-pc-windows-msvc file: OpenNote_Windows64.exe + features: native-engines,embed-llm-host - target: i686-pc-windows-msvc file: OpenNote_Windows32.exe + features: '' - target: aarch64-pc-windows-msvc file: OpenNote_WindowsARM64.exe + features: '' env: RELEASE_TARGET: ${{ matrix.target }} RELEASE_FILE: ${{ matrix.file }} + # A release build stops without a committed update key (app/src-tauri/build.rs). A dry run may build without + # one, because nothing it builds is published. + OPENNOTE_DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }} + RELEASE_FEATURES: ${{ matrix.features }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 persist-credentials: false + # No npm or Rust build cache: the CI jobs of the same ref can write those caches, so each exe is always built + # from a clean start. - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 - cache: npm - # No Rust build cache: each exe is always built from a clean target directory. - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: toolchain: stable targets: ${{ matrix.target }} - name: Install tools run: npm ci + - name: Build the opennote command + # The app carries the command-line tool inside it and installs it as bin\opennote.exe (install/cli.rs). + run: cargo build --release --locked -p opennote-cli --target $env:RELEASE_TARGET + - name: Build the language model program + # llama.cpp lives in a program of its own, which the exe carries inside it, because whisper.cpp and llama.cpp + # each bundle ggml and cannot be linked into one program. Windows runners have CMake and LLVM. + if: matrix.features != '' + env: + LIBCLANG_PATH: C:\Program Files\LLVM\bin + run: cargo build --release --locked -p opennote-llm --features llama --target $env:RELEASE_TARGET - name: Build ${{ matrix.file }} - run: npx tauri build --no-bundle --target $env:RELEASE_TARGET + env: + LIBCLANG_PATH: C:\Program Files\LLVM\bin + OPENNOTE_LLM_HOST_BIN: ${{ github.workspace }}\target\${{ matrix.target }}\release\opennote-llm.exe + run: | + $env:OPENNOTE_CLI_EXE = (Resolve-Path "target/$env:RELEASE_TARGET/release/opennote-cli.exe").Path + $build = @('tauri', 'build', '--no-bundle', '--target', $env:RELEASE_TARGET) + if ($env:RELEASE_FEATURES) { $build += @('--features', $env:RELEASE_FEATURES) } + npx @build - name: Collect the exe run: | New-Item -ItemType Directory -Force dist-release | Out-Null Copy-Item "target/$env:RELEASE_TARGET/release/opennote.exe" "dist-release/$env:RELEASE_FILE" - - name: Keep the exe for the publish job + - name: Check that the exe may ship + # Fails for an exe built with the test-endpoints feature, or with an update key that is not committed. + run: node app/scripts/check-release-exe.ts "dist-release/$env:RELEASE_FILE" + - id: hash + name: Report the hash of the exe + run: | + $hash = (Get-FileHash "dist-release/$env:RELEASE_FILE" -Algorithm SHA256).Hash.ToLowerInvariant() + "sha256-$env:RELEASE_TARGET=$hash" >> $env:GITHUB_OUTPUT + - name: Keep the exe for the package job uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: exe-${{ matrix.target }} @@ -89,49 +217,277 @@ jobs: if-no-files-found: error retention-days: 1 - publish: - # The only job that can write releases. It builds nothing and runs no install scripts, - # so the only project dependency that runs next to the signing key is the Tauri CLI. - needs: build + sbom: + # Lists everything built into the exes. It runs cargo metadata, which downloads crate sources, so it runs apart + # from the signing step. It gets no secrets. + needs: version runs-on: windows-latest permissions: - contents: write + contents: read + outputs: + # The package job publishes the bill of materials only if it still has this hash. + sha256: ${{ steps.hash.outputs.sha256 }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - name: Install tools without install scripts + run: npm ci --ignore-scripts + - name: Write the software bill of materials + env: + TAG: ${{ needs.version.outputs.tag }} + run: | + New-Item -ItemType Directory -Force dist-sbom | Out-Null + node app/scripts/release/sbom.ts $env:TAG dist-sbom/OpenNote.sbom.cdx.json + - id: hash + name: Report the hash of the bill of materials + run: | + $hash = (Get-FileHash dist-sbom/OpenNote.sbom.cdx.json -Algorithm SHA256).Hash.ToLowerInvariant() + "sha256=$hash" >> $env:GITHUB_OUTPUT + - name: Keep the bill of materials for the package job + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: sbom + path: dist-sbom/OpenNote.sbom.cdx.json + if-no-files-found: error + retention-days: 1 + + package: + # Signs the exes, writes the manifest and the other release files, and checks them the way the app will. This + # is the only job that gets the signing secrets, and only in the step that signs. It can only read the + # repository, and it runs no install scripts, so the Tauri command-line tool is the only project + # dependency that runs next to the key. A dry run signs with a throwaway key instead, and gets no secrets. + needs: [version, checklist, build, sbom] + runs-on: windows-latest + # The signing secrets live in the release environment, which only v* tags may use, after a maintainer approves. + # A workflow file on any other ref can't read them. A dry run uses no environment. + environment: ${{ github.event_name == 'push' && 'release' || '' }} + permissions: + contents: read + env: + TAG: ${{ needs.version.outputs.tag }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: + # The release notes list the merges since the previous tag. + fetch-depth: 0 persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 - name: Install tools without install scripts run: npm ci --ignore-scripts - - name: Download the exes + # Each artifact by its exact name, never by a pattern. Any job in the run can upload artifacts, including the CI + # jobs. Their actions and test tools come from outside this repository. + - name: Download the 64-bit exe + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: exe-x86_64-pc-windows-msvc + path: dist-release + - name: Download the 32-bit exe + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: exe-i686-pc-windows-msvc + path: dist-release + - name: Download the ARM64 exe uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - pattern: exe-* + name: exe-aarch64-pc-windows-msvc path: dist-release - merge-multiple: true + - name: Download the bill of materials + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: sbom + path: dist-release + - name: Check the downloads before signing + # Stops unless dist-release holds only the three exes and the bill of materials, each with the hash its job + # reported. It also checks each exe again for test endpoints and update keys that aren't committed. + env: + BUILD_SHA256: ${{ toJSON(needs.build.outputs) }} + SBOM_SHA256: ${{ needs.sbom.outputs.sha256 }} + run: node app/scripts/release/check-downloads.ts dist-release + - name: Write the release notes + run: node app/scripts/release/notes.ts $env:TAG --markdown release-notes.md --plain release-notes.txt + # Add the Authenticode code-signing step here, before the update signatures, once the certificate exists. + # See docs/RELEASING.md, Add code signing later. - name: Sign the updates + if: github.event_name == 'push' env: TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }} + # Stops with an error unless TAURI_SIGNING_PRIVATE_KEY is set, so nothing is published unsigned. + run: node app/scripts/release/sign.ts dist-release $env:TAG + - name: Sign with a throwaway key (dry run) + if: github.event_name == 'workflow_dispatch' + run: node app/scripts/release/sign.ts dist-release $env:TAG --dry-run --pubkey-out dry-run/update.pub + - name: Write the update manifest + env: + RELEASE_NOTES_FILE: release-notes.txt + # Stops unless every exe is here, with a signature for this version. + run: node app/scripts/write-manifest.ts dist-release $env:TAG + - name: Write the checksums + run: node app/scripts/release/checksums.ts dist-release + - name: Check the release the way the app will + if: github.event_name == 'push' + env: + ENFORCE: ${{ needs.version.outputs.enforce }} + # Uses the update keys committed in app/src-tauri/keys, which are the keys built into the exe. A release + # also needs an Authenticode signature, unless it is a test build. run: | - if (-not $env:TAURI_SIGNING_PRIVATE_KEY) { - Write-Output '::error::The TAURI_SIGNING_PRIVATE_KEY secret is not set. See docs/RELEASING.md.' - exit 1 + $extra = if ($env:ENFORCE -eq 'true') { @('--require-authenticode') } else { @() } + node app/scripts/release/verify.ts dist-release $env:TAG @extra + - name: Check the dry run the way the app will + if: github.event_name == 'workflow_dispatch' + run: node app/scripts/release/verify.ts dist-release $env:TAG --pubkey dry-run/update.pub + - name: Write the winget manifests + # winget takes stable releases only. + if: ${{ !contains(needs.version.outputs.tag, '-') }} + run: node app/scripts/release/winget.ts dist-release $env:TAG winget-manifests + - name: Summarize what would be published + if: always() + run: | + "## Release files for $env:TAG" >> $env:GITHUB_STEP_SUMMARY + if ($env:GITHUB_EVENT_NAME -eq 'workflow_dispatch') { + "Dry run: nothing is published. The update signatures use a throwaway key." >> $env:GITHUB_STEP_SUMMARY } - # --app-version puts the version in the signed trusted comment, so it can't be changed later. - $version = $env:GITHUB_REF_NAME -replace '^v', '' - foreach ($exe in Get-ChildItem dist-release -Filter *.exe) { - npx tauri signer sign --app-version $version $exe.FullName - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if (Test-Path dist-release) { + foreach ($file in Get-ChildItem dist-release) { + "- $($file.Name) ($($file.Length) bytes)" >> $env:GITHUB_STEP_SUMMARY + } + } + - name: Keep the release files + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-files + path: | + dist-release + release-notes.md + dry-run/update.pub + if-no-files-found: error + retention-days: 7 + - name: Keep the winget manifests + if: ${{ !contains(needs.version.outputs.tag, '-') }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: winget-manifests + path: winget-manifests + if-no-files-found: error + retention-days: 90 + + update-test: + # The update-from-earlier-beta test: the updater's test app, built from the last beta tag, updates itself to + # this release's x64 exe from a local server, checking the hash, and the signature on the way. A dry run checks + # against the throwaway key, and a tag against the committed key. It gets no secrets. + needs: [version, package] + runs-on: windows-latest + permissions: + contents: read + env: + TAG: ${{ needs.version.outputs.tag }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + # Every tag, to find the last beta. + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - name: Download the release files + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: release-files + path: . + - name: Build the test app from the last beta + # Without an earlier beta tag, the test app is built from this commit, and the summary says so. + run: | + $earlier = node app/scripts/release/update-test.ts earlier $env:TAG + if ($earlier) { + git worktree add earlier-beta $earlier + $from = $earlier -replace '^v', '' + Push-Location earlier-beta + } else { + "No earlier beta tag: the test app is built from this commit." >> $env:GITHUB_STEP_SUMMARY + $from = '0.0.1' + } + cargo build --example fake_app -p opennote-updater + if ($LASTEXITCODE -ne 0) { exit 1 } + if ($earlier) { Pop-Location; $app = 'earlier-beta/target/debug/examples/fake_app.exe' } + else { $app = 'target/debug/examples/fake_app.exe' } + "FAKE_APP=$app" >> $env:GITHUB_ENV + "FROM_VERSION=$from" >> $env:GITHUB_ENV + - name: Update from the last beta (dry run) + if: github.event_name == 'workflow_dispatch' + run: >- + node app/scripts/release/update-test.ts run dist-release $env:TAG --fake-app $env:FAKE_APP + --from $env:FROM_VERSION --pubkey dry-run/update.pub --report update-test.json + - name: Update from the last beta (tag) + if: github.event_name == 'push' + run: >- + node app/scripts/release/update-test.ts run dist-release $env:TAG --fake-app $env:FAKE_APP + --from $env:FROM_VERSION --report update-test.json + - name: Show the update test in the run summary + if: always() + run: if (Test-Path update-test.json) { Get-Content update-test.json >> $env:GITHUB_STEP_SUMMARY } + - name: Keep the update test report for the sign-off + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: update-test + path: update-test.json + if-no-files-found: ignore + retention-days: 30 + + publish: + # The only job that can write releases. It publishes the files the package job checked, and nothing else. + # It gets no secrets and runs no project code. A dry run never reaches it. + if: github.event_name == 'push' + needs: [version, package, update-test] + runs-on: ubuntu-latest + # A maintainer approves again here, after reading the package job's summary, before anything goes live. + environment: release + permissions: + contents: write + steps: + - name: Download the release files + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: release-files + path: . + - name: Check that the beta channel can be updated + # The beta channel reads beta.json from the fixed channel-manifests prerelease (crates/updater/src/config.rs), + # not from the tag's own release. Stop before publishing anything if that prerelease is missing. + if: needs.version.outputs.enforce == 'true' && contains(needs.version.outputs.tag, '-') + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + $prerelease = gh release view channel-manifests --json isPrerelease --jq .isPrerelease + if ($LASTEXITCODE -ne 0 -or $prerelease -ne 'true') { + "::error::The channel-manifests prerelease is missing. See docs/RELEASING.md, Create the beta channel." + exit 1 } - - name: Write the update manifest - # Stops unless every exe in RELEASE_FILES is here, with a signature for this version. - run: node app/scripts/write-manifest.ts dist-release $env:GITHUB_REF_NAME - name: Publish the release uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 with: files: dist-release/* - prerelease: ${{ contains(github.ref_name, '-') }} - generate_release_notes: true + body_path: release-notes.md + fail_on_unmatched_files: true + prerelease: ${{ contains(needs.version.outputs.tag, '-') }} + - name: Update the beta channel + # Only for a beta. A test build writes beta.json too, but it must never reach beta copies. Its version sorts + # above every beta of the same release, so they would install it and then refuse the later betas. + if: needs.version.outputs.enforce == 'true' && contains(needs.version.outputs.tag, '-') + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: gh release upload channel-manifests dist-release/beta.json --clobber diff --git a/.gitignore b/.gitignore index 901df2f9..386ea862 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,13 @@ tests/e2e/logs/ tests/ui/report/ tests/ui/results/ tests/perf/typing/results/ +extensions/dist/ + +# The owner of a build keeps the connectors app registrations here (docs/CONNECTORS.md). +/connectors.config.json +tests/soak/results/ +tests/nvda/results/ + +# Cross-check tools built from pinned upstream revisions (tests/crosscheck) +tests/crosscheck/.tools/ +tests/crosscheck/.corpus/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..f195f2f3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,126 @@ +# Changelog + +All notable changes to OpenNote are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Changed + +- Quick capture no longer takes Ctrl+Alt+Q, which AltGr+Q also presses and which types @ on many keyboards. It is off until it is turned on in Settings, where the shortcut can be chosen and a clash with OpenNote's own keys is shown. A beta 4 profile that kept Ctrl+Alt+Q gets no shortcut until new keys are chosen, and its switch can still be turned off. +- Search by meaning and text in images keep nothing of a page in a protected section, and drop what they kept of a page whose section becomes protected. + +### Foundation + +- Set up the project structure for Windows development with Tauri 2 +- Integrated TypeScript and React for the user interface +- Configured continuous integration with automated checks and tests +- Established the release workflow for signed, stable builds + +### On-device intelligence + +- The 64-bit build reads speech with whisper.cpp, translates with ONNX Runtime, and writes Ask answers and writing suggestions with a language model that runs on the computer +- Language models, a meaning model, and translation packs download only after the person sees the size and agrees, from addresses pinned to one commit and checked against a published checksum +- Without a downloaded model, Ask quotes the pages and the writing tools use rules, and the screen says which wrote the answer +- Translate, dictation, live captions, and handwriting to math have browser tests + +### Releases + +- Each release ships three Windows files: one for 64-bit PCs, one for 32-bit PCs, and one for Arm PCs +- Each release lists what OpenNote is made of, and publishes a checksum for every file +- The update notice shows the release notes for the new version +- A release is checked the way the app checks an update before anyone can download it +- Windows package manager (winget) files are made for every stable release + +### Changed (may break scripts) + +- The `opennote` command reads `--file` and writes `export` and `render` files only inside the current folder. Scripts that use other folders now need `--read-root FOLDER` or `--write-root FOLDER`; network folders and Windows device names are always refused +- An agent's `--author` label in page history drops the word "via", quotes, and unpaired brackets, so it can't read as another app's change + +### Experiments + +- Evaluated ink drawing performance with Surface Pen and Wacom tablets +- Tested placing text editors on a zoomable canvas alongside handwriting +- Prototyped PDF export to verify page layout fidelity +- Built audio recording support for microphone and system audio capture + +## 0.1.0-beta.4 - 2026-10-03 + +Unsigned, for Windows, with no installer and no self-update. Agents only started it and opened each main screen. The hand test is [the beta 4 checklist](docs/testing/beta-4-checklist.md). + +### Added + +- Smart tables, charts, math, and the grapher, with calculated columns, board, calendar, gallery, and timeline views, Mermaid diagrams, and mind maps +- Search and links: the search panel, the quick switcher, page and paragraph links, backlinks, tags, daily notes, the graph view, and replace across notebooks +- Recording with stamped notes, flags, trim, system audio, transcript blocks, speakers, and meeting recaps +- Import and export for Markdown, Obsidian, Word, HTML, Excel, PowerPoint, email, Kindle, and Readwise files, with a review and undo +- On-device intelligence, all off until turned on: text in pictures, read aloud, summaries, handwriting to text, search by meaning, Ask your notes, and writing tools +- Hardening and privacy: crash reports with consent, a self-check, a feedback file, safe start, and Work offline +- Study tools: flashcards, Anki and CSV, study tape, Upcoming, timers, a calculator, a unit converter, and citations +- Quality-of-life features in eleven areas, 142 in all, such as pen gestures, insert space, page templates, find and replace, layout presets, tabs, focus mode, quick capture, and scheduled backups +- Settings > Connectors, with sign-in for six services and tokens for three more, kept in Windows Credential Manager +- Real screenshots of every screen, 34 redrawn wireframes, and a help page for each area of the app + +### Changed + +- A new page has its title and a gray "Changed" line only, with no date text box +- On lined, grid, and dot paper, text rests on the rules like handwriting, and text boxes snap from rule to rule +- The page area shrinks with a narrower window or pane, with no empty strip to scroll into +- The start-up bundle limit rose from 250 KB to 300 KB gzipped to fit the new features + +### Fixed + +- A stray box beside each to-do checkbox +- A sideways scroll bar under the page at 150 percent display scale + +### Not built yet + +- 26 features, including the pen library, PDF import, the local API, the web clipper, and the speech engine for transcripts. [FEATURES.md](docs/FEATURES.md) marks each one. +- 15 features wait on the owner's accounts, app registrations, or a decision about the note format. + +## 0.1.0-beta.3 - 2026-10-03 + +### Added + +- The typed notes editor: headings, lists, checklists, quotes, code, tables, images, and the formatting bar +- The "desk by the window" look, with new setup screens +- Ink: the Draw tab with pens, a highlighter, two erasers, the lasso, and ink to shape, with palm rejection while a pen is near +- Page views: infinite canvas and pages with breaks, paper and backgrounds, a gallery, slides, and reading aids +- Export as PDF, picture, HTML, or Markdown, and Print + +### Fixed + +- A page with handwriting no longer scrolls sideways +- The lasso's bar stays inside the page view +- Lassoed text boxes move together with their ink + +## 0.1.0-beta.2 - 2026-10-03 + +### Added + +- Trash with Restore for deleted pages, sections, section groups, and notebooks +- Opening a notes folder on another computer shows its notebooks +- Beta 1 notes move over by themselves on the first start, and a copy of the old files is kept + +### Changed + +- The notes folder holds all of your notes in OpenNote's open note format. Each notebook is a folder with its sections, pages, history, images, and Trash. +- The big heading on a page and its name in the list are one title +- New notebook opens the new notebook + +### Fixed + +- A horizontal scroll bar showed under the page when nothing was off to the side + +## 0.1.0-beta.1 - 2026-10-03 + +The first beta build, for Windows, unsigned. + +### Added + +- First-run setup with a look, a notes folder, and a first notebook, and an offer to add OpenNote to the Start menu +- Notebooks, sections, and pages, with headings, bulleted lists, bold text, the slash menu for code blocks and tables, and undo and redo +- Pages that save as you type and come back after a restart + +[Unreleased]: https://github.com/XrxcGH/OpenNote/compare/main...main diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34fb5e31..3f98372d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,7 @@ These documents describe how OpenNote is planned and built: - [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md): the phases, technology choices, tests, and working agreements. - [docs/BRAND.md](docs/BRAND.md): colors, type, motion, voice, accessibility, and performance budgets. - [docs/CHECKS.md](docs/CHECKS.md): the quality gate every file change must pass. +- [docs/HARDENING.md](docs/HARDENING.md): the nightly tests, and how crash reports keep note content out. - [Project wiki](https://github.com/XrxcGH/OpenNote/wiki): the user guide, development updates, and the developer guide. The [documentation index](docs/README.md) lists every document in `docs/`. The [repository map](README.md#repository-map) shows where each part of the code lives. @@ -36,6 +37,7 @@ Windows is the first platform OpenNote supports. Install these tools once, in th 3. **Rust stable, through rustup.** Install it from [rustup.rs](https://rustup.rs/), and keep the default `stable-msvc` toolchain. If you changed it, `rustup default stable-msvc` sets it back. Run `rustup update` now and then. 4. **Node.js 22.18 or newer.** Get the long-term support release from [nodejs.org](https://nodejs.org/). 5. **Git**, to clone the repository. +6. **CMake**, on your `PATH` (`winget install Kitware.CMake`). The printing crates build a download-and-verify helper that needs it, though the shipped app does not. Without CMake, building the app crate fails in `aws-lc-sys`. On macOS or Linux, install the [Tauri prerequisites](https://v2.tauri.app/start/prerequisites/) for your system in place of steps 1 and 2. @@ -53,6 +55,8 @@ The first start compiles the Rust dependencies, which takes several minutes. Lat Also run `npm run setup-hooks` once, so CHECKS runs before each commit. +Audio recording encodes with libopus, which the `opus` crate builds from source with CMake. The Visual Studio C++ tools include CMake, but it may not be on your `PATH`. The encoder sits behind the `opus` feature of `opennote-media`, so you need CMake only to test it with `cargo test -p opennote-media --features opus`. Tests without the feature use a stand-in encoder. + ## npm scripts | Script | What it does | @@ -76,6 +80,7 @@ Also run `npm run setup-hooks` once, so CHECKS runs before each commit. | `npm run checks:all` | Runs CHECKS on every file. | | `npm run checks:fix` | Applies the safe automatic fixes CHECKS offers. | | `npm run test:checks` | Tests the CHECKS program itself. | +| `npm run test:design` | Checks the wireframes and the social preview by the numbers: things stand on their line, arrows have heads. | | `npm run setup-hooks` | Turns on the Git hook that runs CHECKS before each commit. | | `npm run tauri -- ` | Runs the Tauri command line, for example `npm run tauri -- info`. | diff --git a/Cargo.lock b/Cargo.lock index fee6acbb..d94d7927 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout 0.2.2", +] + [[package]] name = "aes" version = "0.9.3" @@ -19,6 +29,20 @@ dependencies = [ "cpufeatures 0.3.1", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -95,6 +119,25 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "argon2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures 0.3.1", + "password-hash", + "zeroize", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "atk" version = "0.18.2" @@ -130,6 +173,36 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" +dependencies = [ + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -154,6 +227,26 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -184,6 +277,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "blake2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -361,6 +463,15 @@ dependencies = [ "toml 1.1.6+spec-1.1.0", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.2.1" @@ -377,7 +488,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", - "shlex", + "jobserver", + "libc", + "shlex 2.0.1", ] [[package]] @@ -386,6 +499,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "cfb" version = "0.14.0" @@ -413,6 +535,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.2" @@ -420,8 +548,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", + "cipher 0.5.2", "cpufeatures 0.3.1", "rand_core 0.10.1", + "zeroize", +] + +[[package]] +name = "chacha20poly1305" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b89e1c441e926b9c82a8d023f6e1b7ae0adcfaa7d621814e4d60789bac751cb" +dependencies = [ + "aead", + "chacha20", + "cipher 0.5.2", + "poly1305", + "zeroize", ] [[package]] @@ -431,8 +574,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] @@ -452,10 +597,22 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ + "block-buffer 0.12.1", "crypto-common 0.2.2", "inout 0.2.2", ] +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading 0.8.9", +] + [[package]] name = "cmake" version = "0.1.58" @@ -465,6 +622,36 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "cmpv2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "961b955a666e25ee5a1091d219128d6e6401e3dab84efb1a2bf6b4035d797b39" +dependencies = [ + "crmf", + "der 0.7.10", + "spki", + "x509-cert", +] + +[[package]] +name = "cms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" +dependencies = [ + "const-oid 0.9.6", + "der 0.7.10", + "spki", + "x509-cert", +] + [[package]] name = "combine" version = "4.6.8" @@ -475,6 +662,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" @@ -491,6 +699,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -514,7 +732,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "064badf302c3194842cf2c5d61f56cc88e54a759313879cdf03abdd27d0c3b97" dependencies = [ "bitflags 2.13.2", - "core-foundation", + "core-foundation 0.10.1", "core-graphics-types", "foreign-types 0.5.0", "libc", @@ -527,7 +745,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" dependencies = [ "bitflags 2.13.2", - "core-foundation", + "core-foundation 0.10.1", "libc", ] @@ -605,6 +823,27 @@ dependencies = [ "libc", ] +[[package]] +name = "craft-print" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=8f5a2831614e632f91ad367f7af2a6e5b61b9529#8f5a2831614e632f91ad367f7af2a6e5b61b9529" +dependencies = [ + "craft-winprint-gdi", + "miniz_oxide 0.8.9", + "roxmltree", + "tempfile", + "winprint", + "winsafe", +] + +[[package]] +name = "craft-winprint-gdi" +version = "0.1.0" +source = "git+https://github.com/storytold/winprint.rs?rev=417ac3c4d14d3da5485a99c7d5ce8f05664ec312#417ac3c4d14d3da5485a99c7d5ce8f05664ec312" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -614,6 +853,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crmf" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fe21b96d5b87f5de4b5b7202ec41c00110ac817ce6728fe75fb2fe5962ed92" +dependencies = [ + "cms", + "der 0.7.10", + "spki", + "x509-cert", +] + [[package]] name = "crossbeam-channel" version = "0.5.17" @@ -702,14 +953,47 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +[[package]] +name = "ctutils" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] @@ -725,17 +1009,37 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling_macro" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core", + "darling_core 0.24.1", "quote", "syn 3.0.6", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dasp_sample" version = "0.11.0" @@ -784,16 +1088,40 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "der_derive", + "flagset", + "pem-rfc7468 0.7.0", + "zeroize", +] + [[package]] name = "der" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" dependencies = [ - "pem-rfc7468", + "pem-rfc7468 1.0.0", "zeroize", ] +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "deranged" version = "0.5.8" @@ -803,6 +1131,37 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -842,8 +1201,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid", + "const-oid 0.10.2", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -1013,6 +1373,26 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1041,10 +1421,48 @@ dependencies = [ ] [[package]] -name = "fastrand" -version = "2.5.0" +name = "esaxx-rs" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "fdeflate" @@ -1065,12 +1483,37 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "find_cuda_helper" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f9e65c593dd01ac77daad909ea4ad17f0d6d1776193fc8ea766356177abdad" +dependencies = [ + "glob", +] + +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flate2" version = "1.1.10" @@ -1082,6 +1525,28 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fmt-derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cb4a4b45c26455885f8b462db12978e51f60cc914337bccb9a36faf93410169" +dependencies = [ + "fmt-derive-proc", +] + +[[package]] +name = "fmt-derive-proc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7704da0760734344ce8753f15b21e97c6be85dc5a8f2c3a963fbd042b3380d23" +dependencies = [ + "proc-macro-crate 3.5.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1149,6 +1614,12 @@ dependencies = [ name = "frame-probe" version = "0.1.0" +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1165,6 +1636,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1338,6 +1810,19 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -1512,17 +1997,57 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] [[package]] name = "heck" @@ -1551,6 +2076,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac-sha256" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad320b3b96fb2a455a0726d16efe0a5afdbd34b71dea5bc53b05ea057714d4e" + [[package]] name = "html5ever" version = "0.39.0" @@ -1619,6 +2150,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1629,6 +2161,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.21" @@ -1648,9 +2195,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -1808,6 +2357,8 @@ dependencies = [ "moxcms", "num-traits", "png 0.18.1", + "zune-core", + "zune-jpeg", ] [[package]] @@ -1887,6 +2438,24 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2044,6 +2613,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.106" @@ -2107,6 +2686,18 @@ dependencies = [ "libc", ] +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid", + "polycool", + "smallvec", +] + [[package]] name = "libappindicator" version = "0.9.0" @@ -2127,7 +2718,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9ec52138abedcc58dc17a7c6c0c00a2bdb4f3427c7f63fa97fd0d859155caf" dependencies = [ "gtk-sys", - "libloading", + "libloading 0.7.4", "once_cell", ] @@ -2156,6 +2747,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libredox" version = "0.1.25" @@ -2165,6 +2766,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2177,6 +2789,33 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +[[package]] +name = "llama-cpp-2" +version = "0.1.159" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fab4a3cd6f539c1d658161c85390c18b1d3f67283275e1567b34417dca3dd8c9" +dependencies = [ + "enumflags2", + "llama-cpp-sys-2", + "thiserror 2.0.21", + "tracing", + "tracing-core", +] + +[[package]] +name = "llama-cpp-sys-2" +version = "0.1.159" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a96d43d9370ff79293fe9777c9a4bda9d3bacb7100b61ed1e3690df39e9ad9e" +dependencies = [ + "bindgen", + "cc", + "cmake", + "find_cuda_helper", + "glob", + "walkdir", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -2211,7 +2850,7 @@ dependencies = [ "itoa", "log", "md-5", - "nom", + "nom 8.0.0", "rand 0.10.3", "rangemap", "rayon", @@ -2221,12 +2860,40 @@ dependencies = [ "weezl", ] +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + [[package]] name = "mach2" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "markup5ever" version = "0.39.0" @@ -2238,6 +2905,16 @@ dependencies = [ "web_atoms", ] +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "md-5" version = "0.11.0" @@ -2269,6 +2946,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "minisign" version = "0.10.0" @@ -2319,6 +3002,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -2373,6 +3078,36 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.9.0" @@ -2409,6 +3144,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -2445,6 +3190,15 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2462,6 +3216,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2741,6 +3504,15 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "ogg" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdab8dcd8d4052eaacaf8fb07a3ccd9a6e26efadb42878a413c68fc4af1dee2b" +dependencies = [ + "byteorder", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2749,30 +3521,83 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "opennote" -version = "0.1.0" +version = "0.1.0-beta.5" dependencies = [ "base64 0.22.1", + "base64 0.23.1", + "crc32fast", + "flate2", + "getrandom 0.3.4", "log", + "notify", + "opennote-api", "opennote-core", + "opennote-crashreport", + "opennote-diagnostics", + "opennote-intel", + "opennote-intel-engines", + "opennote-interop", + "opennote-media", + "opennote-metafile", + "opennote-print", + "opennote-search", "opennote-updater", "proptest", + "rcgen", + "rusqlite", + "rustls", "semver", "serde", "serde_json", + "sha1 0.11.0", "sha2 0.11.0", "tauri", "tauri-build", "tempfile", "ts-rs", "ureq", + "url", "webview2-com", "windows", + "windows-core", + "zeroize", +] + +[[package]] +name = "opennote-api" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "getrandom 0.3.4", + "image", + "log", + "qrcodegen", + "rcgen", + "rustls", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "windows", +] + +[[package]] +name = "opennote-cli" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "opennote-api", + "serde_json", + "tempfile", + "windows", ] [[package]] name = "opennote-core" version = "0.1.0" dependencies = [ + "argon2", + "chacha20poly1305", "crc32fast", "flate2", "getrandom 0.3.4", @@ -2788,6 +3613,20 @@ dependencies = [ "unicode-normalization", "unicode-properties", "windows-sys 0.61.2", + "zeroize", +] + +[[package]] +name = "opennote-crashreport" +version = "0.1.0" +dependencies = [ + "proptest", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.21", + "windows-sys 0.61.2", ] [[package]] @@ -2803,6 +3642,111 @@ dependencies = [ "tempfile", ] +[[package]] +name = "opennote-diagnostics" +version = "0.1.0" +dependencies = [ + "libc", + "opennote-core", + "opennote-crashreport", + "opennote-updater", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.21", + "windows-sys 0.61.2", +] + +[[package]] +name = "opennote-intel" +version = "0.1.0" +dependencies = [ + "opennote-intel", + "proptest", + "serde", + "serde_json", + "thiserror 2.0.21", + "windows", + "windows-collections", +] + +[[package]] +name = "opennote-intel-engines" +version = "0.1.0" +dependencies = [ + "ndarray 0.16.1", + "opennote-intel", + "ort", + "serde", + "serde_json", + "tempfile", + "tokenizers", + "whisper-rs", +] + +[[package]] +name = "opennote-interop" +version = "0.1.0" +dependencies = [ + "argon2", + "base64 0.23.1", + "chacha20poly1305", + "crc32fast", + "encoding_rs", + "flate2", + "getrandom 0.3.4", + "md-5", + "opennote-core", + "opennote-interop", + "opennote-metafile", + "opennote-search", + "proptest", + "pulldown-cmark", + "quick-xml", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.21", + "zeroize", +] + +[[package]] +name = "opennote-llm" +version = "0.1.0" +dependencies = [ + "llama-cpp-2", + "opennote-intel-engines", + "serde_json", +] + +[[package]] +name = "opennote-media" +version = "0.1.0" +dependencies = [ + "cpal", + "ogg", + "opennote-core", + "opus", + "proptest", + "rtrb", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "windows-sys 0.61.2", +] + +[[package]] +name = "opennote-metafile" +version = "0.1.0" +dependencies = [ + "image", + "kurbo", + "thiserror 2.0.21", +] + [[package]] name = "opennote-perf" version = "0.1.0" @@ -2813,6 +3757,30 @@ dependencies = [ "tempfile", ] +[[package]] +name = "opennote-print" +version = "0.1.0" +dependencies = [ + "craft-print", + "serde", + "serde_json", +] + +[[package]] +name = "opennote-search" +version = "0.1.0" +dependencies = [ + "opennote-core", + "proptest", + "regex", + "rusqlite", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.21", + "unicode-normalization", +] + [[package]] name = "opennote-spikes" version = "0.0.0" @@ -2840,6 +3808,8 @@ dependencies = [ "minisign-verify", "opennote-updater", "proptest", + "rcgen", + "rustls", "self-replace", "semver", "serde", @@ -2916,6 +3886,30 @@ dependencies = [ "cmake", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray 0.17.2", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "pango" version = "0.18.3" @@ -2964,6 +3958,27 @@ dependencies = [ "windows-link", ] +[[package]] +name = "password-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +dependencies = [ + "phc", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -2974,6 +3989,25 @@ dependencies = [ "hmac", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -2989,6 +4023,16 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", +] + [[package]] name = "phf" version = "0.13.1" @@ -3093,6 +4137,25 @@ dependencies = [ "miniz_oxide 0.8.9", ] +[[package]] +name = "poly1305" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" +dependencies = [ + "cpufeatures 0.3.1", + "universal-hash", +] + +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -3138,6 +4201,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -3219,12 +4292,29 @@ dependencies = [ "unarray", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags 2.13.2", + "memchr", + "unicase", +] + [[package]] name = "pxfm" version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" +[[package]] +name = "qrcodegen" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4339fc7a1021c9c1621d87f5e3505f2805c8c105420ba2f2a4df86814590c142" + [[package]] name = "quick-error" version = "1.2.3" @@ -3241,23 +4331,80 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.47" +name = "quinn" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ - "proc-macro2", + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", ] [[package]] -name = "r-efi" -version = "5.3.0" +name = "quinn-proto" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "r-efi" -version = "6.0.0" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" @@ -3292,6 +4439,12 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + [[package]] name = "rand_core" version = "0.9.5" @@ -3307,6 +4460,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -3328,6 +4490,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -3338,6 +4506,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.13.0" @@ -3348,6 +4527,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3424,21 +4616,32 @@ checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ "base64 0.23.1", "bytes", + "encoding_rs", + "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", + "mime", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", + "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -3450,6 +4653,26 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + [[package]] name = "rpassword" version = "7.5.4" @@ -3461,6 +4684,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.21", +] + [[package]] name = "rtoolbox" version = "0.0.6" @@ -3471,6 +4704,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rtrb" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fae8ee26b0371a29a77d2b2d6b3ae13aa81def6f9bf1b1b92a32d279a5e709b7" + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags 2.13.2", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3499,15 +4753,83 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ + "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.22.4", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -3526,6 +4848,18 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "ryu-js" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714" + [[package]] name = "salsa20" version = "0.10.2" @@ -3628,7 +4962,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags 2.13.2", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -3750,6 +5084,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_json_canonicalizer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe52319a927259afbfa5180c5157cd8167edfd3e8c254f9558c7fef44c5649f2" +dependencies = [ + "ryu-js", + "serde", + "serde_json", +] + [[package]] name = "serde_repr" version = "0.1.21" @@ -3779,6 +5124,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_with" version = "3.24.0" @@ -3806,7 +5163,7 @@ version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" dependencies = [ - "darling", + "darling 0.24.1", "proc-macro2", "quote", "syn 3.0.6", @@ -3843,6 +5200,28 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -3866,10 +5245,185 @@ dependencies = [ ] [[package]] -name = "shlex" -version = "2.0.1" +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "sigstore-bundle" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7dc53c8a941858d01479622dbb7650aaa2ee0ca1fb58fb6cdddbfc3064abbb" +dependencies = [ + "base64 0.22.1", + "hex", + "serde", + "serde_json", + "sigstore-crypto", + "sigstore-merkle", + "sigstore-rekor", + "sigstore-tsa", + "sigstore-types", + "thiserror 2.0.21", +] + +[[package]] +name = "sigstore-crypto" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da90c3d9af638898a5fdc347479b18f950bb0dde11ecf2244f94cd28135da53b" +dependencies = [ + "aws-lc-rs", + "base64 0.22.1", + "const-oid 0.9.6", + "der 0.7.10", + "digest 0.10.7", + "pem", + "rand_core 0.10.1", + "sha2 0.10.9", + "signature", + "sigstore-types", + "spki", + "thiserror 2.0.21", + "tracing", + "x509-cert", +] + +[[package]] +name = "sigstore-merkle" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c0d53558825c77716855c13794306fff766854c1bf92948e77d7890da3f2455" +dependencies = [ + "base64 0.22.1", + "hex", + "sigstore-crypto", + "sigstore-types", + "thiserror 2.0.21", +] + +[[package]] +name = "sigstore-rekor" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "641fdaaa40d0cd6e249cf1671c2240beff1a3ece578062cf43e750779e8db2b3" +dependencies = [ + "base64 0.22.1", + "hex", + "reqwest", + "serde", + "serde_json", + "sigstore-crypto", + "sigstore-merkle", + "sigstore-types", + "thiserror 2.0.21", + "url", +] + +[[package]] +name = "sigstore-trust-root" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6b0d2550f05c096400066e858db13d91f9de3b750a7541fb1291c1dc80ca290" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "rustls-pki-types", + "serde", + "serde_json", + "sigstore-crypto", + "sigstore-types", + "thiserror 2.0.21", + "x509-cert", +] + +[[package]] +name = "sigstore-tsa" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bc86c8abc1ece6832236e7e34e9baebb5d4d40490c0332b814a0d8624ca7255" +dependencies = [ + "aws-lc-rs", + "base64 0.22.1", + "chrono", + "cmpv2", + "cms", + "const-oid 0.9.6", + "der 0.7.10", + "hex", + "rand 0.10.3", + "reqwest", + "rustls-pki-types", + "rustls-webpki", + "sigstore-crypto", + "sigstore-types", + "thiserror 2.0.21", + "tracing", + "x509-cert", + "x509-tsp", +] + +[[package]] +name = "sigstore-types" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8d870c9bcfdf83396ac2bd2d6a23c5d09ec02cc9fda50fa1cbb3dd71a9bee53" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "pem", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "sigstore-verify" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +checksum = "d30aff69036e4579b416956e9ba68cec45b793116f78f10842d04c7f8b924bab" +dependencies = [ + "base64 0.22.1", + "chrono", + "cms", + "const-oid 0.9.6", + "hex", + "pem", + "rustls-pki-types", + "rustls-webpki", + "serde", + "serde_json", + "serde_json_canonicalizer", + "sigstore-bundle", + "sigstore-crypto", + "sigstore-merkle", + "sigstore-rekor", + "sigstore-trust-root", + "sigstore-tsa", + "sigstore-types", + "thiserror 2.0.21", + "tls_codec", + "tracing", + "x509-cert", +] [[package]] name = "simd-adler32" @@ -3921,6 +5475,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "softbuffer" version = "0.4.8" @@ -3969,12 +5534,52 @@ dependencies = [ "system-deps", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der 0.7.10", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "string_cache" version = "0.9.0" @@ -4016,6 +5621,27 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "subtle" version = "2.6.1" @@ -4085,6 +5711,27 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.2", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-deps" version = "6.2.2" @@ -4106,7 +5753,7 @@ checksum = "f37f381f4e048e6cdf038b5705f8cf14ad108279d46eb968140a7b291aba9400" dependencies = [ "bitflags 2.13.2", "block2", - "core-foundation", + "core-foundation 0.10.1", "core-graphics", "crossbeam-channel", "dbus", @@ -4149,6 +5796,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -4485,6 +6143,60 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex", + "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "paste", + "rand 0.9.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.21", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.53.1" @@ -4499,6 +6211,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -4508,6 +6230,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -4660,9 +6383,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -4670,6 +6405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", ] [[package]] @@ -4740,6 +6476,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -4761,6 +6503,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" @@ -4773,6 +6524,34 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "ureq" version = "3.4.2" @@ -4780,13 +6559,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" dependencies = [ "base64 0.23.1", - "der", + "der 0.8.2", "log", "native-tls", "percent-encoding", "rustls-pki-types", + "socks", "ureq-proto", "utf8-zero", + "webpki-root-certs", "winreg 0.56.0", ] @@ -4851,6 +6632,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -5077,6 +6864,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webview2-com" version = "0.39.1" @@ -5117,6 +6913,29 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2" +[[package]] +name = "whisper-rs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2088172d00f936c348d6a72f488dc2660ab3f507263a195df308a3c2383229f6" +dependencies = [ + "libc", + "whisper-rs-sys", +] + +[[package]] +name = "whisper-rs-sys" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6986c0fe081241d391f09b9a071fbcbb59720c3563628c3c829057cf69f2a56f" +dependencies = [ + "bindgen", + "cfg-if", + "cmake", + "fs_extra", + "semver", +] + [[package]] name = "winapi" version = "0.3.9" @@ -5247,6 +7066,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -5532,6 +7362,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "winprint" +version = "0.2.1" +source = "git+https://github.com/storytold/winprint.rs?rev=5f80b9baf6b58937c401656634e42d33f00ddf49#5f80b9baf6b58937c401656634e42d33f00ddf49" +dependencies = [ + "flate2", + "fmt-derive", + "reqwest", + "scopeguard", + "sigstore-trust-root", + "sigstore-types", + "sigstore-verify", + "strum", + "tar", + "thiserror 2.0.21", + "windows", + "windows-core", + "windows-implement", + "xml", +] + [[package]] name = "winreg" version = "0.55.0" @@ -5552,6 +7403,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winsafe" +version = "0.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ef0ffc427f045c0cc9ebffd6f4f91153dcd2a1547b5c3c29ee3f541e16e95c6" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -5629,6 +7486,56 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid 0.9.6", + "der 0.7.10", + "sha1 0.10.7", + "signature", + "spki", + "tls_codec", +] + +[[package]] +name = "x509-tsp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5ceece934a21607055b7ac5c25adb56a2ff559804b10705dc674d1d838c15e1" +dependencies = [ + "cmpv2", + "cms", + "der 0.7.10", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "xml" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f45bb2c13fec6a6cb4c0f76a7e94839e110a14ec803ec2940777a94c347bc52" + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.3" @@ -5698,6 +7605,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" @@ -5743,3 +7664,18 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 2f049b0f..a2b9fe88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,19 @@ [workspace] members = [ "app/src-tauri", + "crates/api", + "crates/cli", "crates/core", + "crates/crashreport", + "crates/diagnostics", + "crates/intel", + "crates/intel-engines", + "crates/llm-host", + "crates/interop", + "crates/media", + "crates/print", + "crates/third_party/metafile", + "crates/search", "crates/updater", "spikes/harness", "tests/crash", @@ -9,9 +21,20 @@ members = [ "tests/perf/frame-probe", ] # The fuzz crate needs a nightly toolchain, so it stays out of the workspace. -exclude = ["crates/core/fuzz"] +exclude = ["crates/core/fuzz", "crates/interop/fuzz"] resolver = "2" +# SQLite is C code, so building it without optimization makes the search tests several times slower. +[profile.dev.package.libsqlite3-sys] +opt-level = 3 + +# Argon2id fills 64 MiB for each password lock, which takes many seconds unoptimized. +[profile.dev.package.argon2] +opt-level = 3 + +[profile.dev.package.blake2] +opt-level = 3 + # Versions the app and the updater crate share, so both always build against the same ones. [workspace.dependencies] log = "0.4" diff --git a/README.md b/README.md index b62806b2..412e8e30 100644 --- a/README.md +++ b/README.md @@ -4,32 +4,68 @@ OpenNote is an open-source note-taking app that combines your favorite writing, drawing, and recording features for professional and personal use. Windows comes first; macOS, Linux, iOS, and Android follow. -**New here?** The [project wiki](https://github.com/XrxcGH/OpenNote/wiki) holds the user guide, development updates, and the developer guide. +**New here?** The [help pages](docs/help/README.md) explain each part of the app. The [project wiki](https://github.com/XrxcGH/OpenNote/wiki) holds the user guide, development updates, and the developer guide. ## Status -OpenNote is in early development. Each phase of the [development plan](docs/DEVELOPMENT.md#5-phases) gets its own branch and merges into `main` when it meets its exit gate. +OpenNote is in beta testing on Windows. Each phase of the [development plan](docs/DEVELOPMENT.md#5-phases) gets its own branch and merges into `main` when it meets its exit gate. Phases 5 to 14 are built and wait together on branch `beta` for their hand tests. -| Phase | Focus | Status | -| --------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | -| 0 | Project foundation | Done and merged | -| 1 | Spikes that test ink, text, PDF export, and audio | Done and merged | -| 2 | [App shell and navigation](https://github.com/XrxcGH/OpenNote/pull/8) | Done and merged 2026-10-02 | -| 3 | [Document model and storage](https://github.com/XrxcGH/OpenNote/pull/9) | Done and merged 2026-10-03 | -| 4 | [Typed notes (text, Markdown, tables, code)](https://github.com/XrxcGH/OpenNote/pull/15) | Done and merged 2026-10-03 | -| [5 to 14](docs/DEVELOPMENT.md#5-phases) | Ink, export, charts, search, audio, math, import, intelligence, hardening, and release | In progress, being wired into the app; a first beta build exists on branch `beta` | +| Phase | Focus | Status | +| ----------------------------------------- | ---------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| 0 | Project foundation | Done and merged | +| 1 | Spikes that test ink, text, PDF export, and audio | Done and merged | +| 2 | [App shell and navigation](https://github.com/XrxcGH/OpenNote/pull/8) | Done and merged 2026-10-02 | +| 3 | [Document model and storage](https://github.com/XrxcGH/OpenNote/pull/9) | Done and merged 2026-10-03 | +| 4 | [Typed notes (text, Markdown, tables, code)](https://github.com/XrxcGH/OpenNote/pull/15) | Done and merged 2026-10-03 | +| [5 to 12](docs/DEVELOPMENT.md#5-phases) | Ink, page views and export, tables and charts, search, audio, math, import, intelligence | Built, and in beta testing on branch `beta` ([pull request #16](https://github.com/XrxcGH/OpenNote/pull/16), 0.1.0-beta.5) | +| [13 and 14](docs/DEVELOPMENT.md#5-phases) | Hardening and privacy; the release pipeline | Built, waiting on the beta period, code signing, and the update key, which are the owner's | + +Agents compiled and unit-tested beta 4 and started the real app. Nobody has used most of it by hand yet. [docs/FEATURES.md](docs/FEATURES.md) marks every feature as built, built but untested by hand, not built yet, or needing the owner. The [changelog](CHANGELOG.md) lists what each beta added. ## What the app does today -OpenNote runs as a Windows desktop app. You can organize notebooks, sections, and pages, and write typed notes on a page, either in a simple document flow or in text boxes you place anywhere. Pages save on their own and recover after a crash. Typed notes support: +OpenNote runs as a Windows desktop app. You organize notebooks, sections, and pages in a notes folder you choose, as plain folders in OpenNote's open note format. Pages save on their own and recover after a crash. Each area below has a help page. + +| Area | What you can do | +| ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| [Typed notes](docs/help/typed-notes.md) | Text boxes, headings, lists, checklists, tables, code, callouts, Markdown, templates, and find and replace | +| [Ink and palm rejection](docs/help/ink-and-palm-rejection.md) | Pens, a highlighter, erasers, the lasso, shapes, and gestures, with your palm resting on the screen | +| [Paper and page views](docs/help/paper-and-page-views.md) | Flow, paginated, and canvas pages, with lined, grid, and dot paper that text sits on; slides, print, and PDF | +| [Tables and charts](docs/help/tables-and-charts.md) | Smart tables with views and formulas, and charts that update with them | +| [Math and graphs](docs/help/math-and-graphs.md) | Equations, quick math, the grapher, Mermaid diagrams, and mind maps | +| [Search and links](docs/help/search-and-links.md) | Search, page links, backlinks, tags, daily notes, and the graph view | +| [Study tools](docs/help/study-tools.md) | Flashcards, Anki, study tape, Upcoming, timers, a calculator, and citations | +| [Recording](docs/help/recording.md) | Recording with stamped notes, trim, transcript blocks, speakers, and meeting recaps | +| [Import and export](docs/help/import-and-export.md) | Obsidian, Markdown, Word, Excel, PowerPoint, and more in; Markdown, Word, PDF, and more out | +| [On-device intelligence](docs/help/on-device-intelligence.md) | Text in pictures, read aloud, summaries, Ask your notes, and writing tools, each off until turned on | +| [Privacy and Work offline](docs/help/privacy-and-work-offline.md) | Crash reports with consent, a self-check, a feedback file, safe start, and a switch that stops all network use | +| [Connectors](docs/help/connectors.md) | Signing in to other services; no feature uses a connection yet | + +![A page of typed notes with a heading, a list, and a page link](docs/screens/workspace-light.png) + +| Ink | Lined paper | Tables and charts | +| ------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| ![Ink on a page: a highlighter, three pens, and a pencil sketch](docs/screens/ink-pens-light.png) | ![Typed notes on lined paper, with the text sitting on the rules](docs/screens/lined-paper-light.png) | ![A smart table with a formula cell and the bar chart made from it](docs/screens/table-chart-light.png) | + +| Math and graphs | Search | Recording | +| ---------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| ![An inline equation in a sentence and a graph block with its function](docs/screens/math-graph-light.png) | ![The search panel with a result, the matched word in bold, and a preview](docs/screens/search-light.png) | ![The recording options under the Record button](docs/screens/recording-light.png) | + +Every screen is in [docs/screens](docs/screens/README.md), in light and dark. Not built yet in beta 4: the pen library, PDF import, the local API, the web clipper, and the speech engine for transcripts. The full list is in [docs/FEATURES.md](docs/FEATURES.md). + +## Try the beta + +There is no public release yet. The beta is one unsigned file, `OpenNote_Windows64.exe`, with no installer and no self-update. It is built from branch `beta`, as the [releasing guide](docs/RELEASING.md) describes for a release: + +```sh +git switch beta +npm install +npx tauri build --no-bundle --target x86_64-pc-windows-msvc +``` -- Text boxes, headings, lists, checkboxes, bold and italic, links, quotes, and callouts -- Markdown shortcuts as you type (for example `# ` for a heading), and Markdown paste -- Images by paste, drag, or file picker, with resize and crop -- Tables, and code blocks with syntax highlighting -- Spell check through the Windows spelling service +The first build needs CMake for the audio encoder. [CONTRIBUTING.md](CONTRIBUTING.md) says where to find it. When you run the file, Windows SmartScreen warns about an unknown publisher. Choose More info, then Run anyway. -Ink, export, charts, search, audio, math, and import are still being wired into the app. +Test with a throwaway profile so your real notes stay safe. Set `OPENNOTE_PROFILE_DIR` to an empty folder before you start the exe. The [beta 4 hand test checklist](docs/testing/beta-4-checklist.md) has the exact commands and a list of steps for every area. Report a problem with Help, then Send feedback, which builds a file with the log and a self-check. ## Downloads @@ -67,26 +103,40 @@ cargo test --workspace # Rust tests ## Repository map ```text -app/ The desktop app (Phases 2 and 4) +app/ The desktop app src/ TypeScript and React interface + core/ Interface-side models: ink, audio, page layout and rules editor/ The typed notes editor: schema, Markdown, tables, and highlighting - features/ - page/ The page view: text boxes, images, paste, and spell check - services/ - pages/ The page service contract, with its Tauri and in-memory versions - src-tauri/ Rust shell, built on Tauri 2 -brand/ Design tokens, logo, and app icon + features/ One folder per feature, each with a README: page, pages (views, paper, export), ink, + tables, math, study, tools, search, audio, interop, intel, diagnostics, connectors, + qol (shell extras), tree, settings, setup, and more + services/ The notes and page service contracts, with their Tauri and in-memory versions + shell/ The window, panes, command bar, and title bar + strings/ Every word the interface shows, ready for translation + src-tauri/ Rust shell, built on Tauri 2: commands for notes, search, audio, import and export, + intelligence, diagnostics, connectors, and the shell extras +brand/ Design tokens, logo, app icon, and the social preview checks/ The CHECKS quality gate -crates/ Rust libraries (Phase 3) - core/ Document model, file format, storage, undo, and search +crates/ Rust libraries + core/ Document model, file format, storage, undo, and search (Phase 3) + crashreport/ Crash reports with consent and scrubbing (Phase 13) + diagnostics/ The self-check and the feedback file (Phase 13) + intel/ On-device intelligence through Windows (Phase 12) + interop/ Import and export of other apps' formats (Phase 11) + media/ Recording, Opus encoding, and playback (Phase 9) + search/ The search index (Phase 8) updater/ Self-updater (fetch, verify, install, rollback) -docs/ Plans, specs, and guides - adr/ Architecture decision records, including Phase 4's 0020 to 0026 +docs/ Plans, specs, guides, and pictures + adr/ Architecture decision records design/ Screen wireframes and the generator that draws them format/ Note file format specification and tools - perf/ Performance records, starting with Phase 4's typing numbers + help/ Plain guides for using the app, one for each area + perf/ Performance records for the phases + releases/ Release sign-off files + screens/ Real screenshots of every screen, in light and dark + testing/ Hand test checklists: the beta, palm rejection, and keyboard and screen reader spikes/ Throwaway Phase 1 experiments and their results -tests/ End-to-end, UI, performance, and crash tests (Phase 3) +tests/ End-to-end, UI, performance, and crash tests crash/ Crash safety tests that kill the app during saves e2e/ Real-app tests: the smoke set (keyboard, screen reader) and typed notes perf/ Performance benchmarks for every budget @@ -99,23 +149,29 @@ The root also holds tool settings such as `package.json`, `Cargo.toml`, and `che ## Documentation -The [documentation index](docs/README.md) lists everything in `docs/`. Start with the wiki for guides, or pick a document below. - -| Document | What it covers | -| ------------------------------------------------------- | -------------------------------------------------------------------------------------- | -| [Project wiki](https://github.com/XrxcGH/OpenNote/wiki) | The user guide, development updates, and the developer guide | -| [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md) | The step-by-step plan for the Windows prototype, with tests and release gates | -| [docs/FEATURES.md](docs/FEATURES.md) | The feature spec: Office and Google, handwriting, transcripts, study tools, and more | -| [docs/RESEARCH.md](docs/RESEARCH.md) | Popular and rising note apps, the features people love, and the gaps OpenNote can fill | -| [docs/BRAND.md](docs/BRAND.md) | Colors, type, layout, motion, accessibility, and performance budgets | -| [docs/design/SCREENS.md](docs/design/SCREENS.md) | Wireframes of each major screen, with sizes and keep-out zones | -| [docs/CHECKS.md](docs/CHECKS.md) | The quality gate every file change must pass | -| [docs/format/README.md](docs/format/README.md) | The note file format specification and tools (Phase 3) | -| [docs/adr/README.md](docs/adr/README.md) | The architecture decision records (ADRs), and how to write one | -| [docs/RELEASING.md](docs/RELEASING.md) | The maintainer guide to cutting, checking, and rolling back a release | -| [CONTRIBUTING.md](CONTRIBUTING.md) | How to set up, make a change, and get it merged | -| [SECURITY.md](SECURITY.md) | How to report a security problem privately | -| [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) | How we treat each other | +The [documentation index](docs/README.md) lists everything in `docs/`. Start with the help pages to use the app, or pick a document below. + +| Document | What it covers | +| -------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| [Help pages](docs/help/README.md) | How to use each part of the app, with screenshots | +| [Project wiki](https://github.com/XrxcGH/OpenNote/wiki) | The user guide, development updates, and the developer guide | +| [CHANGELOG.md](CHANGELOG.md) | What each beta added, changed, and fixed | +| [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md) | The step-by-step plan for the Windows prototype, with a beta 4 status under each phase | +| [docs/FEATURES.md](docs/FEATURES.md) | The feature spec, with what is built, untested, not built, or waiting on the owner | +| [docs/RESEARCH.md](docs/RESEARCH.md) | Popular and rising note apps, the features people love, and the gaps OpenNote can fill | +| [docs/BRAND.md](docs/BRAND.md) | Colors, type, layout, motion, accessibility, and performance budgets | +| [docs/design/SCREENS.md](docs/design/SCREENS.md) | Wireframes of each major screen, with sizes and keep-out zones | +| [docs/screens/README.md](docs/screens/README.md) | Real screenshots of every screen, in light and dark | +| [docs/CHECKS.md](docs/CHECKS.md) | The quality gate every file change must pass | +| [docs/format/README.md](docs/format/README.md) | The note file format specification and tools | +| [docs/adr/README.md](docs/adr/README.md) | Every architecture decision record (ADR), its status, and how to write one | +| [docs/testing/beta-4-checklist.md](docs/testing/beta-4-checklist.md) | The hand test checklist for beta 4, starting with the safest way to run it | +| [docs/HARDENING.md](docs/HARDENING.md) | Crash reports, the self-check, the feedback file, and safe start | +| [docs/RELEASING.md](docs/RELEASING.md) | The maintainer guide to cutting, checking, and rolling back a release | +| [docs/CONNECTORS.md](docs/CONNECTORS.md) | Registering OpenNote with each connected service, and where the client IDs go | +| [CONTRIBUTING.md](CONTRIBUTING.md) | How to set up, make a change, and get it merged | +| [SECURITY.md](SECURITY.md) | How to report a security problem privately | +| [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) | How we treat each other | ## License diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 00000000..e02246c4 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,181 @@ +# Third-party notices + +OpenNote is licensed under the Apache License 2.0 (see LICENSE). It includes the following third-party material. This file is shown in the app under Settings, then About, then Open-source notices. The release also lists every npm package and Rust crate that is built into the exes in its software bill of materials, `OpenNote.sbom.cdx.json`. This file names the code that OpenNote depends on by a pinned source revision rather than a registry release, and code copied into this repository, and keeps the notices that code asks to be kept. Copied code also keeps its original notices beside it. + +## Code pinned to a source revision or copied into this repository + +| Component | Used for | Source and revision | License | +|---|---|---|---| +| `craft-print` | Sending a PDF to the system's printers: the in-process Windows XPS path and CUPS. Used by the `opennote-print` crate. | `https://github.com/storytold/craft-libs`, commit `8f5a2831614e632f91ad367f7af2a6e5b61b9529` | MIT OR Apache-2.0. OpenNote elects Apache-2.0. | +| `craft-winprint-gdi` (a dependency of `craft-print`) | Windows driver settings and printable geometry | `https://github.com/storytold/winprint.rs`, commit `417ac3c4d14d3da5485a99c7d5ce8f05664ec312` | MIT OR Apache-2.0. OpenNote elects Apache-2.0. | +| `winprint` (a dependency of `craft-print`) | Listing Windows printers and drawing a PDF into a print job | `https://github.com/storytold/winprint.rs`, commit `5f80b9baf6b58937c401656634e42d33f00ddf49`, a fork of `ArcticLampyrid/winprint.rs` | BSD-3-Clause. The notice is below. | +| Metafile reader (WMF, EMF, EMF+) | Drawing the WMF and EMF pictures in imported Office files. Vendored in `crates/third_party/metafile`, with its license text (`LICENSE-APACHE`, `NOTICE.upstream`); changes are listed in `UPSTREAM.md`, and `src/svg.rs` is new. | `crates/metafile` of `https://github.com/storytold/deckcraft`, commit `9859da39`. Copyright (c) 2026 ArtCraft Team and the DeckCraft contributors. | MIT OR Apache-2.0. OpenNote elects Apache-2.0. | + +No code from the printing projects is copied into this repository: Cargo fetches each one at the commit above, and a change to a commit is a reviewed change to this file. The metafile reader is the one copied component. + +## craft-libs + +The text of craft-libs' own NOTICE file, which Apache-2.0 section 4(d) asks to be carried with a redistributed work: + +```text +craft-libs +Copyright (c) 2026 ArtCraft Team and the craft-libs contributors. +Licensed under the MIT License or the Apache License, Version 2.0, at your option +(LICENSE-MIT, LICENSE-APACHE). + +The PDF spooler in crates/print is extracted from: +- PdfCraft, crates/print/src/spool.rs, windows_print.ps1 and spooler tests, + revision 33c14e07cfd742a82bf03baa6b103c01008fa68c. + https://github.com/storytold/pdfcraft + Copyright (c) 2026 ArtCraft Team and the PdfCraft contributors. + Licensed MIT OR Apache-2.0. PDF composition and application code are not copied. + +The Windows printer duplex-capability query and parser additionally draw on PdfCraft, +crates/print/src/spool.rs, revision fb97ce958dfe92b6762ce40615c7836551710139 (PR #1154), under the same licenses above. + +The native Windows backend (crates/print/src/windows.rs, ticket.rs and the printer-list and +Windows spooler tests) is ported from PdfCraft's native printing work, which carries the +approach of PdfCraft PR #599 (Win32 printer list, Windows.Data.Pdf into the XPS print API, the +PrintTicket and its driver-paper preference): +- PdfCraft, crates/print/src/spool/windows.rs, spool.rs and tests.rs, branch + claude/native-windows-print-826c99 of PR #1132, revision a435117 (jcl-bit/printcraft). + https://github.com/storytold/pdfcraft + Copyright (c) 2026 ArtCraft Team and the PdfCraft contributors. + Licensed MIT OR Apache-2.0. +``` + +That file also names craft-libs' other parts (color management, the PDF writer, the PDF reader with its vendored crates and fonts, and the icons). OpenNote depends on the `craft-print` crate only, so none of those is built into OpenNote. + +## winprint + +The fork's `LICENSE.md` is the upstream BSD-3-Clause text unchanged, with the one copyright line below. No source file in the fork has its own copyright or license header, and its README says it retains the upstream notices, so there are no other copyright holders to name. The `pdfium` feature of `winprint` (it downloads PDFium when the crate is built) is off: `craft-print` turns default features off, and nothing in OpenNote enables it. No PDFium is downloaded or bundled. + +```text +BSD 3-Clause License + +Copyright (c) 2021 - 2024, alampy.com + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +## Other crates the print path adds + +`cargo metadata` on the new entries in `Cargo.lock` (72 crates over the previous beta lock) shows only permissive licenses: MIT, Apache-2.0, ISC, BSD-3-Clause, Zlib, BSL-1.0, and CDLA-Permissive-2.0 (the root certificate list in `webpki-root-certs`). None is copyleft. The `cargo deny` check in CI (see `deny.toml`) keeps it that way. + +- Built into the Windows app, on top of what was already there: `winsafe`, `roxmltree`, `xml`, `strum` and `strum_macros`, `fmt-derive` and `fmt-derive-proc`, and `windows-core` and `windows-implement`. All are MIT or MIT OR Apache-2.0. +- Used only while `winprint` builds, never linked into OpenNote: `reqwest` and its TLS stack (`rustls`, `aws-lc-rs`, `aws-lc-sys`, `ring`, `webpki-root-certs`, `quinn`, and others), the `sigstore-*` crates and their X.509 helpers, and `tar`. They exist for the optional PDFium download, which is off. Building them needs CMake, which is why [CONTRIBUTING.md](CONTRIBUTING.md) lists it. + +## Fonts + +The fonts below are bundled in the app, so nothing downloads at run time. They come from the `@fontsource` npm packages, which repackage the fonts unchanged. The fonts are licensed under the SIL Open Font License, Version 1.1, copied in full below. + +- Atkinson Hyperlegible Next. Copyright 2020-2024 The Atkinson Hyperlegible Next Project Authors (https://github.com/googlefonts/atkinson-hyperlegible-next) +- Atkinson Hyperlegible Mono. Copyright 2020-2024 The Atkinson Hyperlegible Mono Project Authors (https://github.com/googlefonts/atkinson-hyperlegible-next-mono) +- Literata. Copyright 2017 The Literata Project Authors (https://github.com/googlefonts/literata) + +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. + +## Software libraries + +The npm packages the interface imports and the Rust crates the app is built from each keep their own license. Their names, versions, licenses, and hashes are listed in the bill of materials (`OpenNote.sbom.cdx.json`) that comes with every release, and in `package-lock.json` and `Cargo.lock`. + +## Development tools + +Release checks build independent document readers from pinned revisions of open-source projects (MIT OR Apache-2.0, used here under Apache-2.0). They run in continuous integration only. Nothing from them is copied into or shipped with OpenNote. See docs/CHECKS.md. diff --git a/app/bundle-budget.json b/app/bundle-budget.json index 45b6ba25..ed11f1fc 100644 --- a/app/bundle-budget.json +++ b/app/bundle-budget.json @@ -1,7 +1,22 @@ { - "startup": { "jsKb": 185, "cssKb": 18 }, - "after": { "within": "features/page/", "entry": "features/page/PageBody" }, + "startup": { "jsKb": 300, "cssKb": 22 }, + "after": { + "within": [ + "features/page/", + "features/pages/", + "features/math/", + "features/tables/", + "editor/extensions/graphFence", + "@observablehq/plot" + ], + "entry": "features/page/PageBody" + }, "lazyKb": { + "mermaid/node_modules/katex": 80, + "node_modules/mermaid": 600, + "@mermaid-js/parser": 130, + "features/pages/live": 40, + "features/pages/ui": 60, "editor/markdown/worker": 45, "editor/highlight": 20, "highlight.js/lib/languages": 5, @@ -9,14 +24,21 @@ "features/page/readAloud": 10, "features/page/styles": 15, "features/page/readingOrder": 15, - "features/page/PageBody": 225, + "features/page/PageBody": 242, + "@observablehq/plot": 135, + "features/math": 175, + "features/tables": 50, + "features/search/links": 195, + "features/search/lineTags": 160, "features/page": 40, + "features/ink/view/install": 110, "features/palette": 40, "features/settings": 40, "features/setup": 40, "features/shortcuts": 40, "features/trash": 40, - "features/updates": 40 + "features/updates": 40, + "features/tools": 260 }, "lazyDefaultKb": 40 } diff --git a/app/gallery.html b/app/gallery.html new file mode 100644 index 00000000..d791295e --- /dev/null +++ b/app/gallery.html @@ -0,0 +1,12 @@ + + + + + + OpenNote component gallery + + +
+ + + diff --git a/app/print.html b/app/print.html new file mode 100644 index 00000000..6e30a748 --- /dev/null +++ b/app/print.html @@ -0,0 +1,10 @@ + + + + + OpenNote print + + + + + diff --git a/app/scripts/check-bundle.ts b/app/scripts/check-bundle.ts index bf740c98..c3d310c5 100644 --- a/app/scripts/check-bundle.ts +++ b/app/scripts/check-bundle.ts @@ -31,10 +31,10 @@ export interface Budget { lazyKb: Record; lazyDefaultKb: number; /** - * Lazy groups that load only after another one: a group whose source path contains `within` counts without the - * files of the group whose path contains `entry`, which is loaded by then. + * Lazy groups that load only after another one: a group whose source path contains `within` (or any of them, when + * it is a list) counts without the files of the group whose path contains `entry`, which is loaded by then. */ - after?: { within: string; entry: string }; + after?: { within: string | readonly string[]; entry: string }; } export interface Group { @@ -63,7 +63,8 @@ export function groups(manifest: Manifest, budget: Budget): Group[] { const keys = Object.keys(manifest); const startup = staticClosure( manifest, - keys.filter((key) => manifest[key].isEntry), + // print.html is the hidden print window's page; it never loads with the app. + keys.filter((key) => manifest[key].isEntry && key !== 'print.html'), ); const startupChunks = [...startup].map((key) => manifest[key]); const result: Group[] = [ @@ -78,7 +79,8 @@ export function groups(manifest: Manifest, budget: Budget): Group[] { const first = budget.after && lazyKeys.find((k) => k.includes(budget.after!.entry)); const loadedFirst = first ? staticClosure(manifest, [first]) : new Set(); for (const key of lazyKeys) { - const follows = first !== undefined && key !== first && key.includes(budget.after!.within); + const follows = + first !== undefined && key !== first && [budget.after!.within].flat().some((fragment) => key.includes(fragment)); const loaded = (k: string) => startup.has(k) || (follows && loadedFirst.has(k)); const own = [...staticClosure(manifest, [key])].filter((k) => !loaded(k)).map((k) => manifest[k]); const limit = Object.entries(budget.lazyKb).find(([fragment]) => key.includes(fragment))?.[1]; diff --git a/app/scripts/dictionary/wordnet.test.ts b/app/scripts/dictionary/wordnet.test.ts new file mode 100644 index 00000000..5742a43f --- /dev/null +++ b/app/scripts/dictionary/wordnet.test.ts @@ -0,0 +1,86 @@ +// The WordNet converter on a small sample written in WordNet's own file layout. +import { describe, expect, it } from 'vitest'; +import { MAX_SENSES, buildChunks, parseData, parseIndex, wordOf } from './wordnet.ts'; + +const LICENSE_HEADER = + ' 1 This software and database is being provided to you, the LICENSEE, by Princeton University...\n'; + +const DATA_NOUN = + LICENSE_HEADER + + // checks-disable-next-line length: a line of WordNet data, which the parser reads as one line + '02084071 05 n 03 dog 0 domestic_dog 0 Canis_familiaris 0 002 @ 02083346 n 0000 ~ 01317541 n 0000 | a member of the genus Canis (probably descended from the common wolf) that has been domesticated by man; "the dog barked all night"\n' + + // checks-disable-next-line length: a line of WordNet data, which the parser reads as one line + '10114209 18 n 02 dog 1 frump 0 001 @ 09624168 n 0000 | a dull unattractive unpleasant girl or woman; "she got a reputation as a frump"; "she\'s a real dog"\n'; +const INDEX_NOUN = LICENSE_HEADER + 'dog n 2 2 @ ~ 2 1 02084071 10114209 \n'; + +const DATA_VERB = + LICENSE_HEADER + + '01990806 38 v 02 run 0 go 5 001 @ 01839234 v 0000 | move fast by using one\'s feet; "he runs fast"\n'; +const INDEX_VERB = LICENSE_HEADER + 'run v 1 1 @ 1 1 01990806 \n'; + +const DATA_ADJ = + LICENSE_HEADER + '01123148 00 a 02 good 0 well(p) 0 001 ! 01123711 a 0101 | having desirable or positive qualities\n'; +const INDEX_ADJ = LICENSE_HEADER + 'good a 1 1 ! 1 1 01123148 \nwell a 1 1 ! 1 0 01123148 \n'; + +const empty = { index: '', data: '' }; + +describe('reading WordNet files', () => { + it('reads a synset: its words, its meaning, and its first example', () => { + const synsets = parseData(DATA_NOUN, 'n'); + expect(synsets.size).toBe(2); + expect(synsets.get('02084071')).toEqual({ + pos: 'n', + words: ['dog', 'domestic dog', 'Canis familiaris'], + definition: + 'a member of the genus Canis (probably descended from the common wolf) that has been domesticated by man', + example: 'the dog barked all night', + }); + expect(synsets.get('10114209')?.example).toBe('she got a reputation as a frump'); + }); + + it('turns underscores into spaces and drops an adjective marker', () => { + expect(wordOf('domestic_dog')).toBe('domestic dog'); + expect(wordOf('well(p)')).toBe('well'); + }); + + it('reads the synsets of a word in the order WordNet lists them', () => { + expect(parseIndex(INDEX_NOUN).get('dog')).toEqual(['02084071', '10114209']); + expect(parseIndex(INDEX_ADJ).get('well')).toEqual(['01123148']); + }); +}); + +describe('making chunks', () => { + const chunks = buildChunks({ + n: { index: INDEX_NOUN, data: DATA_NOUN }, + v: { index: INDEX_VERB, data: DATA_VERB }, + a: { index: INDEX_ADJ, data: DATA_ADJ }, + r: empty, + }); + + it('files each word under its first letter, with the most common sense first', () => { + expect(Object.keys(chunks).sort()).toEqual(['d', 'g', 'r', 'w']); + const dog = chunks.d.dog; + expect(dog).toHaveLength(1); + expect(dog[0][0]).toBe('n'); + expect(dog[0][1][0][1]).toEqual(['domestic dog', 'Canis familiaris']); + expect(dog[0][1][0][2]).toBe('the dog barked all night'); + expect(dog[0][1][1][1]).toEqual(['frump']); + }); + + it('leaves out the word itself from its synonyms, and keeps senses without an example short', () => { + expect(chunks.r.run[0][1][0]).toEqual(["move fast by using one's feet", ['go'], 'he runs fast']); + expect(chunks.g.good[0][1][0]).toEqual(['having desirable or positive qualities', ['well']]); + }); + + it('keeps no more senses than the limit', () => { + const many = Array.from({ length: MAX_SENSES + 3 }, (_, at) => String(at).padStart(8, '0')); + const data = many.map((offset) => `${offset} 00 n 01 thing 0 000 | sense ${offset}`).join('\n'); + const made = buildChunks({ + n: { index: `thing n ${many.length} 0 ${many.length} 0 ${many.join(' ')} \n`, data }, + v: empty, + a: empty, + r: empty, + }); + expect(made.t.thing[0][1]).toHaveLength(MAX_SENSES); + }); +}); diff --git a/app/scripts/dictionary/wordnet.ts b/app/scripts/dictionary/wordnet.ts new file mode 100644 index 00000000..98727a4d --- /dev/null +++ b/app/scripts/dictionary/wordnet.ts @@ -0,0 +1,115 @@ +// Turns Princeton WordNet's database files (the "dict" folder of WordNet 3.1: index.noun, data.noun, and the same for +// verb, adj, and adv) into the dictionary's chunks, one JSON file for each first letter, in +// app/src/features/tools/dictionary/data/en. WordNet's license lets the data ship with an app as long as its notice +// goes with it; app/src/features/tools/dictionary/README.md says where the notice goes. +// +// Run with: node app/scripts/dictionary/wordnet.ts [output folder] +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export type Pos = 'n' | 'v' | 'a' | 'r'; +type Sense = [definition: string, synonyms: string[], example?: string]; +type Chunk = Record; + +/** The most senses kept for a word in one part of speech, and the most synonyms for one sense. */ +export const MAX_SENSES = 8; +export const MAX_SYNONYMS = 12; + +export interface Synset { + pos: Pos; + words: string[]; + definition: string; + example: string | null; +} + +/** A WordNet word as the app writes it: spaces for underscores, no adjective marker such as "(a)". */ +export const wordOf = (raw: string): string => raw.replace(/\((?:a|p|ip)\)$/, '').replace(/_/g, ' '); + +/** The synsets of a data file, by their offset. Header lines (they start with spaces) are skipped. */ +export function parseData(text: string, pos: Pos): Map { + const synsets = new Map(); + for (const line of text.split('\n')) { + if (line === '' || line.startsWith(' ')) continue; + const bar = line.indexOf(' | '); + if (bar < 0) continue; + const fields = line.slice(0, bar).split(' '); + const count = Number.parseInt(fields[3] ?? '', 16); + if (!Number.isFinite(count) || count < 1) continue; + const words = Array.from({ length: count }, (_, at) => wordOf(fields[4 + at * 2] ?? '')).filter(Boolean); + const gloss = line.slice(bar + 3).trim(); + const split = gloss.search(/;\s+"/); + const definition = (split < 0 ? gloss : gloss.slice(0, split)).trim(); + const example = /"([^"]+)"/.exec(split < 0 ? '' : gloss.slice(split))?.[1] ?? null; + synsets.set(fields[0], { pos, words, definition, example }); + } + return synsets; +} + +/** The synsets of each word in an index file, the most common first, as WordNet lists them. */ +export function parseIndex(text: string): Map { + const index = new Map(); + for (const line of text.split('\n')) { + if (line === '' || line.startsWith(' ')) continue; + const fields = line.split(' '); + const synsetCount = Number(fields[2]); + const pointers = Number(fields[3]); + if (!Number.isInteger(synsetCount) || !Number.isInteger(pointers)) continue; + // After the pointer symbols come two counts (senses, and senses seen in text), then the offsets. + const offsets = fields.slice(4 + pointers + 2, 4 + pointers + 2 + synsetCount); + if (offsets.length > 0) index.set(wordOf(fields[0]), offsets); + } + return index; +} + +export interface Files { + n: { index: string; data: string }; + v: { index: string; data: string }; + a: { index: string; data: string }; + r: { index: string; data: string }; +} + +/** The first-letter chunks for the four parts of speech. */ +export function buildChunks(files: Files): Record { + const chunks: Record = {}; + for (const pos of ['n', 'v', 'a', 'r'] as const) { + const synsets = parseData(files[pos].data, pos); + for (const [lemma, offsets] of parseIndex(files[pos].index)) { + const senses: Sense[] = []; + for (const offset of offsets.slice(0, MAX_SENSES)) { + const synset = synsets.get(offset); + if (!synset || synset.definition === '') continue; + const synonyms = [...new Set(synset.words.filter((word) => word.toLowerCase() !== lemma))].slice( + 0, + MAX_SYNONYMS, + ); + senses.push(synset.example ? [synset.definition, synonyms, synset.example] : [synset.definition, synonyms]); + } + if (senses.length === 0) continue; + const letter = /^[a-z]/.test(lemma) ? lemma[0] : '_'; + const chunk = (chunks[letter] ??= {}); + (chunk[lemma] ??= []).push([pos, senses]); + } + } + return chunks; +} + +function main(): void { + const [from, to] = process.argv.slice(2); + if (!from) { + console.error('Usage: node app/scripts/dictionary/wordnet.ts [output folder]'); + process.exit(1); + } + const out = to ?? join(import.meta.dirname, '..', '..', 'src', 'features', 'tools', 'dictionary', 'data', 'en'); + const read = (name: string) => readFileSync(join(from, name), 'utf8'); + const file = (kind: string) => ({ index: read(`index.${kind}`), data: read(`data.${kind}`) }); + const chunks = buildChunks({ n: file('noun'), v: file('verb'), a: file('adj'), r: file('adv') }); + mkdirSync(out, { recursive: true }); + let words = 0; + for (const [letter, chunk] of Object.entries(chunks)) { + words += Object.keys(chunk).length; + writeFileSync(join(out, `${letter}.json`), `${JSON.stringify(chunk)}\n`); + } + console.log(`${words} words in ${Object.keys(chunks).length} files, written to ${out}`); +} + +if (process.argv[1] && import.meta.filename === process.argv[1]) main(); diff --git a/app/scripts/release/README.md b/app/scripts/release/README.md new file mode 100644 index 00000000..6d8881ef --- /dev/null +++ b/app/scripts/release/README.md @@ -0,0 +1,95 @@ +# Release scripts + +These scripts make everything a release needs apart from the exes. That is the signatures, the update manifest, the release notes, the software bill of materials, the winget files, and the release checklist. [The release workflow](../../../.github/workflows/release.yml) runs them, and each one also runs by hand. The maintainer's guide is [docs/RELEASING.md](../../../docs/RELEASING.md). + +## Contents + +- [How the scripts fit together](#how-the-scripts-fit-together) +- [The scripts](#the-scripts) +- [Running them by hand](#running-them-by-hand) +- [Public API](#public-api) +- [Tests](#tests) +- [What the interface will need](#what-the-interface-will-need) + +## How the scripts fit together + +A release goes through these steps, in this order: + +1. `check-version.ts` stops the release when the tag and the version files differ. +2. `checklist.ts` checks section 10 of the development plan. +3. The build job makes the three exes. `check-release-exe.ts` checks each one. +4. `sbom.ts` lists what is built into the exes. +5. `check-downloads.ts` checks that the package job got exactly the files the build and sbom jobs made. +6. `notes.ts` writes the release notes. +7. `sign.ts` signs each exe for the updater. +8. `write-manifest.ts` writes `latest.json`, or `beta.json` for a prerelease. +9. `checksums.ts` writes `SHA256SUMS.txt`. +10. `verify.ts` checks the finished folder the way a running copy of the app will. +11. `winget.ts` writes the winget files for a stable release. +12. `update-test.ts` lets the last beta's updater install the release, in the workflow's `update-test` job. + +The workflow publishes only after step 10 passes. A dry run does steps 1 to 11 with a throwaway signing key and publishes nothing. + +## The scripts + +| Script | What it does | +|---|---| +| `minisign.ts` | Verifies minisign signatures with Node's crypto, using the same rules as the updater in `crates/updater` | +| `pe.ts` | Tells whether an exe has an Authenticode signature, and asks Windows whether it is valid | +| `sign.ts` | Signs the exes with the Tauri command-line tool, with the real key or a throwaway one | +| `verify.ts` | Checks a release folder: manifest, sizes, hashes, signatures, and the version each signature names | +| `notes.ts` | Writes the release page and the short notes for the update notice | +| `versions.ts` | Compares semantic versions | +| `sbom.ts` | Writes a CycloneDX bill of materials. Its helpers are `sbom-imports.ts`, `sbom-npm.ts`, and `sbom-cargo.ts` | +| `winget.ts` | Writes the three winget manifest files from the release manifest and `app/winget.json` | +| `check-downloads.ts` | Checks the files the package job downloaded against the hashes the build and sbom jobs reported, and refuses any other file | +| `checksums.ts` | Writes `SHA256SUMS.txt` | +| `checklist.ts` | Answers each item of the release checklist. Its helpers are `checklist-doc.ts` and `checklist-signoff.ts` | +| `signoff.ts` | Records one person's sign-off of a checklist item in `docs/releases/.signoff.json`, refusing an entry the checklist would not count | +| `update-test.ts` | The update-from-earlier-beta test: the updater's test app, built from the last beta tag, updates itself to the release from a local server | +| `test-support.ts` | Test helpers: a throwaway signing key and a sample release folder | + +The scripts have no dependencies of their own. They use Node 22.18 or newer, which runs TypeScript files directly, and the tools that `npm ci` installs. + +## Running them by hand + +The npm script `release:check` runs the checklist. Run the others with `node`, from the repository root: + +```sh +npm run release:check -- --report +node app/scripts/release/notes.ts v1.0.0 --markdown notes.md --plain notes.txt +node app/scripts/release/sbom.ts 1.0.0 sbom.cdx.json +node app/scripts/release/sign.ts dist-release v1.0.0 --dry-run --pubkey-out dry-run/update.pub +node app/scripts/release/verify.ts dist-release v1.0.0 --pubkey dry-run/update.pub +``` + +`sign.ts` without `--dry-run` needs the `TAURI_SIGNING_PRIVATE_KEY` environment variable, and stops without it. A dry run ignores that variable and the password variable, so a real key in your shell is never used by mistake. + +## Public API + +Every function is pure unless its description says it reads files or runs a program. + +- `minisign.ts`: `parsePublicKey(text)`, `parseSignature(text)`, `verify(data, signature, key)`, `verifyWithAny(data, signature, keys)`, `trustedFields(comment)`, `onlyField(comment, name)`. Both parsers take the base64 form that `tauri signer` writes or the plain minisign text. +- `pe.ts`: `hasAuthenticode(exe)` and `windowsSignatureStatus(path)`, which runs PowerShell on Windows. +- `sign.ts`: `signRelease({ dir, tag, env, dryRun, pubkeyOut, tauri })`. The `tauri` option replaces the command-line tool in tests. +- `verify.ts`: `verifyRelease({ dir, tag, keys, requireAuthenticode })` returns a list of problems, and nothing when all is well. `keysIn(dir)` reads the committed update keys, and `manifestName(version)` picks `latest.json` or `beta.json`. +- `notes.ts`: `markdownNotes(input)`, `plainNotes(input)`, `changelogSection(text, version)`, `changesBetween(previous, ref)`, `previousTag(tags, tag)`. +- `sbom.ts`: `buildSbom(input)` and `readInput(version, serial, timestamp)`, which runs `cargo metadata`. +- `winget.ts`: `wingetFiles(config, manifest, tag)` returns the file paths and their text. `configProblems(config)` checks `app/winget.json`. +- `check-downloads.ts`: `downloadProblems(dir, expected, options)` reads the folder and returns a list of problems, and nothing when all is well. `expectedFrom(env)` reads the hashes from `BUILD_SHA256` and `SBOM_SHA256`. +- `checklist.ts`: `evaluate(items, context)`, `textReport(results)`, `markdownReport(results, version)`, and `verdict(results, requireAll)`. + +## Tests + +Run them with `npm test`, or `npx vitest run --project unit scripts/release` from the `app` folder. Some tests use real tools. One signs and verifies a release with the Tauri command-line tool. One runs `git` in a temporary repository. One compares a signature made by the Tauri tool with our verifier. A test also reads the real checklist, and fails if an item has no check. Set `WINGET_VALIDATE=1` to run `winget validate` on the generated files. + +## What the interface will need + +Nothing in the app calls these scripts. When the interface is wired up, these are the pieces it reads: + +- **Update notice.** The `notes` field of the manifest is plain text, at most 8 lines and 4096 bytes. The updater passes it on as `Offer.notes`. Show it as text, not Markdown, and show at most 8 lines. +- **What's new.** After an update, the app can show the same notes again. It can read the `notes` it saved with the offer, or the changelog entry for its version. +- **Open-source licenses.** The `OpenNote.sbom.cdx.json` file in each release lists every package with its license. A licenses screen in Settings, then About, can bundle or fetch it. +- **Going back.** The previous exe is kept by the updater. Nothing here changes that. + +A copy installed with winget is a portable package, so it updates itself and winget does not know. See [the winget section of RELEASING.md](../../../docs/RELEASING.md#winget). diff --git a/app/scripts/release/check-downloads.test.ts b/app/scripts/release/check-downloads.test.ts new file mode 100644 index 00000000..485fda75 --- /dev/null +++ b/app/scripts/release/check-downloads.test.ts @@ -0,0 +1,99 @@ +// @vitest-environment node +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { TEST_ENDPOINTS_MARKER } from '../check-release-exe.ts'; +import { RELEASE_FILES } from '../write-manifest.ts'; +import { SBOM_FILE, downloadProblems, expectedFrom, hashOutput, type Expected } from './check-downloads.ts'; + +const dirs: string[] = []; +afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); + +const sha = (text: string) => createHash('sha256').update(text).digest('hex'); +const SBOM = '{"bomFormat":"CycloneDX"}'; +const keys = new Set(); + +/** A folder as the package job downloads it, with the hashes the build and sbom jobs reported. */ +function downloaded(): { dir: string; expected: Expected } { + const dir = mkdtempSync(join(tmpdir(), 'opennote-downloads-')); + dirs.push(dir); + const build: Record = {}; + for (const { target, file } of RELEASE_FILES) { + writeFileSync(join(dir, file), `exe for ${target}`); + build[hashOutput(target)] = sha(`exe for ${target}`); + } + writeFileSync(join(dir, SBOM_FILE), SBOM); + return { dir, expected: { build, sbom: sha(SBOM) } }; +} + +describe('downloadProblems', () => { + it('passes the files the build and sbom jobs made, and nothing else', () => { + const { dir, expected } = downloaded(); + expect(downloadProblems(dir, expected, { keys })).toEqual([]); + }); + + it('refuses an exe that another job of the run put in place of the one the build job made', () => { + const { dir, expected } = downloaded(); + writeFileSync(join(dir, 'OpenNote_Windows64.exe'), 'something else'); + const built = sha('exe for x86_64-pc-windows-msvc'); + expect(downloadProblems(dir, expected, { keys })).toEqual([ + `OpenNote_Windows64.exe isn't the file its job made: its SHA-256 is ${sha('something else')}, not ${built}.`, + ]); + }); + + it('refuses any other file or folder, which would be signed and published with the release', () => { + const { dir, expected } = downloaded(); + writeFileSync(join(dir, 'OpenNote_Setup.exe'), 'extra'); + mkdirSync(join(dir, 'exe-zz')); + expect(downloadProblems(dir, expected, { keys })).toEqual([ + `"OpenNote_Setup.exe" isn't a release file, so no job of the release may add it.`, + `"exe-zz" isn't a release file, so no job of the release may add it.`, + ]); + }); + + it('refuses a missing file, and a file whose job reported no hash', () => { + const { dir, expected } = downloaded(); + rmSync(join(dir, 'OpenNote_Windows32.exe')); + const build = { ...expected.build, [hashOutput('aarch64-pc-windows-msvc')]: '' }; + expect(downloadProblems(dir, { build, sbom: undefined }, { keys })).toEqual([ + 'OpenNote_Windows32.exe is missing.', + 'The job that makes OpenNote_WindowsARM64.exe reported no SHA-256 for it.', + `The job that makes ${SBOM_FILE} reported no SHA-256 for it.`, + ]); + }); + + it('refuses a bill of materials that differs from the one the sbom job made', () => { + const { dir, expected } = downloaded(); + writeFileSync(join(dir, SBOM_FILE), '{}'); + expect(downloadProblems(dir, expected, { keys })).toEqual([ + `${SBOM_FILE} isn't the file its job made: its SHA-256 is ${sha('{}')}, not ${sha(SBOM)}.`, + ]); + }); +}); + +describe('downloadProblems on the exes themselves', () => { + it('checks each exe again for test endpoints, as the build job did', () => { + const { dir, expected } = downloaded(); + const contents = `exe ${TEST_ENDPOINTS_MARKER}`; + writeFileSync(join(dir, 'OpenNote_WindowsARM64.exe'), contents); + const build = { ...expected.build, [hashOutput('aarch64-pc-windows-msvc')]: sha(contents) }; + expect(downloadProblems(dir, { ...expected, build }, { keys })).toEqual([ + `OpenNote_WindowsARM64.exe: It was built with the test-endpoints feature: it contains ${TEST_ENDPOINTS_MARKER}.`, + ]); + }); +}); + +describe('expectedFrom', () => { + it("reads the build job's outputs as JSON, and the hash of the bill of materials", () => { + const env = { BUILD_SHA256: '{\n "sha256-x": "abc"\n}', SBOM_SHA256: 'def' }; + expect(expectedFrom(env)).toEqual({ build: { 'sha256-x': 'abc' }, sbom: 'def' }); + }); + + it('stops when the build outputs are missing or not an object', () => { + expect(() => expectedFrom({})).toThrow('BUILD_SHA256 is not set.'); + expect(() => expectedFrom({ BUILD_SHA256: '[]' })).toThrow('must be a JSON object'); + expect(() => expectedFrom({ BUILD_SHA256: 'null' })).toThrow('must be a JSON object'); + }); +}); diff --git a/app/scripts/release/check-downloads.ts b/app/scripts/release/check-downloads.ts new file mode 100644 index 00000000..fbcaeff8 --- /dev/null +++ b/app/scripts/release/check-downloads.ts @@ -0,0 +1,105 @@ +// Checks the files the package job downloaded, before it signs anything (docs/RELEASING.md, The package job). +// Any job in a run can upload an artifact, including the CI jobs. Their actions and test tools come from outside +// this repository. So the package job signs only the exes the build job made, byte for byte, and nothing else. +// +// Usage: node app/scripts/release/check-downloads.ts +// BUILD_SHA256 holds the build job's outputs as JSON, with a "sha256-" entry for each exe. +// SBOM_SHA256 holds the sbom job's hash of the bill of materials. + +import { createHash } from 'node:crypto'; +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { committedKeys, problems as exeProblems } from '../check-release-exe.ts'; +import { RELEASE_FILES, type ReleaseFile } from '../write-manifest.ts'; + +/** The bill of materials that the sbom job writes. */ +export const SBOM_FILE = 'OpenNote.sbom.cdx.json'; + +/** The build job's output that holds the SHA-256 of the exe built for a Rust target. */ +export const hashOutput = (target: string): string => `sha256-${target}`; + +/** The hashes the jobs reported for the files they made, as lowercase hexadecimal. */ +export interface Expected { + /** The build job's outputs, with a SHA-256 under `hashOutput(target)` for each exe. */ + readonly build: Readonly>; + /** The sbom job's SHA-256 of the bill of materials. */ + readonly sbom: string | undefined; +} + +export interface CheckOptions { + /** The exes the release ships. Defaults to app/release-files.json. */ + readonly files?: readonly ReleaseFile[]; + /** The update keys an exe may hold. Defaults to the committed app/src-tauri/keys. */ + readonly keys?: ReadonlySet; +} + +const SHA256 = /^[0-9a-f]{64}$/; + +const isFile = (path: string): boolean => { + try { + return statSync(path).isFile(); + } catch { + return false; + } +}; + +/** + * What's wrong with the downloaded folder, one sentence each; empty when it holds exactly the files the jobs made. + * Reads files. + */ +export function downloadProblems(dir: string, expected: Expected, options: CheckOptions = {}): string[] { + const files = options.files ?? RELEASE_FILES; + const keys = options.keys ?? committedKeys(); + const wanted = new Map([ + ...files.map(({ target, file }) => [file, { hash: expected.build[hashOutput(target)], exe: true }] as const), + [SBOM_FILE, { hash: expected.sbom, exe: false }], + ]); + const found = readdirSync(dir) + .filter((name) => !wanted.has(name)) + .sort() + // Quoted, so a strange name can't start a workflow command of its own in the log. + .map((name) => `${JSON.stringify(name)} isn't a release file, so no job of the release may add it.`); + for (const [name, { hash, exe }] of wanted) { + const path = join(dir, name); + if (!isFile(path)) { + found.push(`${name} is missing.`); + continue; + } + if (typeof hash !== 'string' || !SHA256.test(hash)) { + found.push(`The job that makes ${name} reported no SHA-256 for it.`); + continue; + } + const contents = readFileSync(path); + const actual = createHash('sha256').update(contents).digest('hex'); + if (actual !== hash) { + found.push(`${name} isn't the file its job made: its SHA-256 is ${actual}, not ${hash}.`); + } else if (exe) { + found.push(...exeProblems(contents, keys).map((problem) => `${name}: ${problem}`)); + } + } + return found; +} + +/** The hashes from the variables the package job sets. */ +export function expectedFrom(env: Readonly>): Expected { + if (env.BUILD_SHA256 === undefined) throw new Error('BUILD_SHA256 is not set.'); + const build: unknown = JSON.parse(env.BUILD_SHA256); + if (typeof build !== 'object' || build === null || Array.isArray(build)) { + throw new Error('BUILD_SHA256 must be a JSON object of the build job outputs.'); + } + return { build: build as Record, sbom: env.SBOM_SHA256 }; +} + +function main(): void { + const dir = process.argv[2]; + if (!dir) throw new Error('Usage: node app/scripts/release/check-downloads.ts '); + const found = downloadProblems(dir, expectedFrom(process.env)); + found.forEach((problem) => console.error(`::error::${problem}`)); + if (found.length > 0) process.exitCode = 1; + else + console.log( + `${dir} holds the ${RELEASE_FILES.length} exes and the bill of materials the jobs made, and nothing else.`, + ); +} + +if (import.meta.main) main(); diff --git a/app/scripts/release/checklist-doc.ts b/app/scripts/release/checklist-doc.ts new file mode 100644 index 00000000..e2d9142e --- /dev/null +++ b/app/scripts/release/checklist-doc.ts @@ -0,0 +1,39 @@ +// Reads the release checklist and the device test matrix out of the development plan, so the script checks what +// the document says, and the two can't drift apart. If an item is added to the document, the checklist script fails +// until a check for it exists. + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export const PLAN_PATH = 'docs/DEVELOPMENT.md'; + +/** The text between a `## ` heading that starts with `title` and the next `## ` heading. */ +export function sectionOf(markdown: string, title: string): string { + const lines = markdown.split(/\r?\n/); + const start = lines.findIndex((line) => line.startsWith('## ') && line.slice(3).startsWith(title)); + if (start === -1) throw new Error(`${PLAN_PATH} has no section titled "${title}".`); + const end = lines.findIndex((line, at) => at > start && line.startsWith('## ')); + return lines.slice(start + 1, end === -1 ? undefined : end).join('\n'); +} + +/** The checklist items of section 10, with the check box and Markdown marks removed. */ +export function checklistItems(markdown: string): string[] { + const items = [...sectionOf(markdown, '10. Release checklist').matchAll(/^- \[[ xX]\] (.+)$/gm)]; + return items.map((match) => match[1].replace(/`/g, '').trim()); +} + +/** The device types in the first column of the device test matrix, section 7. */ +export function deviceTypes(markdown: string): string[] { + const rows = sectionOf(markdown, '7. Device test matrix') + .split('\n') + .filter((line) => line.startsWith('|')); + return rows + .slice(2) + .map((row) => row.split('|')[1]?.trim() ?? '') + .filter(Boolean); +} + +/** Reads the development plan from a repository. */ +export function readPlan(root: string): string { + return readFileSync(join(root, PLAN_PATH), 'utf8'); +} diff --git a/app/scripts/release/checklist-signoff.ts b/app/scripts/release/checklist-signoff.ts new file mode 100644 index 00000000..40b243e2 --- /dev/null +++ b/app/scripts/release/checklist-signoff.ts @@ -0,0 +1,78 @@ +// The sign-off file for a release: where a person records the checklist items that only a person can check, such as +// pen testing on real devices. It is committed as docs/releases/.signoff.json before the release is +// tagged, so the record stays with the code it covers and the workflow can read it. + +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +/** The checklist items that need a person. The keys are the ids in the sign-off file. */ +export const MANUAL_ITEMS = ['budgets', 'accessibility', 'pen', 'upgrade', 'format', 'updates', 'docs'] as const; +export type ManualItem = (typeof MANUAL_ITEMS)[number]; + +export interface Device { + /** A device type from the device test matrix, section 7 of the development plan. */ + type: string; + /** The device itself, such as "Surface Laptop Studio 2". */ + name: string; +} + +export interface Entry { + by: string; + /** The day the person checked it, as YYYY-MM-DD. */ + date: string; + notes?: string; + /** For pen testing: the devices it was tested on. */ + devices?: Device[]; +} + +export interface Signoff { + version: string; + items: Partial>; +} + +/** Where the sign-off for a version lives, relative to the repository root. */ +export const signoffPath = (version: string): string => `docs/releases/${version}.signoff.json`; + +/** Reads a sign-off file. A missing file is an empty sign-off, so every manual item reads as not yet checked. */ +export function readSignoff(root: string, version: string, path: string = signoffPath(version)): Signoff { + const file = join(root, path); + if (!existsSync(file)) return { version, items: {} }; + return JSON.parse(readFileSync(file, 'utf8')) as Signoff; +} + +/** A blank sign-off for `--init`. Every entry is empty, so it fails until a person fills it in. */ +export function signoffTemplate(version: string): Signoff { + const items: Partial> = {}; + for (const item of MANUAL_ITEMS) { + items[item] = { by: '', date: '', notes: '', ...(item === 'pen' ? { devices: [] } : {}) }; + } + return { version, items }; +} + +/** + * Why a sign-off entry doesn't count yet, or undefined when it does. It needs a name and a real date that is not in + * the future. Pen testing also needs two different devices, each with a type from the device test matrix. + */ +export function entryProblem( + item: ManualItem, + entry: Entry | undefined, + today: string, + deviceTypes: readonly string[], +): string | undefined { + if (!entry || !entry.by?.trim()) return 'No one has signed this off.'; + const day = /^(\d{4})-(\d{2})-(\d{2})$/.exec(entry.date ?? ''); + const real = day && new Date(`${entry.date}T00:00:00Z`).toISOString().startsWith(entry.date); + if (!real) return `The sign-off by ${entry.by} has no date in the form YYYY-MM-DD.`; + if (entry.date > today) return `The sign-off date ${entry.date} is in the future.`; + return item === 'pen' ? deviceProblem(entry, deviceTypes) : undefined; +} + +function deviceProblem(entry: Entry, deviceTypes: readonly string[]): string | undefined { + const devices = entry.devices ?? []; + const names = new Set(devices.map((device) => device.name?.trim().toLowerCase()).filter(Boolean)); + if (names.size < 2) return 'Pen testing needs at least two different devices.'; + const unknown = devices.find( + (device) => !deviceTypes.some((type) => type.toLowerCase() === device.type?.toLowerCase()), + ); + return unknown ? `"${unknown.type}" is not a device type in the device test matrix.` : undefined; +} diff --git a/app/scripts/release/checklist.test.ts b/app/scripts/release/checklist.test.ts new file mode 100644 index 00000000..c7fca6ca --- /dev/null +++ b/app/scripts/release/checklist.test.ts @@ -0,0 +1,273 @@ +// @vitest-environment node +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { checklistItems, deviceTypes, readPlan, sectionOf } from './checklist-doc.ts'; +import { + MANUAL_ITEMS, + entryProblem, + readSignoff, + signoffPath, + signoffTemplate, + type Entry, + type Signoff, +} from './checklist-signoff.ts'; +import { + CHECKS, + evaluate, + markdownReport, + textReport, + unusedChecks, + verdict, + type Context, + type Env, + type Result, +} from './checklist.ts'; +import { fakePe, newTestKey, writeSampleRelease } from './test-support.ts'; + +const ROOT = join(import.meta.dirname, '..', '..', '..'); +const dirs: string[] = []; +afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); + +function folder(): string { + const dir = mkdtempSync(join(tmpdir(), 'opennote-checklist-')); + dirs.push(dir); + return dir; +} + +const TYPES = ['Reference laptop', 'Pen tablet PC', 'Drawing tablet']; +const DAY = '2026-10-20'; +const entry = (extra: Partial = {}): Entry => ({ by: 'Eric', date: DAY, ...extra }); +const pen = entry({ + devices: [ + { type: 'Pen tablet PC', name: 'Surface Laptop Studio 2' }, + { type: 'Drawing tablet', name: 'Wacom Intuos' }, + ], +}); +const fullSignoff = (): Signoff => ({ + version: '1.0.0', + items: { + budgets: entry(), + accessibility: entry(), + pen, + upgrade: entry(), + format: entry(), + updates: entry(), + docs: entry(), + }, +}); + +describe('the checklist in the development plan', () => { + const plan = readPlan(ROOT); + const items = checklistItems(plan); + + it('is read from section 10', () => { + expect(items.length).toBeGreaterThanOrEqual(11); + expect(items[0]).toBe('npm run checks:all passes with no errors.'); + }); + + it('has a check for every item, and no check for an item that is gone', () => { + const unchecked = items.filter((item) => !CHECKS.some((check) => check.match.test(item))); + expect(unchecked).toEqual([]); + expect(unusedChecks(items)).toEqual([]); + }); + + it('gives each item one check, so no two checks answer the same item', () => { + for (const item of items) + expect( + CHECKS.filter((check) => check.match.test(item)), + item, + ).toHaveLength(1); + }); + + it('lists the device types of the test matrix', () => { + expect(deviceTypes(plan)).toEqual(expect.arrayContaining(['Reference laptop', 'Pen tablet PC', 'Drawing tablet'])); + }); + + it('stops when a section is missing', () => { + expect(() => sectionOf('## 1. Intro\n', '10. Release checklist')).toThrow('no section titled'); + }); +}); + +describe('sign-off entries', () => { + const problem = (value: Entry | undefined, item: (typeof MANUAL_ITEMS)[number] = 'budgets') => + entryProblem(item, value, DAY, TYPES); + + it('count with a name and a real date that is not in the future', () => { + expect(problem(entry())).toBeUndefined(); + expect(problem(entry({ date: '2026-10-19' }))).toBeUndefined(); + }); + + it('do not count when empty, undated, misdated, or from the future', () => { + expect(problem(undefined)).toBe('No one has signed this off.'); + expect(problem(entry({ by: ' ' }))).toBe('No one has signed this off.'); + expect(problem(entry({ date: '' }))).toContain('YYYY-MM-DD'); + expect(problem(entry({ date: '2026-02-31' }))).toContain('YYYY-MM-DD'); + expect(problem(entry({ date: '20/10/2026' }))).toContain('YYYY-MM-DD'); + expect(problem(entry({ date: '2026-10-21' }))).toBe('The sign-off date 2026-10-21 is in the future.'); + }); + + it('need two different devices from the matrix for pen testing', () => { + expect(problem(pen, 'pen')).toBeUndefined(); + expect(problem(entry(), 'pen')).toBe('Pen testing needs at least two different devices.'); + const same = entry({ devices: [pen.devices![0], { ...pen.devices![0] }] }); + expect(problem(same, 'pen')).toBe('Pen testing needs at least two different devices.'); + const odd = entry({ devices: [pen.devices![0], { type: 'Toaster', name: 'Bread' }] }); + expect(problem(odd, 'pen')).toBe('"Toaster" is not a device type in the device test matrix.'); + }); + + it('start blank in a template, which fails until a person fills it in', () => { + const template = signoffTemplate('1.0.0'); + expect(Object.keys(template.items)).toEqual([...MANUAL_ITEMS]); + for (const item of MANUAL_ITEMS) expect(problem(template.items[item], item), item).toBeDefined(); + }); + + it('are read from docs/releases, and a missing file is an empty sign-off', () => { + const root = folder(); + expect(readSignoff(root, '1.0.0')).toEqual({ version: '1.0.0', items: {} }); + mkdirSync(join(root, 'docs', 'releases'), { recursive: true }); + writeFileSync(join(root, signoffPath('1.0.0')), JSON.stringify(fullSignoff())); + expect(readSignoff(root, '1.0.0').items.pen?.devices).toHaveLength(2); + }); +}); + +/** An environment with answers set by the test. */ +function env( + root: string, + answers: { checks?: boolean; ci?: string; nightly?: string; issues?: Record } = {}, +): Env { + const { checks = true, ci = '[{"conclusion":"success"}]', nightly = '[{"conclusion":"success"}]' } = answers; + return { + root, + run: (command, args) => { + if (command === 'git' && args[0] === 'rev-parse') return { ok: true, output: 'abcdef1234567\n' }; + if (command === 'git') return { ok: true, output: '' }; + return { ok: checks, output: checks ? '' : 'one\ntwo\n3 errors' }; + }, + gh: (args) => { + if (args[0] === 'issue') return answers.issues?.[args[args.indexOf('--label') + 1]] ?? '[]'; + return args.includes('ci.yml') ? ci : nightly; + }, + today: () => DAY, + }; +} + +function context(root: string, overrides: Partial = {}, answers: Parameters[1] = {}): Context { + return { + env: env(root, answers), + version: '1.0.0', + tag: 'v1.0.0', + signoff: fullSignoff(), + deviceTypes: TYPES, + keys: [], + windowsStatus: () => 'Valid', + ...overrides, + }; +} + +/** A repository root with a changelog, and a folder of signed release files for the signing check. */ +function repository(changelog = '## [1.0.0] - 2026-10-20\n\n- Everything.\n') { + const root = folder(); + writeFileSync(join(root, 'CHANGELOG.md'), changelog); + const key = newTestKey(); + const signed = (file: string) => Buffer.concat([fakePe({ signed: true }), Buffer.from(file)]); + const artifacts = folder(); + writeSampleRelease(artifacts, 'v1.0.0', key, { exe: signed }); + return { root, key, artifacts }; +} + +const items = checklistItems(readPlan(ROOT)); +const byId = (results: Result[], id: string) => results.find((result) => result.id === id) as Result; + +describe('evaluate', () => { + it('passes every item when everything is checked and signed off', () => { + const { root, key, artifacts } = repository(); + const results = evaluate(items, context(root, { artifacts, keys: [key.publicKey] })); + expect(results.map((result) => `${result.id}: ${result.status}`)).toEqual( + CHECKS.map((check) => `${check.id}: pass`), + ); + expect(verdict(results, true)).toBe(true); + }); + + it('shows what is left when nothing has been signed off and nothing can be looked up', () => { + const { root } = repository(); + const bare: Env = { ...env(root), gh: () => undefined }; + const results = evaluate(items, context(root, { env: bare, signoff: { version: '1.0.0', items: {} } })); + const status = (id: string) => byId(results, id).status; + expect(['tests', 'issues', 'signing'].map(status)).toEqual(['pending', 'pending', 'pending']); + expect(['budgets', 'accessibility', 'pen', 'upgrade', 'format', 'updates', 'docs'].map(status)).toEqual( + Array(7).fill('manual'), + ); + expect(byId(results, 'budgets').detail).toContain('docs/releases/1.0.0.signoff.json'); + expect(verdict(results, false)).toBe(false); + }); + + it('fails CHECKS errors, and says what they were', () => { + const { root } = repository(); + const result = byId(evaluate(items, context(root, {}, { checks: false })), 'checks'); + expect(result).toMatchObject({ status: 'fail', detail: 'one two 3 errors' }); + }); + + it('fails when CI or the nightly run did not pass', () => { + const { root } = repository(); + const noCi = byId(evaluate(items, context(root, {}, { ci: '[]' })), 'tests'); + expect(noCi).toMatchObject({ status: 'fail', detail: 'CI has no passing run for commit abcdef1.' }); + const badNightly = byId(evaluate(items, context(root, {}, { nightly: '[{"conclusion":"failure"}]' })), 'tests'); + expect(badNightly.detail).toBe('The latest nightly run on main did not pass.'); + }); + + it('fails on an open issue labeled data-loss, crash, or security', () => { + const { root } = repository(); + const issues = { crash: '[{"number":4},{"number":9}]', security: '[{"number":2}]' }; + const result = byId(evaluate(items, context(root, {}, { issues })), 'issues'); + expect(result).toMatchObject({ status: 'fail', detail: 'Open issues: 2 labeled crash, 1 labeled security.' }); + }); + + it('fails when the exes are not signed, or signed for another version', () => { + const { root, key, artifacts } = repository(); + const unsigned = folder(); + writeSampleRelease(unsigned, 'v1.0.0', key); + const notSigned = byId(evaluate(items, context(root, { artifacts: unsigned, keys: [key.publicKey] })), 'signing'); + expect(notSigned.status).toBe('fail'); + expect(notSigned.detail).toContain('no Authenticode signature'); + const otherKey = byId(evaluate(items, context(root, { artifacts, keys: [newTestKey().publicKey] })), 'signing'); + expect(otherKey.detail).toContain('not signed with a key that this release trusts'); + }); + + it('fails when the changelog has no entry for the version, even if the docs are signed off', () => { + const { root } = repository('## [Unreleased]\n\n- Soon.\n'); + expect(byId(evaluate(items, context(root)), 'docs')).toMatchObject({ + status: 'fail', + detail: 'CHANGELOG.md has no entry with notes for 1.0.0.', + }); + }); + + it('fails an item that has no check, so a new item in the plan cannot slip through', () => { + const { root } = repository(); + const [result] = evaluate(['The moon is full.'], context(root)); + expect(result).toMatchObject({ id: 'unknown', status: 'fail' }); + }); +}); + +describe('reports', () => { + const results: Result[] = [ + { item: 'A thing passes.', id: 'a', status: 'pass', detail: 'Fine.' }, + { item: 'Another needs a person.', id: 'b', status: 'manual', detail: 'Sign it.' }, + { item: 'One cannot be told.', id: 'c', status: 'pending', detail: 'Needs gh.' }, + ]; + + it('write a line for each item in text and a row for each in Markdown', () => { + expect(textReport(results)).toContain('Needs sign-off Another needs a person.'); + const table = markdownReport(results, '1.0.0'); + expect(table).toContain('## Release checklist for 1.0.0'); + expect(table).toContain('| Not checked | One cannot be told. | Needs gh. |'); + }); + + it('allows a pending item unless everything is required', () => { + const passing = [results[0], results[2]]; + expect(verdict(passing, false)).toBe(true); + expect(verdict(passing, true)).toBe(false); + expect(verdict(results, false)).toBe(false); + }); +}); diff --git a/app/scripts/release/checklist.ts b/app/scripts/release/checklist.ts new file mode 100644 index 00000000..239ec1e0 --- /dev/null +++ b/app/scripts/release/checklist.ts @@ -0,0 +1,276 @@ +// Checks the release checklist in section 10 of docs/DEVELOPMENT.md. It runs the checks a computer can run, reads +// the sign-off file for the ones that need a person, and says which items are not done. It reads the list from the +// document, so a new item there fails here until it has a check. +// Usage: node app/scripts/release/checklist.ts [--version ] [--tag ] [--signoff ] [--artifacts ] +// [--pubkey ]... [--markdown ] [--report] [--require-all] [--init] + +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { versionsIn } from '../check-version.ts'; +import { checklistItems, deviceTypes, readPlan } from './checklist-doc.ts'; +import { + MANUAL_ITEMS, + entryProblem, + readSignoff, + signoffPath, + signoffTemplate, + type ManualItem, + type Signoff, +} from './checklist-signoff.ts'; +import { parsePublicKey, type PublicKey } from './minisign.ts'; +import { changelogSection, previousTag } from './notes.ts'; +import { keysIn, verifyRelease } from './verify.ts'; + +const ROOT = join(import.meta.dirname, '..', '..', '..'); +const BLOCKING_LABELS = ['data-loss', 'crash', 'security']; + +/** + * `pass` and `fail` are answers. `manual` means a person has not signed it off. `pending` means this run can't + * tell. + */ +export type Status = 'pass' | 'fail' | 'manual' | 'pending'; +export interface Outcome { + status: Status; + detail: string; +} +export interface Result extends Outcome { + item: string; + id: string; +} + +/** What the checks use from the machine, so tests can replace it. */ +export interface Env { + root: string; + /** Runs a program. The output is its standard output and error together. */ + run(command: string, args: string[]): { ok: boolean; output: string }; + /** Runs the GitHub command-line tool. Undefined when it is missing, not signed in, or fails. */ + gh(args: string[]): string | undefined; + /** Today as YYYY-MM-DD. */ + today(): string; +} + +export interface Context { + env: Env; + version: string; + tag: string; + signoff: Signoff; + deviceTypes: readonly string[]; + /** A folder of release files to check the signing item against. */ + artifacts?: string; + keys: readonly PublicKey[]; + /** Asks Windows about an Authenticode signature. Tests replace it, because PowerShell takes seconds to start. */ + windowsStatus?: (path: string) => string | undefined; +} + +interface Check { + id: string; + match: RegExp; + run(context: Context): Outcome; +} + +export function realEnv(root: string = ROOT): Env { + return { + root, + run(command, args) { + const done = spawnSync(command, args, { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }); + return { ok: done.status === 0, output: `${done.stdout ?? ''}${done.stderr ?? ''}` }; + }, + gh(args) { + const done = spawnSync('gh', args, { cwd: root, encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 }); + return done.error || done.status !== 0 ? undefined : done.stdout; + }, + today: () => new Date().toISOString().slice(0, 10), + }; +} + +const lastLines = (text: string, count = 3): string => text.trim().split(/\r?\n/).slice(-count).join(' '); + +/** A sign-off item: it passes when the sign-off file has a valid entry, and needs a person otherwise. */ +function manual(id: ManualItem, extra: (context: Context) => string = () => ''): Check['run'] { + return (context) => { + const entry = context.signoff.items[id]; + const problem = entryProblem(id, entry, context.env.today(), context.deviceTypes); + if (problem) + return { + status: 'manual', + detail: `${problem} Record it in ${signoffPath(context.version)}. ${extra(context)}`.trim(), + }; + return { status: 'pass', detail: `Signed off by ${entry?.by} on ${entry?.date}.` }; + }; +} + +function checksPass({ env }: Context): Outcome { + const done = env.run(process.execPath, ['checks/cli.ts', '--all', '--quiet']); + return { status: done.ok ? 'pass' : 'fail', detail: done.ok ? 'CHECKS found no errors.' : lastLines(done.output) }; +} + +function json(text: string | undefined): unknown[] | undefined { + try { + return text === undefined ? undefined : (JSON.parse(text) as unknown[]); + } catch { + return undefined; + } +} + +function testsPass({ env }: Context): Outcome { + const sha = env.run('git', ['rev-parse', 'HEAD']).output.trim(); + const ci = json(env.gh(['run', 'list', '--workflow', 'ci.yml', '--commit', sha, '--json', 'conclusion'])); + const nightly = json( + env.gh(['run', 'list', '--workflow', 'nightly.yml', '--branch', 'main', '--limit', '1', '--json', 'conclusion']), + ); + if (!ci || !nightly) return { status: 'pending', detail: 'Needs the GitHub command-line tool (gh), signed in.' }; + const passed = (runs: unknown[]) => runs.some((run) => (run as { conclusion: string }).conclusion === 'success'); + if (!passed(ci)) return { status: 'fail', detail: `CI has no passing run for commit ${sha.slice(0, 7)}.` }; + if (!passed(nightly)) return { status: 'fail', detail: 'The latest nightly run on main did not pass.' }; + return { status: 'pass', detail: `CI passed for ${sha.slice(0, 7)}, and so did the latest nightly run.` }; +} + +function noBlockingIssues({ env }: Context): Outcome { + const found: string[] = []; + for (const label of BLOCKING_LABELS) { + const issues = json(env.gh(['issue', 'list', '--label', label, '--state', 'open', '--json', 'number'])); + if (!issues) return { status: 'pending', detail: 'Needs the GitHub command-line tool (gh), signed in.' }; + if (issues.length > 0) found.push(`${issues.length} labeled ${label}`); + } + return found.length > 0 + ? { status: 'fail', detail: `Open issues: ${found.join(', ')}.` } + : { status: 'pass', detail: `No open issues labeled ${BLOCKING_LABELS.join(', ')}.` }; +} + +function signedAndVersioned(context: Context): Outcome { + if (!context.artifacts) + return { status: 'pending', detail: 'Needs the release files. Run with --artifacts .' }; + const problems = verifyRelease({ + dir: context.artifacts, + tag: context.tag, + keys: context.keys, + requireAuthenticode: true, + windowsStatus: context.windowsStatus, + }); + return problems.length === 0 + ? { status: 'pass', detail: 'Every exe has an Authenticode signature and an update signature for this version.' } + : { status: 'fail', detail: problems.slice(0, 3).join(' ') }; +} + +function changelogAndDocs(context: Context): Outcome { + const changelog = readFileSync(join(context.env.root, 'CHANGELOG.md'), 'utf8'); + if (!changelogSection(changelog, context.version)) { + return { status: 'fail', detail: `CHANGELOG.md has no entry with notes for ${context.version}.` }; + } + return manual('docs')(context); +} + +function formatHint({ env, tag }: Context): string { + const tags = env.run('git', ['tag', '--list', 'v*']).output.split(/\s+/).filter(Boolean); + const previous = previousTag(tags, tag); + if (!previous) return ''; + const changed = env.run('git', [ + 'diff', + '--name-only', + `${previous}..HEAD`, + '--', + 'crates/core/src/format', + 'docs/format', + ]); + const count = changed.output.split('\n').filter(Boolean).length; + return `${count} format files changed since ${previous}.`; +} + +/** Every check, in the order of the checklist. The `match` text picks the document's item that it answers. */ +export const CHECKS: readonly Check[] = [ + { id: 'checks', match: /checks:all/, run: checksPass }, + { id: 'tests', match: /automated tests pass/, run: testsPass }, + { id: 'budgets', match: /budget/i, run: manual('budgets') }, + { id: 'issues', match: /open issues labeled/, run: noBlockingIssues }, + { id: 'accessibility', match: /keyboard and screen reader/, run: manual('accessibility') }, + { id: 'pen', match: /pen testing/i, run: manual('pen') }, + { id: 'upgrade', match: /Upgrading from the previous release/, run: manual('upgrade') }, + { id: 'format', match: /File format changes/, run: manual('format', formatHint) }, + { id: 'signing', match: /code-signed/, run: signedAndVersioned }, + { id: 'updates', match: /Updating from each of the last three/, run: manual('updates') }, + { id: 'docs', match: /Changelog and documentation/, run: changelogAndDocs }, +]; + +/** Runs the check for each item of the checklist. An item with no check fails, so the list and the script agree. */ +export function evaluate(items: readonly string[], context: Context): Result[] { + return items.map((item) => { + const check = CHECKS.find((candidate) => candidate.match.test(item)); + if (!check) + return { item, id: 'unknown', status: 'fail', detail: 'No check exists for this item. Add one to checklist.ts.' }; + return { item, id: check.id, ...check.run(context) }; + }); +} + +/** The checks that no item of the document points at, which means the document dropped an item. */ +export function unusedChecks(items: readonly string[]): string[] { + return CHECKS.filter((check) => !items.some((item) => check.match.test(item))).map((check) => check.id); +} + +const LABELS: Record = { pass: 'Pass', fail: 'Fail', manual: 'Needs sign-off', pending: 'Not checked' }; + +/** A plain-text report, one line for each item. */ +export function textReport(results: readonly Result[]): string { + return results + .map((result) => `${LABELS[result.status].padEnd(14)} ${result.item}\n${' '.repeat(15)}${result.detail}`) + .join('\n'); +} + +/** The same report as a Markdown table, for the summary of a workflow run. */ +export function markdownReport(results: readonly Result[], version: string): string { + const rows = results.map((result) => `| ${LABELS[result.status]} | ${result.item} | ${result.detail} |`); + return [ + `## Release checklist for ${version}`, + '', + '| Status | Item | Detail |', + '| --- | --- | --- |', + ...rows, + '', + ].join('\n'); +} + +/** Whether the release may go ahead: nothing failed, and nothing is waiting for a person. */ +export function verdict(results: readonly Result[], requireAll: boolean): boolean { + return results.every((result) => result.status === 'pass' || (result.status === 'pending' && !requireAll)); +} + +function option(args: string[], name: string): string | undefined { + const at = args.indexOf(name); + return at === -1 ? undefined : args[at + 1]; +} + +function initSignoff(version: string, path: string): void { + const file = join(ROOT, path); + if (existsSync(file)) throw new Error(`${path} already exists.`); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, `${JSON.stringify(signoffTemplate(version), null, 2)}\n`); + console.log(`Wrote ${path}. Fill in each entry when you have checked the item: ${MANUAL_ITEMS.join(', ')}.`); +} + +function main(): void { + const args = process.argv.slice(2); + const version = option(args, '--version') ?? Object.values(versionsIn(ROOT)).find(Boolean) ?? ''; + const signoff = option(args, '--signoff') ?? signoffPath(version); + if (args.includes('--init')) return initSignoff(version, signoff); + const pubkeys = args.flatMap((arg, at) => (args[at - 1] === '--pubkey' ? [arg] : [])); + const plan = readPlan(ROOT); + const items = checklistItems(plan); + const context: Context = { + env: realEnv(), + version, + tag: option(args, '--tag') ?? `v${version}`, + signoff: readSignoff(ROOT, version, signoff), + deviceTypes: deviceTypes(plan), + artifacts: option(args, '--artifacts'), + keys: pubkeys.length > 0 ? pubkeys.map((path) => parsePublicKey(readFileSync(path, 'utf8'))) : keysIn(), + }; + const results = evaluate(items, context); + console.log(textReport(results)); + const markdown = option(args, '--markdown'); + if (markdown) writeFileSync(markdown, markdownReport(results, version)); + const ok = verdict(results, args.includes('--require-all')); + console.log(ok ? `\nThe checklist for ${version} is complete.` : `\nThe checklist for ${version} is not complete.`); + if (!ok && !args.includes('--report')) process.exitCode = 1; +} + +if (import.meta.main) main(); diff --git a/app/scripts/release/checksums.test.ts b/app/scripts/release/checksums.test.ts new file mode 100644 index 00000000..cf852440 --- /dev/null +++ b/app/scripts/release/checksums.test.ts @@ -0,0 +1,42 @@ +// @vitest-environment node +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { CHECKSUMS_FILE, checksumLines, writeChecksums } from './checksums.ts'; + +const dirs: string[] = []; +afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); + +function folder(): string { + const dir = mkdtempSync(join(tmpdir(), 'opennote-sums-')); + dirs.push(dir); + return dir; +} + +const sha = (text: string) => createHash('sha256').update(text).digest('hex'); + +describe('checksumLines', () => { + it('writes hash, two spaces, and name, sorted by name', () => { + const dir = folder(); + writeFileSync(join(dir, 'b.txt'), 'bee'); + writeFileSync(join(dir, 'a.txt'), 'ay'); + expect(checksumLines(dir, ['b.txt', 'a.txt'])).toEqual([`${sha('ay')} a.txt`, `${sha('bee')} b.txt`]); + }); +}); + +describe('writeChecksums', () => { + it('lists every file but itself and any folder, and can be run again with the same result', () => { + const dir = folder(); + writeFileSync(join(dir, 'OpenNote_Windows64.exe'), 'exe'); + writeFileSync(join(dir, 'latest.json'), '{}'); + mkdirSync(join(dir, 'nested')); + expect(writeChecksums(dir)).toBe(2); + const first = readFileSync(join(dir, CHECKSUMS_FILE), 'utf8'); + // Names sort by character code, so the capital O of OpenNote comes before the lowercase l of latest. + expect(first).toBe(`${sha('exe')} OpenNote_Windows64.exe\n${sha('{}')} latest.json\n`); + expect(writeChecksums(dir)).toBe(2); + expect(readFileSync(join(dir, CHECKSUMS_FILE), 'utf8')).toBe(first); + }); +}); diff --git a/app/scripts/release/checksums.ts b/app/scripts/release/checksums.ts new file mode 100644 index 00000000..060fdb82 --- /dev/null +++ b/app/scripts/release/checksums.ts @@ -0,0 +1,34 @@ +// Writes SHA256SUMS.txt for the files of a release, in the format `sha256sum --check` reads, so anyone can confirm +// a download on Windows, macOS, or Linux. The app itself checks the hashes in the manifest, not this file. +// Usage: node app/scripts/release/checksums.ts + +import { createHash } from 'node:crypto'; +import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export const CHECKSUMS_FILE = 'SHA256SUMS.txt'; + +/** The lines of a checksum file: a lowercase hash, two spaces, and the file name, in name order. */ +export function checksumLines(dir: string, names: readonly string[]): string[] { + return [...names].sort().map( + (name) => + `${createHash('sha256') + .update(readFileSync(join(dir, name))) + .digest('hex')} ${name}`, + ); +} + +/** Writes the checksum file for every file in `dir` except itself, and returns how many it lists. */ +export function writeChecksums(dir: string): number { + const names = readdirSync(dir).filter((name) => name !== CHECKSUMS_FILE && statSync(join(dir, name)).isFile()); + writeFileSync(join(dir, CHECKSUMS_FILE), `${checksumLines(dir, names).join('\n')}\n`); + return names.length; +} + +function main(): void { + const dir = process.argv[2]; + if (!dir) throw new Error('Usage: node app/scripts/release/checksums.ts '); + console.log(`Wrote ${CHECKSUMS_FILE} for ${writeChecksums(dir)} files.`); +} + +if (import.meta.main) main(); diff --git a/app/scripts/release/minisign.test.ts b/app/scripts/release/minisign.test.ts new file mode 100644 index 00000000..e424b008 --- /dev/null +++ b/app/scripts/release/minisign.test.ts @@ -0,0 +1,91 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { onlyField, parsePublicKey, parseSignature, trustedFields, verify, verifyWithAny } from './minisign.ts'; +import { newTestKey } from './test-support.ts'; + +// Made by the real Tauri command-line tool (tauri signer generate, then tauri signer sign --app-version 1.2.3), so +// these tests prove that our check agrees with what the release workflow signs. It holds no private key. +const TAURI_PUBLIC_KEY = [ + 'dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IDRFQzhDRDMyMUZDMjM3Q0IKUldUTE44SWZNczNJVGd4d2dW', + 'a1dtOHJnSmlHa1hNWFo3ZkxWT0gvdSs2UGlvTzBoTDhsNlJHa0cK', +].join(''); +const TAURI_SIGNATURE = [ + 'dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVUTE44SWZNczNJVGdGR29ZUDRFcHZp', + 'MnBGRXNLRzExcmtFNzdFLzhaUFc4N2RuLzZCNEZBV0hPY1ZZRWJrODBaVzUwWU45RUtFTExZLytQaHZhQjJlL2hxZGdUTUxZcmd3', + 'PQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNzkwOTk3NTY1CWZpbGU6T3Blbk5vdGVfV2luZG93czY0LmV4ZQl2ZXJzaW9u', + 'OjEuMi4zCkZPMWZlSUtka2dwN2g4WjlNeFZlNzhXUzZ0QWpSTkxuY0lJWWJnZnFYTHRHSXEyRDRWcXBWM3cvZ0l0OUwvRjZISks3', + 'UDBIZHRjSUR4MENQYlNaNkJRPT0K', +].join(''); +const TAURI_FILE = Buffer.from('hello exe'); + +describe('a signature made by tauri signer sign', () => { + const key = parsePublicKey(TAURI_PUBLIC_KEY); + const signature = parseSignature(TAURI_SIGNATURE); + + it('verifies and returns its trusted comment', () => { + const comment = verify(TAURI_FILE, signature, key); + expect(onlyField(comment, 'version')).toBe('1.2.3'); + expect(onlyField(comment, 'file')).toBe('OpenNote_Windows64.exe'); + }); + + it('fails for a changed file', () => { + expect(() => verify(Buffer.from('hello exe!'), signature, key)).toThrow('does not match the file'); + }); + + it('fails when the trusted comment is changed', () => { + const edited = { ...signature, trustedComment: signature.trustedComment.replace('1.2.3', '9.9.9') }; + expect(() => verify(TAURI_FILE, edited, key)).toThrow('trusted comment was changed'); + }); + + it('fails for another key', () => { + expect(() => verify(TAURI_FILE, signature, newTestKey().publicKey)).toThrow('different key'); + expect(() => verifyWithAny(TAURI_FILE, signature, [newTestKey().publicKey])).toThrow('not signed with a key'); + }); + + it('reads the plain minisign text as well as the base64 form', () => { + const text = Buffer.from(TAURI_SIGNATURE, 'base64').toString('utf8'); + expect(parseSignature(text)).toEqual(signature); + expect(parsePublicKey(Buffer.from(TAURI_PUBLIC_KEY, 'base64').toString('utf8'))).toEqual(key); + }); +}); + +describe('signatures made by the test key', () => { + it('verify with any of several trusted keys', () => { + const key = newTestKey(); + const data = Buffer.from('exe'); + const signature = parseSignature(key.signFile(data, 'a.exe', '1.0.0')); + const comment = verifyWithAny(data, signature, [newTestKey().publicKey, parsePublicKey(key.pubFile)]); + expect(comment).toContain('version:1.0.0'); + }); + + it('refuse minisign legacy mode', () => { + const key = newTestKey(); + const signature = parseSignature(key.signFile(Buffer.from('x'), 'a.exe', '1.0.0')); + expect(() => verify(Buffer.from('x'), { ...signature, algorithm: 'Ed' }, key.publicKey)).toThrow('legacy'); + }); +}); + +describe('malformed input', () => { + it('is refused with a sentence, not a crash', () => { + expect(() => parseSignature('not a signature')).toThrow('not a minisign signature'); + expect(() => parseSignature('')).toThrow('not a minisign signature'); + expect(() => parsePublicKey('bm90IGEga2V5')).toThrow('not a minisign Ed25519 key'); + expect(() => parsePublicKey('')).toThrow('not a minisign Ed25519 key'); + }); +}); + +describe('trusted comment fields', () => { + it('lists every field in order and finds the only one', () => { + expect(trustedFields('timestamp:1\tfile:a.exe\tversion:1.0.0')).toEqual([ + ['timestamp', '1'], + ['file', 'a.exe'], + ['version', '1.0.0'], + ]); + expect(onlyField('file:a.exe\tversion:1.0.0', 'version')).toBe('1.0.0'); + }); + + it('treats a missing or repeated field as no value, so a repeat cannot hide the real one', () => { + expect(onlyField('file:a.exe', 'version')).toBeUndefined(); + expect(onlyField('version:1.0.0\tversion:9.9.9', 'version')).toBeUndefined(); + }); +}); diff --git a/app/scripts/release/minisign.ts b/app/scripts/release/minisign.ts new file mode 100644 index 00000000..347f15e0 --- /dev/null +++ b/app/scripts/release/minisign.ts @@ -0,0 +1,112 @@ +// Checks minisign signatures the way the updater in crates/updater does, using only Node's crypto. The release +// workflow uses it to confirm that every exe it is about to publish carries a signature the app will accept. +// It only verifies. Signing is left to `tauri signer sign`, so the private key is never read by our own code. + +import { createHash, createPublicKey, verify as edVerify } from 'node:crypto'; + +/** The 12 bytes that turn a raw 32-byte Ed25519 key into the DER form that Node reads. */ +const SPKI_PREFIX = Buffer.from('302a300506032b6570032100', 'hex'); + +export interface PublicKey { + /** The 8-byte key id that a signature names, so a signature picks the key it was made with. */ + id: Buffer; + /** The raw 32-byte Ed25519 public key. */ + key: Buffer; +} + +export interface Signature { + /** `ED` signs the BLAKE2b-512 hash of the file. `Ed` is minisign's legacy mode, which signs the whole file. */ + algorithm: string; + keyId: Buffer; + signature: Buffer; + /** The comment after `trusted comment: `. The global signature covers it, so it can't change unnoticed. */ + trustedComment: string; + globalSignature: Buffer; +} + +function lines(text: string): string[] { + return text.split(/\r?\n/).filter((line) => line.length > 0); +} + +/** Unwraps the base64 form that `tauri signer` writes: the whole minisign file, encoded again. */ +function plainText(text: string, firstLine: string): string { + return text.trimStart().startsWith(firstLine) ? text : Buffer.from(text.trim(), 'base64').toString('utf8'); +} + +/** + * Reads a public key from a `.pub` file as `tauri signer generate` writes it (base64 text of the two-line minisign + * file), or from the two-line minisign text itself. Throws when it isn't a minisign Ed25519 key. + */ +export function parsePublicKey(text: string): PublicKey { + const parts = lines(plainText(text, 'untrusted comment:')); + const bytes = Buffer.from(parts.at(-1) ?? '', 'base64'); + if (bytes.length !== 42 || bytes.subarray(0, 2).toString('latin1') !== 'Ed') { + throw new Error('The public key is not a minisign Ed25519 key.'); + } + return { id: bytes.subarray(2, 10), key: bytes.subarray(10) }; +} + +/** + * Reads a signature from a `.sig` file as `tauri signer sign` writes it (base64 text of the four-line minisign + * file), which is also the form the manifest holds, or from the four-line minisign text. Throws when it is malformed. + */ +export function parseSignature(text: string): Signature { + const parts = lines(plainText(text, 'untrusted comment:')); + const comment = parts[2] ?? ''; + const prefix = 'trusted comment: '; + const bytes = Buffer.from(parts[1] ?? '', 'base64'); + const global = Buffer.from(parts[3] ?? '', 'base64'); + if (parts.length !== 4 || !comment.startsWith(prefix) || bytes.length !== 74 || global.length !== 64) { + throw new Error('The signature is not a minisign signature with a trusted comment.'); + } + return { + algorithm: bytes.subarray(0, 2).toString('latin1'), + keyId: bytes.subarray(2, 10), + signature: bytes.subarray(10), + trustedComment: comment.slice(prefix.length), + globalSignature: global, + }; +} + +function ed25519(key: PublicKey, message: Buffer, signature: Buffer): boolean { + const publicKey = createPublicKey({ key: Buffer.concat([SPKI_PREFIX, key.key]), format: 'der', type: 'spki' }); + return edVerify(null, message, publicKey, signature); +} + +/** + * Checks `data` against a signature and returns its trusted comment, or throws why it can't be trusted. Like the + * updater, it refuses the legacy mode, a key id that isn't the key's, and a comment the global signature doesn't cover. + */ +export function verify(data: Buffer, signature: Signature, key: PublicKey): string { + if (signature.algorithm !== 'ED') throw new Error('The signature uses minisign legacy mode, which is not accepted.'); + if (!signature.keyId.equals(key.id)) throw new Error('The signature was made with a different key.'); + const hash = createHash('blake2b512').update(data).digest(); + if (!ed25519(key, hash, signature.signature)) throw new Error('The signature does not match the file.'); + const covered = Buffer.concat([signature.signature, Buffer.from(signature.trustedComment, 'utf8')]); + if (!ed25519(key, covered, signature.globalSignature)) throw new Error('The trusted comment was changed.'); + return signature.trustedComment; +} + +/** Checks `data` against any of several trusted keys, and returns the trusted comment of the first that matches. */ +export function verifyWithAny(data: Buffer, signature: Signature, keys: readonly PublicKey[]): string { + const key = keys.find((candidate) => candidate.id.equals(signature.keyId)); + if (!key) throw new Error('The file is not signed with a key that this release trusts.'); + return verify(data, signature, key); +} + +/** + * The fields of a trusted comment, in order: `timestamp:1790997565file:OpenNote_Windows64.exeversion:1.2.3`. + * A field can appear twice, so this returns a list, and the caller decides what a repeat means. + */ +export function trustedFields(comment: string): [string, string][] { + return comment + .split('\t') + .filter((field) => field.includes(':')) + .map((field): [string, string] => [field.slice(0, field.indexOf(':')), field.slice(field.indexOf(':') + 1).trim()]); +} + +/** The one value of a field, or undefined when it is missing or appears more than once. */ +export function onlyField(comment: string, name: string): string | undefined { + const values = trustedFields(comment).filter(([key]) => key === name); + return values.length === 1 ? values[0][1] : undefined; +} diff --git a/app/scripts/release/notes.test.ts b/app/scripts/release/notes.test.ts new file mode 100644 index 00000000..2a74dcb3 --- /dev/null +++ b/app/scripts/release/notes.test.ts @@ -0,0 +1,275 @@ +// @vitest-environment node +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + MAX_PLAIN_LINES, + bulletsOf, + changeFromMerge, + changelogSection, + changesBetween, + markdownNotes, + parseLog, + plainLine, + plainNotes, + previousTag, + type Change, +} from './notes.ts'; +import { compareVersions } from './versions.ts'; + +const CHANGELOG = `# Changelog + +## [Unreleased] + +- Not released yet. + +## [1.0.0] - 2026-11-01 + +### Notes + +- Write **typed notes** with [links](https://example.com) and \`code\`. +- Draw with a pen. + +### Fixes + +* Fixed a crash on start-up. + +## [0.9.0] - 2026-10-01 + +- Older. + +[Unreleased]: https://github.com/XrxcGH/OpenNote/compare/v1.0.0...main +`; + +describe('compareVersions', () => { + it('orders versions by number, and a prerelease before its release', () => { + const ordered = [ + '0.9.0', + '1.0.0-alpha', + '1.0.0-alpha.1', + '1.0.0-alpha.beta', + '1.0.0-beta', + '1.0.0-beta.2', + '1.0.0-beta.11', + '1.0.0-rc.1', + '1.0.0', + '1.0.1', + '1.10.0', + '2.0.0', + ]; + for (let at = 1; at < ordered.length; at++) { + expect(compareVersions(ordered[at - 1], ordered[at]), `${ordered[at - 1]} < ${ordered[at]}`).toBe(-1); + expect(compareVersions(ordered[at], ordered[at - 1])).toBe(1); + } + expect(compareVersions('1.0.0+build.5', '1.0.0')).toBe(0); + }); + + it('refuses text that is not a version', () => { + expect(() => compareVersions('1.0', '1.0.0')).toThrow('not a semantic version'); + expect(() => compareVersions('v1.0.0', '1.0.0')).toThrow('not a semantic version'); + }); +}); + +describe('changelogSection', () => { + it('finds the section for a version, with or without brackets, and stops at the next heading', () => { + const section = changelogSection(CHANGELOG, '1.0.0'); + expect(section).toContain('### Notes'); + expect(section).toContain('Fixed a crash'); + expect(section).not.toContain('Older'); + expect(section).not.toContain('compare/'); + expect(changelogSection('## 0.5.0\n\n- A thing.\n', '0.5.0')).toBe('- A thing.'); + }); + + it('does not mistake one version for the start of another', () => { + expect(changelogSection(CHANGELOG, '1.0')).toBeUndefined(); + expect(changelogSection('## [1.0.0-beta.1]\n\n- Beta.\n', '1.0.0')).toBeUndefined(); + }); + + it('has no section for a version that is missing or empty', () => { + expect(changelogSection(CHANGELOG, '3.0.0')).toBeUndefined(); + expect(changelogSection('## [1.0.0]\n\n## [0.9.0]\n- Older\n', '1.0.0')).toBeUndefined(); + }); +}); + +describe('plain text', () => { + it('removes links, emphasis, code marks, and bullets', () => { + expect(plainLine('- Write **typed notes** with [links](https://example.com) and `code`.')).toBe( + 'Write typed notes with links and code.', + ); + }); + + it('lists only the bullets of a section', () => { + expect(bulletsOf(changelogSection(CHANGELOG, '1.0.0') ?? '')).toEqual([ + 'Write typed notes with links and code.', + 'Draw with a pen.', + 'Fixed a crash on start-up.', + ]); + }); +}); + +describe('changeFromMerge', () => { + it('reads a GitHub pull request merge from its body', () => { + expect(changeFromMerge('a1', 'Merge pull request #12 from XrxcGH/phase-3', 'add storage\n\nMore text.')).toEqual({ + sha: 'a1', + pr: 12, + text: 'Add storage', + }); + }); + + it('reads a local merge named for its branch', () => { + expect(changeFromMerge('b2', 'Merge phase-2: app shell and navigation', '')).toEqual({ + sha: 'b2', + text: 'App shell and navigation', + }); + expect(changeFromMerge('c3', 'Merge the verified corrections to ADR 0006', '')?.text).toBe( + 'The verified corrections to ADR 0006', + ); + }); + + it('skips merges that only bring a branch up to date', () => { + for (const subject of [ + "Merge branch 'main' into phase-3", + 'Merge main into phase-3', + 'Merge remote-tracking branch x', + ]) { + expect(changeFromMerge('d4', subject, ''), subject).toBeUndefined(); + } + }); +}); + +describe('previousTag', () => { + const tags = ['v0.9.0', 'v1.0.0-beta.1', 'v1.0.0-beta.2', 'v1.0.0', 'v1.1.0-beta.1', 'v1.1.0']; + + it('looks back to the previous stable release for a stable release', () => { + expect(previousTag(tags, 'v1.1.0')).toBe('v1.0.0'); + expect(previousTag(tags, 'v1.0.0')).toBe('v0.9.0'); + }); + + it('looks back to the previous release of any kind for a prerelease', () => { + expect(previousTag(tags, 'v1.0.0-beta.2')).toBe('v1.0.0-beta.1'); + expect(previousTag(tags, 'v1.1.0-beta.1')).toBe('v1.0.0'); + }); + + it('has no previous tag for the first release, or for a tag that is not yet made', () => { + expect(previousTag(tags, 'v0.9.0')).toBeUndefined(); + expect(previousTag(tags, 'v2.0.0')).toBe('v1.1.0'); + }); +}); + +describe('the release page', () => { + const changes: Change[] = [ + { sha: 'a', text: 'Add storage', pr: 12 }, + { sha: 'b', text: 'App shell' }, + ]; + + it('shows what is new, the downloads, and all changes', () => { + const page = markdownNotes({ tag: 'v1.0.0', changelog: CHANGELOG, changes, previous: 'v0.9.0' }); + expect(page).toContain("## What's new"); + expect(page).toContain('### Notes'); + expect(page).toContain('| 64-bit Windows, on most PCs | `OpenNote_Windows64.exe` |'); + expect(page).toContain('| Windows on Arm, such as Snapdragon laptops | `OpenNote_WindowsARM64.exe` |'); + expect(page).toContain('All changes (2)'); + expect(page).toContain('- Add storage ([#12](https://github.com/XrxcGH/OpenNote/pull/12))'); + expect(page).toContain('compare/v0.9.0...v1.0.0'); + }); + + it('shows angle brackets in a commit subject as text, not as an HTML tag', () => { + const page = markdownNotes({ tag: 'v1.0.0', changes: [{ sha: 'a', text: 'Name files OpenNote_' }] }); + expect(page).toContain('- Name files OpenNote_<OS spec>'); + }); + + it('still reads well with no changelog entry and no changes', () => { + const page = markdownNotes({ tag: 'v1.0.0', changes: [] }); + expect(page).toContain('## OpenNote 1.0.0'); + expect(page).not.toContain('
'); + }); +}); + +describe('the update notice', () => { + it('lists the changelog bullets', () => { + expect(plainNotes({ tag: 'v1.0.0', changelog: CHANGELOG, changes: [] })).toBe( + 'Write typed notes with links and code.\nDraw with a pen.\nFixed a crash on start-up.', + ); + }); + + it('falls back to the merged changes, then to the tag', () => { + expect(plainNotes({ tag: 'v2.0.0', changelog: CHANGELOG, changes: [{ sha: 'a', text: 'Add storage' }] })).toBe( + 'Add storage', + ); + expect(plainNotes({ tag: 'v2.0.0', changelog: CHANGELOG, changes: [] })).toBe('OpenNote v2.0.0'); + }); + + it('fits in the lines the notice can show, with a link to the rest', () => { + const many = Array.from({ length: 20 }, (_, at) => `- Change ${at + 1}.`).join('\n'); + const text = plainNotes({ tag: 'v1.0.0', changelog: `## [1.0.0]\n\n${many}\n`, changes: [] }); + const lines = text.split('\n'); + expect(lines).toHaveLength(MAX_PLAIN_LINES); + expect(lines[0]).toBe('Change 1.'); + expect(lines.at(-2)).toBe('And 14 more.'); + expect(lines.at(-1)).toBe('Full notes: https://github.com/XrxcGH/OpenNote/releases/tag/v1.0.0'); + }); + + it('gives up on the list when it is too big for the updater', () => { + const huge = `- ${'word '.repeat(2000)}.`; + expect(plainNotes({ tag: 'v1.0.0', changelog: `## [1.0.0]\n\n${huge}\n`, changes: [] })).toBe( + 'OpenNote v1.0.0\nFull notes: https://github.com/XrxcGH/OpenNote/releases/tag/v1.0.0', + ); + }); +}); + +describe('parseLog', () => { + it('reads records of hash, subject, and body', () => { + const log = 'h1\x1fMerge phase-1: spikes\x1f\x1e\nh2\x1fMerge main into phase-1\x1f\x1e\nh3\x1fadd x\x1f\x1e'; + expect(parseLog(log, true).map((change) => change.text)).toEqual(['Spikes', 'Add x']); + expect(parseLog(log, false).map((change) => change.text)).toHaveLength(3); + }); +}); + +describe('changesBetween, in a real repository', { timeout: 60_000 }, () => { + const dirs: string[] = []; + afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); + + function repository(): string { + const dir = mkdtempSync(join(tmpdir(), 'opennote-notes-')); + dirs.push(dir); + const git = (...args: string[]) => + execFileSync( + 'git', + ['-c', 'user.name=T', '-c', 'user.email=t@example.com', '-c', 'commit.gpgsign=false', ...args], + { + cwd: dir, + encoding: 'utf8', + }, + ); + git('init', '-q', '-b', 'main'); + const commit = (file: string, message: string) => { + writeFileSync(join(dir, file), file); + git('add', file); + git('commit', '-q', '-m', message); + }; + const merge = (branch: string, file: string, message: string) => { + git('switch', '-q', '-c', branch); + commit(file, `work on ${file}`); + git('switch', '-q', 'main'); + git('merge', '-q', '--no-ff', branch, '-m', message); + }; + commit('a', 'first'); + merge('p1', 'one', 'Merge phase-1: spikes'); + git('tag', 'v0.1.0'); + merge('p2', 'two', 'Merge phase-2: app shell'); + merge('p3', 'three', 'Merge pull request #7 from XrxcGH/p3\n\nAdd the document model'); + return dir; + } + + it('lists the merges since the previous tag, newest first', () => { + const changes = changesBetween('v0.1.0', 'HEAD', repository()); + expect(changes.map((change) => change.text)).toEqual(['Add the document model', 'App shell']); + expect(changes[0].pr).toBe(7); + }); + + it('lists every merge when there is no previous tag', () => { + expect(changesBetween(undefined, 'HEAD', repository()).map((change) => change.text)).toHaveLength(3); + }); +}); diff --git a/app/scripts/release/notes.ts b/app/scripts/release/notes.ts new file mode 100644 index 00000000..d2d8c04e --- /dev/null +++ b/app/scripts/release/notes.ts @@ -0,0 +1,188 @@ +// Writes the release notes. People read what is in CHANGELOG.md for the version, and the merge commits since the +// previous release give the full list of changes. The same text feeds the GitHub Release page (Markdown) and the +// update notice in the app (plain text, at most 8 lines, because that is all the notice shows). +// Usage: node app/scripts/release/notes.ts [--prev ] [--ref ] [--changelog ] +// [--markdown ] [--plain ] + +import { execFileSync } from 'node:child_process'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { REPO, RELEASE_FILES, versionOf } from '../write-manifest.ts'; +import { compareVersions } from './versions.ts'; + +const ROOT = join(import.meta.dirname, '..', '..', '..'); + +/** The update notice shows at most this many lines of notes. */ +export const MAX_PLAIN_LINES = 8; +export const MAX_PLAIN_BYTES = 4096; + +/** What each download is for, in the words people see on the release page. */ +const FILE_LABELS: Record = { + 'OpenNote_Windows64.exe': '64-bit Windows, on most PCs', + 'OpenNote_Windows32.exe': '32-bit Windows', + 'OpenNote_WindowsARM64.exe': 'Windows on Arm, such as Snapdragon laptops', +}; + +export interface Change { + sha: string; + /** What changed, as one sentence-case line. */ + text: string; + pr?: number; +} + +/** The part of CHANGELOG.md under `## [version]`, up to the next `## ` heading. Undefined when there is none. */ +export function changelogSection(changelog: string, version: string): string | undefined { + const escaped = version.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const heading = new RegExp(`^## \\[?${escaped}\\]?(?:\\s.*)?$`, 'm').exec(changelog); + if (!heading) return undefined; + const rest = changelog.slice(heading.index + heading[0].length); + const next = /^## /m.exec(rest); + const body = (next ? rest.slice(0, next.index) : rest).replace(/^\[[^\]]+\]: .*$/gm, '').trim(); + return body === '' ? undefined : body; +} + +/** A line of release notes with its Markdown removed: links keep their words, and emphasis marks go. */ +export function plainLine(markdown: string): string { + return markdown + .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') + .replace(/[*_`]/g, '') + .replace(/^\s*[-*+]\s+/, '') + .trim(); +} + +/** The bullet lines of a changelog section as plain sentences, without headings or blank lines. */ +export function bulletsOf(section: string): string[] { + return section + .split(/\r?\n/) + .filter((line) => /^\s*[-*+]\s+\S/.test(line)) + .map(plainLine); +} + +function capitalized(text: string): string { + return text.charAt(0).toUpperCase() + text.slice(1); +} + +/** + * One merge commit as a change, or undefined for a merge that only brings a branch up to date. It reads GitHub's + * `Merge pull request #12 from owner/branch` (the title is in the body), and `Merge phase-2: app shell`. + */ +export function changeFromMerge(sha: string, subject: string, body: string): Change | undefined { + const pull = /^Merge pull request #(\d+) from \S+/.exec(subject); + if (pull) { + const title = body.split(/\r?\n/).find((line) => line.trim() !== ''); + return { sha, pr: Number(pull[1]), text: capitalized(title?.trim() ?? subject) }; + } + if (/^Merge (branch|remote-tracking branch|tag) /.test(subject) || /^Merge \S+ into \S+$/.test(subject)) { + return undefined; + } + const named = /^Merge [\w./-]+: (.+)$/.exec(subject); + return { sha, text: capitalized(named ? named[1] : subject.replace(/^Merge /, '')) }; +} + +/** Parses `git log --format=%H%x1f%s%x1f%b%x1e` output into changes, newest first. */ +export function parseLog(log: string, merges: boolean): Change[] { + const changes: Change[] = []; + for (const record of log.split('\x1e')) { + const [sha, subject, body] = record.replace(/^\s+/, '').split('\x1f'); + if (!sha || subject === undefined) continue; + const change = merges ? changeFromMerge(sha, subject, body ?? '') : { sha, text: capitalized(subject) }; + if (change) changes.push(change); + } + return changes; +} + +/** The newest tag that is older than `tag`. A stable release looks back to the previous stable one. */ +export function previousTag(tags: readonly string[], tag: string): string | undefined { + const stable = !versionOf(tag).includes('-'); + const older = tags.filter((other) => other !== tag && compareVersions(versionOf(other), versionOf(tag)) < 0); + const candidates = stable ? older.filter((other) => !versionOf(other).includes('-')) : older; + return candidates.sort((a, b) => compareVersions(versionOf(b), versionOf(a)))[0]; +} + +export interface NotesInput { + tag: string; + changelog?: string; + changes: readonly Change[]; + previous?: string; +} + +function downloads(): string { + const rows = RELEASE_FILES.map(({ file }) => `| ${FILE_LABELS[file] ?? file} | \`${file}\` |`); + return ['| For | Download |', '| --- | --- |', ...rows].join('\n'); +} + +function changeLine(change: Change): string { + const link = change.pr ? ` ([#${change.pr}](https://github.com/${REPO}/pull/${change.pr}))` : ''; + // A commit subject is plain text, but a < in it would start an HTML tag on the release page and hide the words. + const text = change.text.replace(//g, '>'); + return `- ${text}${link}`; +} + +/** The release page: what is new, which file to download, and every change since the last release. */ +export function markdownNotes({ tag, changelog, changes, previous }: NotesInput): string { + const section = changelog === undefined ? undefined : changelogSection(changelog, versionOf(tag)); + const parts = [section ? `## What's new\n\n${section}` : `## OpenNote ${versionOf(tag)}`]; + parts.push(`## Download\n\nChoose the file for your PC. After that, the app updates itself.\n\n${downloads()}`); + if (changes.length > 0) { + const list = changes.map(changeLine).join('\n'); + parts.push(`
\nAll changes (${changes.length})\n\n${list}\n\n
`); + } + if (previous) { + parts.push(`Full list of commits: https://github.com/${REPO}/compare/${previous}...${tag}`); + } + return `${parts.join('\n\n')}\n`; +} + +/** The notes for the update notice in the app: plain sentences, one per line, within what the notice can show. */ +export function plainNotes({ tag, changelog, changes }: NotesInput): string { + const section = changelog === undefined ? undefined : changelogSection(changelog, versionOf(tag)); + const items = section ? bulletsOf(section) : changes.map((change) => change.text); + if (items.length === 0) return `OpenNote ${tag}`; + const link = `Full notes: https://github.com/${REPO}/releases/tag/${tag}`; + const lines = + items.length <= MAX_PLAIN_LINES + ? items + : [...items.slice(0, MAX_PLAIN_LINES - 2), `And ${items.length - (MAX_PLAIN_LINES - 2)} more.`, link]; + const text = lines.join('\n'); + return Buffer.byteLength(text) <= MAX_PLAIN_BYTES ? text : `OpenNote ${tag}\n${link}`; +} + +function git(args: string[], cwd: string = ROOT): string { + return execFileSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }); +} + +/** The changes between two refs: its merge commits on the main line, or its plain commits when there are none. */ +export function changesBetween(previous: string | undefined, ref: string, cwd: string = ROOT): Change[] { + const range = previous ? `${previous}..${ref}` : ref; + const format = '--format=%H%x1f%s%x1f%b%x1e'; + const merges = parseLog(git(['log', '--first-parent', '--merges', format, range], cwd), true); + if (merges.length > 0) return merges; + return parseLog(git(['log', '--first-parent', '--no-merges', format, range], cwd), false); +} + +function option(args: string[], name: string): string | undefined { + const at = args.indexOf(name); + return at === -1 ? undefined : args[at + 1]; +} + +function main(): void { + const args = process.argv.slice(2); + const tag = args[0]; + if (!tag || tag.startsWith('--')) + throw new Error('Usage: node app/scripts/release/notes.ts [--prev ] [--ref ]'); + const ref = option(args, '--ref') ?? 'HEAD'; + const previous = + option(args, '--prev') ?? previousTag(git(['tag', '--list', 'v*']).split(/\s+/).filter(Boolean), tag); + const changelogPath = option(args, '--changelog') ?? join(ROOT, 'CHANGELOG.md'); + const input: NotesInput = { + tag, + changelog: readFileSync(changelogPath, 'utf8'), + changes: changesBetween(previous, ref), + previous, + }; + writeFileSync(option(args, '--markdown') ?? 'release-notes.md', markdownNotes(input)); + writeFileSync(option(args, '--plain') ?? 'release-notes.txt', `${plainNotes(input)}\n`); + console.log(`Wrote release notes for ${tag}: ${input.changes.length} changes since ${previous ?? 'the start'}.`); +} + +if (import.meta.main) main(); diff --git a/app/scripts/release/pe.test.ts b/app/scripts/release/pe.test.ts new file mode 100644 index 00000000..0b9b9ca9 --- /dev/null +++ b/app/scripts/release/pe.test.ts @@ -0,0 +1,47 @@ +// @vitest-environment node +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { hasAuthenticode, windowsSignatureStatus } from './pe.ts'; +import { fakePe } from './test-support.ts'; + +describe('hasAuthenticode', () => { + it.each([true, false])('finds the certificate table of a PE32+ or PE32 file (plus: %s)', (plus) => { + expect(hasAuthenticode(fakePe({ plus, signed: true }))).toBe(true); + expect(hasAuthenticode(fakePe({ plus, signed: false }))).toBe(false); + }); + + it('treats a table that points outside the file as unsigned', () => { + const exe = fakePe({ signed: true }); + exe.writeUInt32LE(0x1000, 0x80 + 24 + 112 + 32); + expect(hasAuthenticode(exe)).toBe(false); + }); + + it('treats a header with too few directories as unsigned', () => { + expect(hasAuthenticode(fakePe({ signed: true, directories: 3 }))).toBe(false); + }); + + it('treats other files as unsigned without crashing', () => { + expect(hasAuthenticode(Buffer.alloc(0))).toBe(false); + expect(hasAuthenticode(Buffer.from('MZ'))).toBe(false); + expect(hasAuthenticode(Buffer.from('not an exe at all, just some text that is long enough'))).toBe(false); + const wrongSignature = fakePe({ signed: true }); + wrongSignature.write('XX', 0x80, 'latin1'); + expect(hasAuthenticode(wrongSignature)).toBe(false); + const pointer = fakePe(); + pointer.writeUInt32LE(0xfffffff0, 0x3c); + expect(hasAuthenticode(pointer)).toBe(false); + }); +}); + +describe.skipIf(process.platform !== 'win32')('on Windows', () => { + it('finds the signature of the Node program that runs the tests, and asks Windows about it', () => { + expect(hasAuthenticode(readFileSync(process.execPath))).toBe(true); + expect(windowsSignatureStatus(process.execPath)).toMatch(/^[A-Za-z]+$/); + }, 60_000); +}); + +describe.skipIf(process.platform === 'win32')('off Windows', () => { + it('has no answer from the operating system', () => { + expect(windowsSignatureStatus(process.execPath)).toBeUndefined(); + }); +}); diff --git a/app/scripts/release/pe.ts b/app/scripts/release/pe.ts new file mode 100644 index 00000000..9457c8f3 --- /dev/null +++ b/app/scripts/release/pe.ts @@ -0,0 +1,48 @@ +// Tells whether a Windows exe carries an embedded Authenticode signature, without asking the operating system, so +// the release checklist can check it on any machine. This only finds the signature. Windows decides whether the +// certificate chain is valid, and `windowsSignatureStatus` asks it when the script runs on Windows. + +import { execFileSync } from 'node:child_process'; + +const PE_POINTER = 0x3c; +const PE32 = 0x10b; +const PE32_PLUS = 0x20b; +/** The certificate table is the fifth entry in the optional header's data directory. */ +const SECURITY_ENTRY = 4; + +/** + * True when the PE file has a certificate table, which is where Authenticode puts its signature. A file that is + * not a PE file, or is cut short, counts as unsigned. + */ +export function hasAuthenticode(exe: Buffer): boolean { + if (exe.length < PE_POINTER + 4) return false; + const header = exe.readUInt32LE(PE_POINTER); + if (header + 24 + 2 > exe.length || exe.toString('latin1', header, header + 4) !== 'PE\0\0') return false; + const optional = header + 24; + const magic = exe.readUInt16LE(optional); + if (magic !== PE32 && magic !== PE32_PLUS) return false; + const countAt = optional + (magic === PE32 ? 92 : 108); + const entryAt = optional + (magic === PE32 ? 96 : 112) + SECURITY_ENTRY * 8; + if (entryAt + 8 > exe.length || exe.readUInt32LE(countAt) <= SECURITY_ENTRY) return false; + const offset = exe.readUInt32LE(entryAt); + const size = exe.readUInt32LE(entryAt + 4); + return offset > 0 && size > 0 && offset + size <= exe.length; +} + +/** + * What Windows says about the signature: `Valid`, or another status such as `NotSigned` or `UnknownError`. It + * returns undefined off Windows, where nothing can answer. + */ +export function windowsSignatureStatus(path: string): string | undefined { + if (process.platform !== 'win32') return undefined; + // The path goes in an environment variable, so no file name can change what the command does. + const script = '(Get-AuthenticodeSignature -LiteralPath $env:OPENNOTE_EXE).Status'; + // PSModulePath is left out so Windows PowerShell builds its own. Started from PowerShell 7 (as on CI runners), the + // inherited one lists PowerShell 7's modules first, and Windows PowerShell then fails to load the Security module. + const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => name.toLowerCase() !== 'psmodulepath')); + const output = execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], { + encoding: 'utf8', + env: { ...env, OPENNOTE_EXE: path }, + }); + return output.trim(); +} diff --git a/app/scripts/release/sbom-cargo.ts b/app/scripts/release/sbom-cargo.ts new file mode 100644 index 00000000..49a54191 --- /dev/null +++ b/app/scripts/release/sbom-cargo.ts @@ -0,0 +1,90 @@ +// The Rust side of the software bill of materials: the crates that are compiled into the exe. +// +// `cargo metadata` knows which crates the app needs on Windows, and whether each is needed to run or only to build. +// The lockfile has the SHA-256 of each downloaded crate. + +export interface CargoDependency { + pkg: string; + dep_kinds: { kind: string | null; target: string | null }[]; +} + +export interface CargoMetadata { + packages: { + id: string; + name: string; + version: string; + license: string | null; + repository: string | null; + source: string | null; + }[]; + resolve: { nodes: { id: string; deps: CargoDependency[] }[] } | null; +} + +export interface Crate { + name: string; + version: string; + license?: string; + repository?: string; + /** SHA-256 of the downloaded crate, as lowercase hex. Local crates have none. */ + sha256?: string; + /** True for a crate in this repository. */ + local: boolean; + /** Ids ("name@version") of the crates this one needs. */ + needs: string[]; +} + +export const crateId = (name: string, version: string): string => `${name}@${version}`; + +/** The checksum of each crate in a Cargo.lock, by "name@version". */ +export function lockChecksums(lock: string): Map { + const sums = new Map(); + for (const block of lock.split('[[package]]').slice(1)) { + const name = /^name = "([^"]+)"/m.exec(block)?.[1]; + const version = /^version = "([^"]+)"/m.exec(block)?.[1]; + const sum = /^checksum = "([0-9a-f]{64})"/m.exec(block)?.[1]; + if (name && version && sum) sums.set(crateId(name, version), sum); + } + return sums; +} + +/** Whether a crate is needed in the finished exe: a normal dependency, or a build dependency used to compile it. */ +const isShipped = (dependency: CargoDependency): boolean => dependency.dep_kinds.some(({ kind }) => kind !== 'dev'); + +/** + * The crates `root` needs, found by following dependencies that aren't dev-only. Pass one metadata result for each + * target the release builds (cargo metadata --filter-platform), and a crate that any target needs is included. + */ +export function shippedCrates( + results: readonly CargoMetadata[], + root: string, + checksums: Map, +): Crate[] { + const crates = new Map(); + for (const metadata of results) { + const byId = new Map(metadata.packages.map((pkg) => [pkg.id, pkg])); + const nodes = new Map((metadata.resolve?.nodes ?? []).map((node) => [node.id, node])); + const start = metadata.packages.find((pkg) => pkg.name === root && pkg.source === null); + if (!start) throw new Error(`The cargo metadata has no crate named ${root}.`); + const visited = new Set(); + const visit = (id: string): void => { + const pkg = byId.get(id); + if (!pkg || visited.has(id)) return; + visited.add(id); + const key = crateId(pkg.name, pkg.version); + const needs = (nodes.get(id)?.deps ?? []).filter(isShipped).flatMap((dep) => byId.get(dep.pkg) ?? []); + const ids = new Set([...(crates.get(key)?.needs ?? []), ...needs.map((dep) => crateId(dep.name, dep.version))]); + crates.set(key, { + name: pkg.name, + version: pkg.version, + license: pkg.license ?? undefined, + repository: pkg.repository ?? undefined, + sha256: checksums.get(key), + local: pkg.source === null, + needs: [...ids].sort(), + }); + needs.forEach((dep) => visit(dep.id)); + }; + visit(start.id); + } + return [...crates.values()].sort((a, b) => crateId(a.name, a.version).localeCompare(crateId(b.name, b.version))); +} diff --git a/app/scripts/release/sbom-imports.test.ts b/app/scripts/release/sbom-imports.test.ts new file mode 100644 index 00000000..473c41bd --- /dev/null +++ b/app/scripts/release/sbom-imports.test.ts @@ -0,0 +1,102 @@ +// @vitest-environment node +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { expandGlob, importedPackages, packageOf, resolveFile } from './sbom-imports.ts'; + +const dirs: string[] = []; +afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); + +/** Writes files (path to text) into a new folder and returns the folder. */ +function project(files: Record): string { + const dir = mkdtempSync(join(tmpdir(), 'opennote-imports-')); + dirs.push(dir); + for (const [path, text] of Object.entries(files)) { + mkdirSync(dirname(join(dir, path)), { recursive: true }); + writeFileSync(join(dir, path), text); + } + return dir; +} + +describe('packageOf', () => { + it('names the package a bare import points to', () => { + expect(packageOf('react-dom/client')).toBe('react-dom'); + expect(packageOf('@tauri-apps/api/core')).toBe('@tauri-apps/api'); + expect(packageOf('react')).toBe('react'); + }); + + it.each(['./a', '../b', '/c', '~/d', 'node:fs', 'virtual:x'])('ignores %s', (specifier) => { + expect(packageOf(specifier)).toBeUndefined(); + }); +}); + +describe('importedPackages', () => { + it('follows imports from the entry file and ignores files nothing loads', () => { + const dir = project({ + 'main.tsx': "import React from 'react';\nimport { App } from './App';\nimport './styles.css';\n", + 'App.tsx': "import { useState } from 'react';\nexport const App = () => import('./lazy.ts');\n", + 'lazy.ts': "export { y } from '@scope/editor/extra';\n", + 'styles.css': "@import '@fontsource/mono';\n@import './more.css';\n", + 'more.css': 'body {}\n', + 'App.test.tsx': "import { it } from 'vitest';\n", + 'test/setup.ts': "import 'jsdom';\n", + 'unused.ts': "import 'lodash';\n", + }); + expect(importedPackages(join(dir, 'main.tsx'))).toEqual(['@fontsource/mono', '@scope/editor', 'react']); + }); + + it('skips type-only imports, which are erased when the code is compiled', () => { + const dir = project({ + 'main.ts': + "import type { A } from 'only-types';\nimport { b } from 'real';\nexport type { C } from 'also-types';\n", + }); + expect(importedPackages(join(dir, 'main.ts'))).toEqual(['real']); + }); + + it('reads imports that span lines, side-effect imports, and index files', () => { + const dir = project({ + 'main.ts': "import {\n one,\n two,\n} from 'multi';\nimport 'side-effect';\nimport './feature';\n", + 'feature/index.ts': "export * from 'deep';\n", + }); + expect(importedPackages(join(dir, 'main.ts'))).toEqual(['deep', 'multi', 'side-effect']); + }); + + it('follows import.meta.glob to the files it loads', () => { + const dir = project({ + 'index.ts': "const modules = import.meta.glob('./*/register.ts', { eager: true });\n", + 'a/register.ts': "import 'from-a';\n", + 'b/register.ts': "import 'from-b';\n", + 'b/other.ts': "import 'not-loaded';\n", + }); + expect(importedPackages(join(dir, 'index.ts'))).toEqual(['from-a', 'from-b']); + }); + + it('survives import cycles', () => { + const dir = project({ 'a.ts': "import './b';\nimport 'x';\n", 'b.ts': "import './a';\nimport 'y';\n" }); + expect(importedPackages(join(dir, 'a.ts'))).toEqual(['x', 'y']); + }); +}); + +describe('resolveFile and expandGlob', () => { + it('tries the extensions the bundler tries', () => { + const dir = project({ 'main.ts': '', 'a.tsx': '', 'b/index.ts': '', 'c.css': '' }); + const main = join(dir, 'main.ts'); + expect(resolveFile(main, './a')).toBe(join(dir, 'a.tsx')); + expect(resolveFile(main, './b')).toBe(join(dir, 'b', 'index.ts')); + expect(resolveFile(main, './c.css')).toBe(join(dir, 'c.css')); + expect(resolveFile(main, './missing')).toBeUndefined(); + }); + + it('matches * within a folder and ** across folders', () => { + const dir = project({ 'main.ts': '', 'x/a.ts': '', 'x/y/b.ts': '', 'z.ts': '' }); + const main = join(dir, 'main.ts'); + const names = (pattern: string) => + expandGlob(main, pattern) + .map((path) => path.slice(dir.length + 1).replace(/\\/g, '/')) + .sort(); + expect(names('./x/*.ts')).toEqual(['x/a.ts']); + expect(names('./**/*.ts')).toEqual(['main.ts', 'x/a.ts', 'x/y/b.ts', 'z.ts']); + expect(names('./nothing/*.ts')).toEqual([]); + }); +}); diff --git a/app/scripts/release/sbom-imports.ts b/app/scripts/release/sbom-imports.ts new file mode 100644 index 00000000..8f341904 --- /dev/null +++ b/app/scripts/release/sbom-imports.ts @@ -0,0 +1,99 @@ +// Finds which npm packages the interface bundle uses, the way the bundler does. It starts at the entry file and +// follows every import to the files it loads. Each package those files import is noted. A file nothing reaches, +// such as a test or a test helper, never ships, so its packages stay out of the bill of materials. Type-only +// imports are +// erased when the code is compiled, so they don't count either. + +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import { dirname, join, relative, resolve } from 'node:path'; + +const EXTENSIONS = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '/index.ts', '/index.tsx', '/index.js']; +const SOURCE = /\.([cm]?[jt]sx?|css)$/; +/** `import x from 'a'`, `export { y } from 'a'`, and the side-effect form `import 'a'`. A `type` import is skipped. */ +const STATEMENT = + /^[ \t]*(?:import|export)\s+(?!type\b)[^;'"]*?\bfrom\s*['"]([^'"]+)['"]|^[ \t]*import\s*['"]([^'"]+)['"]/gm; +const DYNAMIC = /\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g; +const GLOB = /\bimport\.meta\.glob\(\s*['"]([^'"]+)['"]/g; +const CSS_IMPORT = /@import\s+(?:url\()?['"]([^'"]+)['"]/g; + +/** The package a bare import names: `react-dom/client` is `react-dom`, and `@tiptap/core/x` is `@tiptap/core`. */ +export function packageOf(specifier: string): string | undefined { + if (/^[./~]|^node:|^virtual:/.test(specifier)) return undefined; + const parts = specifier.split('/'); + return specifier.startsWith('@') ? parts.slice(0, 2).join('/') : parts[0]; +} + +/** The file a relative import loads, trying the same file extensions and index files as the bundler. */ +export function resolveFile(from: string, specifier: string): string | undefined { + const base = resolve(dirname(from), specifier); + return EXTENSIONS.map((extension) => resolve(base + extension)).find( + (path) => existsSync(path) && statSync(path).isFile(), + ); +} + +function filesUnder(dir: string): string[] { + if (!existsSync(dir)) return []; + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const path = join(dir, entry.name); + return entry.isDirectory() ? filesUnder(path) : [path]; + }); +} + +/** A glob as a regular expression, for `*` (within one folder) and `**` (across folders) only. */ +function globToRegExp(pattern: string): RegExp { + let source = ''; + for (let at = 0; at < pattern.length; at++) { + const char = pattern[at]; + if (char === '*' && pattern[at + 1] === '*') { + const slash = pattern[at + 2] === '/'; + source += slash ? '(?:.*/)?' : '.*'; + at += slash ? 2 : 1; + } else { + source += char === '*' ? '[^/]*' : char.replace(/[.+^${}()|[\]\\?]/g, '\\$&'); + } + } + return new RegExp(`^${source}$`); +} + +/** The files that a glob pattern from import.meta.glob matches, such as the register.ts file in each folder. */ +export function expandGlob(from: string, pattern: string): string[] { + const fixed = pattern.split('*')[0]; + const base = resolve(dirname(from), fixed.slice(0, fixed.lastIndexOf('/') + 1)); + const matcher = globToRegExp(pattern.replace(/^\.\//, '')); + return filesUnder(base).filter((path) => matcher.test(relative(dirname(from), path).replace(/\\/g, '/'))); +} + +/** The import specifiers in a file, and the glob patterns it expands. */ +function referencesIn(path: string, text: string): { specifiers: string[]; globs: string[] } { + const specifiers: string[] = []; + if (path.endsWith('.css')) { + specifiers.push(...[...text.matchAll(CSS_IMPORT)].map((match) => match[1])); + } else { + for (const match of text.matchAll(STATEMENT)) specifiers.push(match[1] ?? match[2]); + specifiers.push(...[...text.matchAll(DYNAMIC)].map((match) => match[1])); + } + return { specifiers, globs: [...text.matchAll(GLOB)].map((match) => match[1]) }; +} + +/** Every package that the files reachable from `entry` import, sorted. */ +export function importedPackages(entry: string): string[] { + const packages = new Set(); + const seen = new Set(); + const queue = [resolve(entry)]; + while (queue.length > 0) { + const path = queue.pop() as string; + if (seen.has(path) || !SOURCE.test(path)) continue; + seen.add(path); + const { specifiers, globs } = referencesIn(path, readFileSync(path, 'utf8')); + for (const specifier of specifiers) { + const name = packageOf(specifier); + if (name) packages.add(name); + else if (specifier.startsWith('.')) { + const file = resolveFile(path, specifier); + if (file) queue.push(file); + } + } + for (const pattern of globs) queue.push(...expandGlob(path, pattern)); + } + return [...packages].sort(); +} diff --git a/app/scripts/release/sbom-npm.ts b/app/scripts/release/sbom-npm.ts new file mode 100644 index 00000000..c500c1ca --- /dev/null +++ b/app/scripts/release/sbom-npm.ts @@ -0,0 +1,79 @@ +// The npm side of the software bill of materials: the packages that end up inside the exe. The interface is built +// by Vite from the packages that the app source imports, and some of them, such as the editor, are devDependencies. +// So "shipped" can't be read from the lockfile's dev flag. This starts from the packages that sbom-imports.ts found +// in the interface bundle, and follows package-lock.json from there. + +export interface LockPackage { + version?: string; + resolved?: string; + integrity?: string; + license?: string; + dependencies?: Record; + optionalDependencies?: Record; + peerDependencies?: Record; + link?: boolean; +} + +export interface Lockfile { + packages: Record; +} + +export interface NpmPackage { + name: string; + version: string; + license?: string; + /** SHA-512 as lowercase hex, from the lockfile's integrity field. */ + sha512?: string; + resolved?: string; + /** Lockfile paths of the packages this one needs. */ + needs: string[]; + path: string; + /** True when the app source imports it, rather than another package needing it. */ + direct: boolean; +} + +/** Where `name` resolves from the package at `from`: its own node_modules, then each parent's, as Node does. */ +export function resolveDependency(lock: Lockfile, from: string, name: string): string | undefined { + let base = from; + for (;;) { + const candidate = `${base ? `${base}/` : ''}node_modules/${name}`; + if (lock.packages[candidate]) return candidate; + if (base === '') return undefined; + const cut = base.lastIndexOf('/node_modules/'); + base = cut === -1 ? '' : base.slice(0, cut); + } +} + +function integrityHex(integrity: string | undefined): string | undefined { + const match = /^sha512-(.+)$/.exec(integrity ?? ''); + return match ? Buffer.from(match[1], 'base64').toString('hex') : undefined; +} + +function nameAt(path: string): string { + return path.slice(path.lastIndexOf('node_modules/') + 'node_modules/'.length); +} + +/** The packages the exe needs: the imported ones and everything they depend on, by the lockfile. */ +export function shippedPackages(lock: Lockfile, imported: readonly string[]): NpmPackage[] { + const result = new Map(); + const roots = imported.flatMap((name) => resolveDependency(lock, '', name) ?? []); + const visit = (path: string): void => { + const entry = lock.packages[path]; + if (result.has(path) || !entry?.version) return; + const names = Object.keys({ ...entry.dependencies, ...entry.optionalDependencies, ...entry.peerDependencies }); + const needs = names.flatMap((name) => resolveDependency(lock, path, name) ?? []); + result.set(path, { + name: nameAt(path), + version: entry.version, + license: entry.license, + sha512: integrityHex(entry.integrity), + resolved: entry.resolved, + needs, + path, + direct: roots.includes(path), + }); + needs.forEach(visit); + }; + roots.forEach(visit); + return [...result.values()].sort((a, b) => a.path.localeCompare(b.path)); +} diff --git a/app/scripts/release/sbom.test.ts b/app/scripts/release/sbom.test.ts new file mode 100644 index 00000000..0f2c8084 --- /dev/null +++ b/app/scripts/release/sbom.test.ts @@ -0,0 +1,231 @@ +// @vitest-environment node +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { lockChecksums, shippedCrates, type CargoMetadata } from './sbom-cargo.ts'; +import { importedPackages } from './sbom-imports.ts'; +import { resolveDependency, shippedPackages, type Lockfile } from './sbom-npm.ts'; +import { buildSbom, cargoPurl, licenseOf, mergeCrates, NATIVE_FEATURES, NATIVE_TARGET, npmPurl } from './sbom.ts'; + +const ROOT = join(import.meta.dirname, '..', '..', '..'); +const HEX = Buffer.from('hash').toString('hex'); +const LOCK: Lockfile = { + packages: { + '': { version: '1.0.0' }, + 'node_modules/react': { + version: '19.0.0', + license: 'MIT', + integrity: `sha512-${Buffer.from('hash').toString('base64')}`, + resolved: 'https://registry.npmjs.org/react/-/react-19.0.0.tgz', + dependencies: { scheduler: '^1' }, + }, + 'node_modules/scheduler': { version: '1.0.0', license: 'MIT' }, + 'node_modules/@scope/editor': { + version: '3.0.0', + license: 'Apache-2.0 OR MIT', + dependencies: { scheduler: '^2' }, + peerDependencies: { missing: '*' }, + }, + 'node_modules/@scope/editor/node_modules/scheduler': { version: '2.0.0', license: 'ISC' }, + 'node_modules/eslint': { version: '10.0.0', license: 'MIT' }, + }, +}; + +describe('which npm packages ship', () => { + it('finds the copy of a package that Node would load', () => { + expect(resolveDependency(LOCK, '', 'scheduler')).toBe('node_modules/scheduler'); + expect(resolveDependency(LOCK, 'node_modules/@scope/editor', 'scheduler')).toBe( + 'node_modules/@scope/editor/node_modules/scheduler', + ); + expect(resolveDependency(LOCK, 'node_modules/react', 'scheduler')).toBe('node_modules/scheduler'); + expect(resolveDependency(LOCK, '', 'missing')).toBeUndefined(); + }); + + it('follows the lockfile from the imports, so dev tools stay out and both schedulers stay in', () => { + const packages = shippedPackages(LOCK, ['react', '@scope/editor', 'not-installed']); + expect(packages.map((pkg) => `${pkg.name}@${pkg.version}`)).toEqual([ + '@scope/editor@3.0.0', + 'scheduler@2.0.0', + 'react@19.0.0', + 'scheduler@1.0.0', + ]); + const react = packages.find((pkg) => pkg.name === 'react'); + expect(react).toMatchObject({ direct: true, sha512: HEX, needs: ['node_modules/scheduler'] }); + expect(packages.find((pkg) => pkg.version === '1.0.0')?.direct).toBe(false); + }); + + it('finds the real dependencies of the app in this repository', () => { + const lock = JSON.parse(readFileSync(join(ROOT, 'package-lock.json'), 'utf8')) as Lockfile; + const names = shippedPackages(lock, importedPackages(join(ROOT, 'app', 'src', 'main.tsx'))).map((pkg) => pkg.name); + expect(names).toEqual(expect.arrayContaining(['react', 'react-dom', '@tauri-apps/api'])); + expect(names).not.toContain('eslint'); + expect(names).not.toContain('vitest'); + }, 60_000); +}); + +const metadata = (extra: Partial = {}): CargoMetadata => ({ + packages: [ + { id: 'app', name: 'opennote', version: '1.0.0', license: 'Apache-2.0', repository: null, source: null }, + { + id: 'serde', + name: 'serde', + version: '1.0.1', + license: 'MIT/Apache-2.0', + repository: 'https://github.com/serde-rs/serde', + source: 'registry', + }, + { id: 'cc', name: 'cc', version: '1.2.3', license: 'MIT OR Apache-2.0', repository: null, source: 'registry' }, + { id: 'proptest', name: 'proptest', version: '1.0.0', license: 'MIT', repository: null, source: 'registry' }, + { id: 'windows', name: 'windows', version: '0.62.0', license: 'MIT', repository: null, source: 'registry' }, + ], + resolve: { + nodes: [ + { + id: 'app', + deps: [ + { pkg: 'serde', dep_kinds: [{ kind: null, target: null }] }, + { pkg: 'cc', dep_kinds: [{ kind: 'build', target: null }] }, + { pkg: 'proptest', dep_kinds: [{ kind: 'dev', target: null }] }, + { pkg: 'windows', dep_kinds: [{ kind: null, target: 'cfg(windows)' }] }, + ], + }, + { id: 'serde', deps: [] }, + { id: 'cc', deps: [] }, + { id: 'proptest', deps: [] }, + { id: 'windows', deps: [] }, + ], + ...extra, + }, +}); + +/** A Cargo.lock with these crates: name, version, and an optional checksum. */ +function cargoLock(crates: [string, string, string?][]): string { + const block = ([name, version, sum]: [string, string, string?]) => + ['[[package]]', `name = "${name}"`, `version = "${version}"`, ...(sum ? [`checksum = "${sum}"`] : [])].join('\n'); + return crates.map(block).join('\n\n'); +} + +describe('which crates ship', () => { + const lock = cargoLock([ + ['serde', '1.0.1', 'ab'.repeat(32)], + ['opennote', '1.0.0'], + ]); + + it('reads checksums from Cargo.lock, and none for crates in this repository', () => { + expect([...lockChecksums(lock)]).toEqual([['serde@1.0.1', 'ab'.repeat(32)]]); + }); + + it('keeps normal and build dependencies and drops dev-only ones', () => { + const crates = shippedCrates([metadata()], 'opennote', lockChecksums(lock)); + expect(crates.map((crate) => crate.name)).toEqual(['cc', 'opennote', 'serde', 'windows']); + const app = crates.find((crate) => crate.name === 'opennote'); + expect(app).toMatchObject({ local: true, needs: ['cc@1.2.3', 'serde@1.0.1', 'windows@0.62.0'] }); + expect(crates.find((crate) => crate.name === 'serde')).toMatchObject({ sha256: 'ab'.repeat(32), local: false }); + }); + + it('includes a crate that any target needs, once', () => { + const arm = metadata(); + arm.resolve?.nodes[0].deps.pop(); + const crates = shippedCrates([arm, metadata()], 'opennote', new Map()); + expect(crates.filter((crate) => crate.name === 'windows')).toHaveLength(1); + expect(crates.find((crate) => crate.name === 'opennote')?.needs).toContain('windows@0.62.0'); + }); + + it('stops when the app crate is not in the metadata', () => { + expect(() => shippedCrates([metadata()], 'other', new Map())).toThrow('no crate named other'); + }); +}); + +describe('licenseOf', () => { + it('writes SPDX expressions as expressions, and old slash forms with OR', () => { + expect(licenseOf('MIT')).toEqual([{ expression: 'MIT' }]); + expect(licenseOf('MIT/Apache-2.0')).toEqual([{ expression: 'MIT OR Apache-2.0' }]); + expect(licenseOf('(MIT OR Apache-2.0) AND BSD-3-Clause')).toEqual([ + { expression: '(MIT OR Apache-2.0) AND BSD-3-Clause' }, + ]); + expect(licenseOf('Apache-2.0 WITH LLVM-exception')).toEqual([{ expression: 'Apache-2.0 WITH LLVM-exception' }]); + }); + + it('keeps anything else as a name, and says nothing when there is no license', () => { + expect(licenseOf('SEE LICENSE IN LICENSE.txt')).toEqual([{ license: { name: 'SEE LICENSE IN LICENSE.txt' } }]); + expect(licenseOf('Custom ')).toEqual([{ license: { name: 'Custom ' } }]); + expect(licenseOf(' ')).toBeUndefined(); + expect(licenseOf(undefined)).toBeUndefined(); + }); +}); + +describe('buildSbom', () => { + const sbom = buildSbom({ + version: '1.0.0', + npm: shippedPackages(LOCK, ['react', '@scope/editor']), + crates: shippedCrates( + [metadata()], + 'opennote', + lockChecksums(`[[package]]\nname = "serde"\nversion = "1.0.1"\nchecksum = "${'ab'.repeat(32)}"\n`), + ), + serial: 'urn:uuid:11111111-2222-4333-8444-555555555555', + timestamp: '2026-10-14T18:02:11.000Z', + }); + + it('is a CycloneDX 1.5 document about OpenNote', () => { + expect(sbom).toMatchObject({ bomFormat: 'CycloneDX', specVersion: '1.5', version: 1 }); + expect(sbom.serialNumber).toMatch(/^urn:uuid:[0-9a-f-]{36}$/); + expect(sbom.metadata.component).toMatchObject({ name: 'OpenNote', version: '1.0.0', type: 'application' }); + }); + + it('lists each component once, with a package URL, license, and hash', () => { + const refs = sbom.components.map((component) => component['bom-ref']); + expect(new Set(refs).size).toBe(refs.length); + const react = sbom.components.find((component) => component.name === 'react'); + expect(react).toMatchObject({ + purl: npmPurl('react', '19.0.0'), + licenses: [{ expression: 'MIT' }], + hashes: [{ alg: 'SHA-512', content: HEX }], + }); + const serde = sbom.components.find((component) => component.name === 'serde'); + expect(serde).toMatchObject({ + purl: cargoPurl('serde', '1.0.1'), + licenses: [{ expression: 'MIT OR Apache-2.0' }], + hashes: [{ alg: 'SHA-256', content: 'ab'.repeat(32) }], + }); + expect(npmPurl('@scope/editor', '3.0.0')).toBe('pkg:npm/%40scope/editor@3.0.0'); + }); + + it('records what needs what, and every reference points at a listed component', () => { + const known = new Set([ + sbom.metadata.component['bom-ref'], + ...sbom.components.map((component) => component['bom-ref']), + ]); + for (const entry of sbom.dependencies) { + expect(known.has(entry.ref), entry.ref).toBe(true); + entry.dependsOn.forEach((ref) => expect(known.has(ref), ref).toBe(true)); + } + const root = sbom.dependencies[0]; + expect(root.dependsOn).toEqual([ + 'pkg:cargo/opennote@1.0.0', + 'pkg:npm/%40scope/editor@3.0.0', + 'pkg:npm/react@19.0.0', + ]); + const reactNeeds = sbom.dependencies.find((entry) => entry.ref === 'pkg:npm/react@19.0.0')?.dependsOn; + expect(reactNeeds).toEqual(['pkg:npm/scheduler@1.0.0']); + }); + + it('is the same every time for the same input', () => { + expect(JSON.stringify(sbom)).toBe(JSON.stringify(structuredClone(sbom))); + }); +}); + +describe('mergeCrates', () => { + const crate = (name: string, needs: string[]) => ({ name, version: '1.0.0', local: false, needs }); + + it('lists a crate once and joins what each list says it needs', () => { + const merged = mergeCrates([crate('a', ['b@1.0.0']), crate('b', [])], [crate('a', ['c@1.0.0']), crate('c', [])]); + expect(merged.map((one) => one.name)).toEqual(['a', 'b', 'c']); + expect(merged[0]?.needs).toEqual(['b@1.0.0', 'c@1.0.0']); + }); + + it('builds the 64-bit exe with the native features and no other', () => { + expect(NATIVE_TARGET).toBe('x86_64-pc-windows-msvc'); + expect(NATIVE_FEATURES).toBe('opennote/native-engines,opennote/embed-llm-host,opennote-llm/llama'); + }); +}); diff --git a/app/scripts/release/sbom.ts b/app/scripts/release/sbom.ts new file mode 100644 index 00000000..1f09dd10 --- /dev/null +++ b/app/scripts/release/sbom.ts @@ -0,0 +1,206 @@ +// Writes the software bill of materials for a release: every npm package and Rust crate that is built into the +// exes, with versions, licenses, and hashes, as CycloneDX 1.5 JSON. It is a release file, so people and tools can +// see what OpenNote is made of without building it. The npm and Rust halves are in sbom-npm.ts and sbom-cargo.ts. +// Usage: node app/scripts/release/sbom.ts [--serial ] [--timestamp ] + +import { execFileSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { RELEASE_FILES, REPO } from '../write-manifest.ts'; +import { lockChecksums, shippedCrates, type CargoMetadata, type Crate } from './sbom-cargo.ts'; +import { importedPackages } from './sbom-imports.ts'; +import { shippedPackages, type Lockfile, type NpmPackage } from './sbom-npm.ts'; + +const ROOT = join(import.meta.dirname, '..', '..', '..'); +const APP_CRATE = 'opennote'; +/** The language model program that travels inside the exe of the target that carries the native engines. */ +const LLM_CRATE = 'opennote-llm'; +/** The one target whose exe carries whisper.cpp, ONNX Runtime, and the language model program. */ +export const NATIVE_TARGET = 'x86_64-pc-windows-msvc'; +/** The features that target is built with (see .github/workflows/release.yml). */ +export const NATIVE_FEATURES = 'opennote/native-engines,opennote/embed-llm-host,opennote-llm/llama'; + +export interface Component { + type: 'application' | 'library'; + 'bom-ref': string; + name: string; + version: string; + purl: string; + scope?: 'required'; + licenses?: ({ expression: string } | { license: { name: string } })[]; + hashes?: { alg: 'SHA-256' | 'SHA-512'; content: string }[]; + externalReferences?: { type: string; url: string }[]; +} + +export interface Bom { + bomFormat: 'CycloneDX'; + specVersion: '1.5'; + serialNumber: string; + version: 1; + metadata: { timestamp: string; tools: { components: { type: 'application'; name: string }[] }; component: Component }; + components: Component[]; + dependencies: { ref: string; dependsOn: string[] }[]; +} + +export interface SbomInput { + /** The app's version, without a leading `v`. */ + version: string; + npm: readonly NpmPackage[]; + crates: readonly Crate[]; + serial: string; + timestamp: string; +} + +/** A Package URL, the name other tools use to find a component. Scoped npm names write `@` as `%40`. */ +export const npmPurl = (name: string, version: string): string => `pkg:npm/${name.replace(/^@/, '%40')}@${version}`; +export const cargoPurl = (name: string, version: string): string => `pkg:cargo/${name}@${version}`; + +/** + * A license field as CycloneDX wants it: an SPDX expression when it looks like one, and the plain name for anything + * else, such as `SEE LICENSE IN LICENSE.txt`. Older crates write `MIT/Apache-2.0`, which means `MIT OR Apache-2.0`. + */ +export function licenseOf(text: string | undefined): Component['licenses'] { + const cleaned = text?.trim().replace(/\s*\/\s*/g, ' OR '); + if (!cleaned) return undefined; + return [looksLikeSpdx(cleaned) ? { expression: cleaned } : { license: { name: cleaned } }]; +} + +/** True for ids joined by AND, OR, and WITH, and grouped in parentheses. Two ids side by side are plain words. */ +function looksLikeSpdx(text: string): boolean { + if (!/^[A-Za-z0-9.+() -]+$/.test(text)) return false; + const tokens = text.split(/[\s()]+/).filter(Boolean); + const isOperator = (token: string): boolean => /^(AND|OR|WITH)$/.test(token); + return tokens.every((token, at) => at === 0 || isOperator(token) || isOperator(tokens[at - 1])); +} + +function npmComponent(pkg: NpmPackage): Component { + const purl = npmPurl(pkg.name, pkg.version); + const references = pkg.resolved?.startsWith('https://') ? [{ type: 'distribution', url: pkg.resolved }] : undefined; + return { + type: 'library', + 'bom-ref': purl, + name: pkg.name, + version: pkg.version, + purl, + scope: 'required', + licenses: licenseOf(pkg.license), + hashes: pkg.sha512 ? [{ alg: 'SHA-512', content: pkg.sha512 }] : undefined, + externalReferences: references, + }; +} + +function crateComponent(crate: Crate): Component { + const purl = cargoPurl(crate.name, crate.version); + const url = crate.repository ?? (crate.local ? `https://github.com/${REPO}` : undefined); + return { + type: 'library', + 'bom-ref': purl, + name: crate.name, + version: crate.version, + purl, + scope: 'required', + licenses: licenseOf(crate.license), + hashes: crate.sha256 ? [{ alg: 'SHA-256', content: crate.sha256 }] : undefined, + externalReferences: url ? [{ type: 'vcs', url }] : undefined, + }; +} + +/** Builds the bill of materials. The same input always gives the same output. */ +export function buildSbom(input: SbomInput): Bom { + const byPath = new Map(input.npm.map((pkg) => [pkg.path, pkg])); + const dependencies = new Map>(); + const components = new Map(); + for (const pkg of input.npm) { + const component = npmComponent(pkg); + components.set(component.purl, component); + const needs = pkg.needs.flatMap((path) => byPath.get(path) ?? []).map((dep) => npmPurl(dep.name, dep.version)); + dependencies.set(component.purl, new Set([...(dependencies.get(component.purl) ?? []), ...needs])); + } + for (const crate of input.crates) { + const component = crateComponent(crate); + components.set(component.purl, component); + const needs = crate.needs.map((id) => id.replace(/^(.*)@(.*)$/, (_, name, version) => cargoPurl(name, version))); + dependencies.set(component.purl, new Set(needs)); + } + const root = `pkg:github/${REPO.toLowerCase()}@v${input.version}`; + const direct = [ + ...input.npm.filter((pkg) => pkg.direct).map((pkg) => npmPurl(pkg.name, pkg.version)), + ...input.crates.filter((crate) => crate.name === APP_CRATE).map((crate) => cargoPurl(crate.name, crate.version)), + ]; + const sorted = [...components.values()].sort((a, b) => a.purl.localeCompare(b.purl)); + return { + bomFormat: 'CycloneDX', + specVersion: '1.5', + serialNumber: input.serial, + version: 1, + metadata: { + timestamp: input.timestamp, + tools: { components: [{ type: 'application', name: 'OpenNote release scripts (app/scripts/release/sbom.ts)' }] }, + component: { + type: 'application', + 'bom-ref': root, + name: 'OpenNote', + version: input.version, + purl: root, + licenses: licenseOf('Apache-2.0'), + externalReferences: [{ type: 'vcs', url: `https://github.com/${REPO}` }], + }, + }, + components: sorted, + dependencies: [ + { ref: root, dependsOn: direct.sort() }, + ...sorted.map((component) => ({ + ref: component.purl, + dependsOn: [...(dependencies.get(component.purl) ?? [])].sort(), + })), + ], + }; +} + +/** Reads the repository: the app's imports, package-lock.json, Cargo.lock, and cargo metadata for each target. */ +export function readInput(version: string, serial: string, timestamp: string): SbomInput { + const lock = JSON.parse(readFileSync(join(ROOT, 'package-lock.json'), 'utf8')) as Lockfile; + const npm = shippedPackages(lock, importedPackages(join(ROOT, 'app', 'src', 'main.tsx'))); + const results = RELEASE_FILES.map(({ target }) => { + const args = ['metadata', '--format-version', '1', '--locked', '--filter-platform', target]; + if (target === NATIVE_TARGET) args.push('--features', NATIVE_FEATURES); + return JSON.parse( + execFileSync('cargo', args, { cwd: ROOT, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }), + ) as CargoMetadata; + }); + const checksums = lockChecksums(readFileSync(join(ROOT, 'Cargo.lock'), 'utf8')); + const app = shippedCrates(results, APP_CRATE, checksums); + // The language model program is inside the exe of the native target only, so only that target's list is read. + const nativeAt = RELEASE_FILES.findIndex(({ target }) => target === NATIVE_TARGET); + const program = nativeAt === -1 ? [] : shippedCrates([results[nativeAt] as CargoMetadata], LLM_CRATE, checksums); + return { version, npm, crates: mergeCrates(app, program), serial, timestamp }; +} + +/** The crates of both lists, once each, with the crates each one needs joined. */ +export function mergeCrates(first: readonly Crate[], second: readonly Crate[]): Crate[] { + const merged = new Map(); + for (const crate of [...first, ...second]) { + const key = `${crate.name}@${crate.version}`; + const have = merged.get(key); + merged.set(key, have ? { ...have, needs: [...new Set([...have.needs, ...crate.needs])] } : crate); + } + return [...merged.values()]; +} + +function option(args: string[], name: string): string | undefined { + const at = args.indexOf(name); + return at === -1 ? undefined : args[at + 1]; +} + +function main(): void { + const args = process.argv.slice(2); + const [version, out] = args; + if (!version || !out) throw new Error('Usage: node app/scripts/release/sbom.ts '); + const serial = option(args, '--serial') ?? `urn:uuid:${randomUUID()}`; + const input = readInput(version.replace(/^v/, ''), serial, option(args, '--timestamp') ?? new Date().toISOString()); + writeFileSync(out, `${JSON.stringify(buildSbom(input), null, 2)}\n`); + console.log(`Wrote ${out}: ${input.npm.length} npm packages and ${input.crates.length} Rust crates.`); +} + +if (import.meta.main) main(); diff --git a/app/scripts/release/sign.test.ts b/app/scripts/release/sign.test.ts new file mode 100644 index 00000000..cd58a076 --- /dev/null +++ b/app/scripts/release/sign.test.ts @@ -0,0 +1,176 @@ +// @vitest-environment node +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { RELEASE_FILES, writeManifest } from '../write-manifest.ts'; +import { parsePublicKey } from './minisign.ts'; +import { runTauri, signRelease, type Tauri } from './sign.ts'; +import { fakeExe, newTestKey } from './test-support.ts'; +import { verifyRelease } from './verify.ts'; + +const dirs: string[] = []; +afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); + +function folder(withExes = true): string { + const dir = mkdtempSync(join(tmpdir(), 'opennote-sign-')); + dirs.push(dir); + if (withExes) RELEASE_FILES.forEach(({ file }) => writeFileSync(join(dir, file), fakeExe(file))); + return dir; +} + +interface Call { + args: string[]; + env: Record; +} + +/** A stand-in for the Tauri tool that records each call, and writes the files the real one would. */ +function recorder(calls: Call[]): Tauri { + return (args, env) => { + calls.push({ args, env }); + if (args[1] === 'sign') writeFileSync(`${args.at(-1)}.sig`, 'signature'); + if (args[1] === 'generate') { + const path = args[args.indexOf('--write-keys') + 1]; + writeFileSync(path, 'private key'); + writeFileSync(`${path}.pub`, 'public key'); + } + }; +} + +describe('signing with the release key', () => { + it('signs every exe for the tag, with the key and password from the environment', () => { + const calls: Call[] = []; + const env = { PATH: '/bin', TAURI_SIGNING_PRIVATE_KEY: 'secret key', TAURI_SIGNING_PRIVATE_KEY_PASSWORD: 'pw' }; + const dir = folder(); + signRelease({ dir, tag: 'v1.2.3', env, tauri: recorder(calls) }); + expect(calls.map((call) => call.args.slice(0, 4))).toEqual( + RELEASE_FILES.map(() => ['signer', 'sign', '--app-version', '1.2.3']), + ); + expect(calls.map((call) => call.args.at(-1))).toEqual(RELEASE_FILES.map(({ file }) => join(dir, file))); + expect(calls[0].env).toEqual(env); + }); + + it('works for a key without a password', () => { + const calls: Call[] = []; + signRelease({ dir: folder(), tag: 'v1.2.3', env: { TAURI_SIGNING_PRIVATE_KEY: 'k' }, tauri: recorder(calls) }); + expect(calls[0].env).toEqual({ TAURI_SIGNING_PRIVATE_KEY: 'k' }); + }); + + it.each([undefined, ''])('stops when the key is %j, so nothing is published unsigned', (key) => { + const calls: Call[] = []; + const env = { TAURI_SIGNING_PRIVATE_KEY: key }; + const run = () => signRelease({ dir: folder(), tag: 'v1.2.3', env, tauri: recorder(calls) }); + expect(run).toThrow('TAURI_SIGNING_PRIVATE_KEY secret is not set'); + expect(calls).toEqual([]); + }); + + it('stops when an exe is missing or the tool writes no signature', () => { + const env = { TAURI_SIGNING_PRIVATE_KEY: 'k' }; + const dir = folder(); + rmSync(join(dir, 'OpenNote_Windows32.exe')); + expect(() => signRelease({ dir, tag: 'v1.2.3', env, tauri: recorder([]) })).toThrow('no OpenNote_Windows32.exe'); + expect(() => signRelease({ dir: folder(), tag: 'v1.2.3', env, tauri: () => undefined })).toThrow( + 'did not write OpenNote_Windows64.exe.sig', + ); + }); +}); + +describe('a dry run', () => { + it('makes a throwaway key, signs with it, keeps only the public key, and never uses the release key', () => { + const calls: Call[] = []; + const env = { PATH: '/bin', TAURI_SIGNING_PRIVATE_KEY: 'real secret', TAURI_SIGNING_PRIVATE_KEY_PASSWORD: 'real' }; + const pubkeyOut = join(folder(false), 'update.pub'); + const dir = folder(); + signRelease({ dir, tag: 'v1.2.3', env, dryRun: true, pubkeyOut, tauri: recorder(calls) }); + expect(calls[0].args.slice(0, 3)).toEqual(['signer', 'generate', '--ci']); + expect(readFileSync(pubkeyOut, 'utf8')).toBe('public key'); + for (const call of calls) { + expect(call.env.PATH).toBe('/bin'); + expect(call.env.TAURI_SIGNING_PRIVATE_KEY).not.toBe('real secret'); + expect(call.env.TAURI_SIGNING_PRIVATE_KEY_PASSWORD).not.toBe('real'); + } + const signing = calls.filter((call) => call.args[1] === 'sign'); + expect(signing).toHaveLength(RELEASE_FILES.length); + expect(signing[0].env.TAURI_SIGNING_PRIVATE_KEY).toBe('private key'); + const keyPath = calls[0].args[calls[0].args.indexOf('--write-keys') + 1]; + expect(existsSync(keyPath)).toBe(false); + }); + + it('makes the folder for the public key when it does not exist', () => { + const pubkeyOut = join(folder(false), 'dry-run', 'nested', 'update.pub'); + signRelease({ dir: folder(), tag: 'v1.2.3', env: {}, dryRun: true, pubkeyOut, tauri: recorder([]) }); + expect(readFileSync(pubkeyOut, 'utf8')).toBe('public key'); + }); + + it('needs somewhere to put the public key', () => { + const run = () => signRelease({ dir: folder(), tag: 'v1.2.3', env: {}, dryRun: true, tauri: recorder([]) }); + expect(run).toThrow('--pubkey-out'); + }); + + it('deletes the throwaway private key even when signing fails', () => { + const calls: Call[] = []; + const failing: Tauri = (args, env) => { + recorder(calls)(args, env); + if (args[1] === 'sign') throw new Error('the tool failed'); + }; + const pubkeyOut = join(folder(false), 'update.pub'); + expect(() => + signRelease({ dir: folder(), tag: 'v1.2.3', env: {}, dryRun: true, pubkeyOut, tauri: failing }), + ).toThrow('the tool failed'); + expect(existsSync(calls[0].args[calls[0].args.indexOf('--write-keys') + 1])).toBe(false); + }); +}); + +describe('the real Tauri command-line tool', () => { + const hasTool = existsSync( + join(import.meta.dirname, '..', '..', '..', 'node_modules', '@tauri-apps', 'cli', 'tauri.js'), + ); + + it.skipIf(!hasTool)( + 'signs in a dry run, and verify.ts accepts what it made', + () => { + const dir = folder(); + const pubkeyOut = join(folder(false), 'update.pub'); + const env = { + ...process.env, + TAURI_SIGNING_PRIVATE_KEY: undefined, + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: undefined, + }; + signRelease({ dir, tag: 'v1.2.3', env, dryRun: true, pubkeyOut, tauri: runTauri }); + writeManifest(dir, 'v1.2.3', 'Notes.', '2026-10-14T18:02:11.000Z'); + const keys = [parsePublicKey(readFileSync(pubkeyOut, 'utf8'))]; + expect(verifyRelease({ dir, tag: 'v1.2.3', keys })).toEqual([]); + expect(verifyRelease({ dir, tag: 'v1.2.4', keys }).length).toBeGreaterThan(0); + expect(verifyRelease({ dir, tag: 'v1.2.3', keys: [newTestKey().publicKey] }).length).toBeGreaterThan(0); + }, + 120_000, + ); + + it.skipIf(!hasTool)( + 'signs with a given key, when its password is right', + () => { + const dir = folder(); + const keys = join(folder(false), 'release.key'); + const env = { + ...process.env, + TAURI_SIGNING_PRIVATE_KEY: undefined, + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: undefined, + }; + const plain = { ...process.env } as Record; + runTauri(['signer', 'generate', '--ci', '--password', 'pw-for-this-test', '--write-keys', keys], plain); + const secret = { + ...env, + TAURI_SIGNING_PRIVATE_KEY: readFileSync(keys, 'utf8'), + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: 'pw-for-this-test', + }; + signRelease({ dir, tag: 'v2.0.0-beta.1', env: secret }); + writeManifest(dir, 'v2.0.0-beta.1', 'Notes.', '2026-10-14T18:02:11.000Z'); + const trusted = [parsePublicKey(readFileSync(`${keys}.pub`, 'utf8'))]; + expect(verifyRelease({ dir, tag: 'v2.0.0-beta.1', keys: trusted })).toEqual([]); + expect(() => + signRelease({ dir, tag: 'v2.0.0', env: { ...secret, TAURI_SIGNING_PRIVATE_KEY_PASSWORD: 'wrong' } }), + ).toThrow(); + }, + 120_000, + ); +}); diff --git a/app/scripts/release/sign.ts b/app/scripts/release/sign.ts new file mode 100644 index 00000000..d071858e --- /dev/null +++ b/app/scripts/release/sign.ts @@ -0,0 +1,112 @@ +// Signs the release exes for the updater with the Tauri command-line tool, one `.sig` file next to each exe. +// A real run reads the private key from TAURI_SIGNING_PRIVATE_KEY, and stops if it isn't set, so a release is never +// published unsigned. A dry run ignores that key, makes a throwaway one, and writes its public half for verify.ts. +// Usage: node app/scripts/release/sign.ts [--dry-run --pubkey-out ] + +import { execFileSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { RELEASE_FILES, versionOf, type ReleaseFile } from '../write-manifest.ts'; + +const TAURI_CLI = join(import.meta.dirname, '..', '..', '..', 'node_modules', '@tauri-apps', 'cli', 'tauri.js'); +const KEY_VARIABLES = [ + 'TAURI_SIGNING_PRIVATE_KEY', + 'TAURI_SIGNING_PRIVATE_KEY_PATH', + 'TAURI_SIGNING_PRIVATE_KEY_PASSWORD', +]; + +/** Runs `tauri ` with exactly this environment. Tests replace it so they need no private key. */ +export type Tauri = (args: string[], env: Record) => void; + +export interface SignOptions { + dir: string; + tag: string; + env: Record; + dryRun?: boolean; + /** Where a dry run writes the throwaway public key. */ + pubkeyOut?: string; + files?: readonly ReleaseFile[]; + tauri?: Tauri; +} + +/** Calls the Tauri command-line tool that `npm ci` installed, without a shell, so no argument can be misread. */ +export const runTauri: Tauri = (args, env) => { + execFileSync(process.execPath, [TAURI_CLI, ...args], { env, stdio: ['ignore', 'inherit', 'inherit'] }); +}; + +/** The environment for the Tauri tool: this one's, minus every signing variable, so no key leaks into a dry run. */ +function plainEnvironment(env: SignOptions['env']): Record { + const entries = Object.entries(env).filter(([name, value]) => value !== undefined && !KEY_VARIABLES.includes(name)); + return Object.fromEntries(entries) as Record; +} + +function signAll(options: SignOptions, signingEnv: Record): void { + const tauri = options.tauri ?? runTauri; + const version = versionOf(options.tag); + for (const { file } of options.files ?? RELEASE_FILES) { + const path = join(options.dir, file); + if (!existsSync(path) || statSync(path).size === 0) + throw new Error(`There is no ${file} to sign in ${options.dir}.`); + // --app-version puts the version in the signed trusted comment, so nobody can change it later. + tauri(['signer', 'sign', '--app-version', version, path], signingEnv); + if (!existsSync(`${path}.sig`)) throw new Error(`The Tauri tool did not write ${file}.sig.`); + } +} + +/** Signs with the key in the environment. Only the signing step of the release workflow should have it. */ +function signWithSecret(options: SignOptions): void { + const key = options.env.TAURI_SIGNING_PRIVATE_KEY; + if (!key) throw new Error('The TAURI_SIGNING_PRIVATE_KEY secret is not set. See docs/RELEASING.md.'); + const signingEnv = { ...plainEnvironment(options.env), TAURI_SIGNING_PRIVATE_KEY: key }; + const password = options.env.TAURI_SIGNING_PRIVATE_KEY_PASSWORD; + signAll( + options, + password === undefined ? signingEnv : { ...signingEnv, TAURI_SIGNING_PRIVATE_KEY_PASSWORD: password }, + ); +} + +/** Makes a key that exists for this run only, signs with it, and keeps just its public half. */ +function signWithThrowawayKey(options: SignOptions): void { + if (!options.pubkeyOut) throw new Error('A dry run needs --pubkey-out, so the signatures can be checked.'); + const tauri = options.tauri ?? runTauri; + const folder = mkdtempSync(join(tmpdir(), 'opennote-dry-run-')); + try { + const password = randomBytes(24).toString('hex'); + const keyPath = join(folder, 'dry-run.key'); + const env = plainEnvironment(options.env); + tauri(['signer', 'generate', '--ci', '--password', password, '--write-keys', keyPath], env); + const signingEnv = { + ...env, + TAURI_SIGNING_PRIVATE_KEY: readFileSync(keyPath, 'utf8'), + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: password, + }; + signAll(options, signingEnv); + mkdirSync(dirname(options.pubkeyOut), { recursive: true }); + copyFileSync(`${keyPath}.pub`, options.pubkeyOut); + } finally { + rmSync(folder, { recursive: true, force: true }); + } +} + +/** Signs every release file. See the top of this file for the real and dry-run modes. */ +export function signRelease(options: SignOptions): void { + if (options.dryRun) signWithThrowawayKey(options); + else signWithSecret(options); +} + +function main(): void { + const args = process.argv.slice(2); + const [dir, tag] = args.filter((arg, at) => !arg.startsWith('--') && args[at - 1] !== '--pubkey-out'); + if (!dir || !tag) + throw new Error('Usage: node app/scripts/release/sign.ts [--dry-run --pubkey-out ]'); + const at = args.indexOf('--pubkey-out'); + const pubkeyOut = at === -1 ? undefined : args[at + 1]; + signRelease({ dir, tag, env: process.env, dryRun: args.includes('--dry-run'), pubkeyOut }); + console.log( + `Signed ${RELEASE_FILES.length} exes for ${tag}${args.includes('--dry-run') ? ' with a throwaway key' : ''}.`, + ); +} + +if (import.meta.main) main(); diff --git a/app/scripts/release/signoff.test.ts b/app/scripts/release/signoff.test.ts new file mode 100644 index 00000000..17cc2c6b --- /dev/null +++ b/app/scripts/release/signoff.test.ts @@ -0,0 +1,62 @@ +// @vitest-environment node +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { signoffTemplate } from './checklist-signoff.ts'; +import { parseDevice, recordSignoff, withEntry } from './signoff.ts'; + +const TYPES = ['Surface Pro', 'Wacom tablet']; + +describe('signoff.ts', () => { + it('writes the sign-off file from the template, with the new entry', () => { + const root = mkdtempSync(join(tmpdir(), 'signoff-')); + const path = recordSignoff( + root, + { version: '0.2.0-beta.1', item: 'docs', by: 'Ada', date: '2026-10-01' }, + TYPES, + '2026-10-07', + ); + expect(path).toBe('docs/releases/0.2.0-beta.1.signoff.json'); + const written = JSON.parse(readFileSync(join(root, path), 'utf8')); + expect(written.items.docs).toEqual({ by: 'Ada', date: '2026-10-01' }); + expect(written.items.pen).toEqual({ by: '', date: '', notes: '', devices: [] }); + + recordSignoff( + root, + { version: '0.2.0-beta.1', item: 'format', by: 'Ada', date: '2026-10-02' }, + TYPES, + '2026-10-07', + ); + const again = JSON.parse(readFileSync(join(root, path), 'utf8')); + expect(again.items.docs.by).toBe('Ada'); + expect(again.items.format.date).toBe('2026-10-02'); + }); + + it('refuses an entry the checklist would not count', () => { + const blank = signoffTemplate('1.0.0'); + const input = { version: '1.0.0', item: 'docs', by: 'Ada', date: '2026-10-09' }; + expect(() => withEntry(blank, input, '2026-10-07', TYPES)).toThrow(/future/); + expect(() => withEntry(blank, { ...input, by: ' ' }, '2026-10-07', TYPES)).toThrow(/No one/); + expect(() => withEntry(blank, { ...input, item: 'vibes' }, '2026-10-07', TYPES)).toThrow(/not a checklist item/); + expect(() => withEntry(blank, { ...input, version: '2.0.0' }, '2026-10-07', TYPES)).toThrow(/for 1.0.0/); + const pen = { ...input, item: 'pen', date: '2026-10-01', devices: [parseDevice('Surface Pro=Pro 9')] }; + expect(() => withEntry(blank, pen, '2026-10-07', TYPES)).toThrow(/two different devices/); + }); + + it('adds a passing update test to the notes, and refuses a failing one', () => { + const blank = signoffTemplate('1.0.0'); + const report = { from: '0.9.0-beta.3', to: '1.0.0', ok: true, detail: '0.9.0-beta.3 updated itself to 1.0.0.' }; + const input = { version: '1.0.0', item: 'updates', by: 'Ada', date: '2026-10-01', updateReport: report }; + expect(withEntry(blank, input, '2026-10-07', TYPES).items.updates?.notes).toBe( + 'Update test: 0.9.0-beta.3 updated itself to 1.0.0.', + ); + const failed = { ...input, updateReport: { ...report, ok: false, detail: 'The update timed out.' } }; + expect(() => withEntry(blank, failed, '2026-10-07', TYPES)).toThrow(/timed out/); + }); + + it('reads devices as type=name', () => { + expect(parseDevice('Wacom tablet=Intuos Pro')).toEqual({ type: 'Wacom tablet', name: 'Intuos Pro' }); + expect(() => parseDevice('Intuos')).toThrow(/=/); + }); +}); diff --git a/app/scripts/release/signoff.ts b/app/scripts/release/signoff.ts new file mode 100644 index 00000000..a315ab1f --- /dev/null +++ b/app/scripts/release/signoff.ts @@ -0,0 +1,105 @@ +// Records one person's sign-off of a release checklist item in docs/releases/.signoff.json, creating the +// file from the blank template when it is missing. It checks the entry the way checklist.ts will, so a sign-off +// that would not count is refused here, with the reason. The update-from-earlier-beta test can add its report to +// the notes of the `updates` item, but a person still signs it. +// +// Usage: node app/scripts/release/signoff.ts --by [--date YYYY-MM-DD] [--notes ] +// [--device =]... [--update-report ] +// Items: budgets, accessibility, pen, upgrade, format, updates, docs. + +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { deviceTypes, readPlan } from './checklist-doc.ts'; +import { + entryProblem, + MANUAL_ITEMS, + readSignoff, + signoffPath, + signoffTemplate, + type Device, + type Entry, + type ManualItem, + type Signoff, +} from './checklist-signoff.ts'; + +const ROOT = join(import.meta.dirname, '..', '..', '..'); + +export interface SignoffInput { + version: string; + item: string; + by: string; + date: string; + notes?: string; + devices?: Device[]; + /** The JSON report of update-test.ts, added to the notes. */ + updateReport?: { from: string; to: string; ok: boolean; detail: string }; +} + +/** Today in UTC as YYYY-MM-DD. */ +export const today = (): string => new Date().toISOString().slice(0, 10); + +/** Reads `type=name` device arguments. */ +export function parseDevice(text: string): Device { + const at = text.indexOf('='); + if (at <= 0 || at === text.length - 1) throw new Error(`"${text}" is not a device in the form =.`); + return { type: text.slice(0, at).trim(), name: text.slice(at + 1).trim() }; +} + +/** The sign-off with the new entry in it, or an error that says why the entry would not count. */ +export function withEntry(current: Signoff, input: SignoffInput, now: string, types: readonly string[]): Signoff { + if (!MANUAL_ITEMS.includes(input.item as ManualItem)) { + throw new Error(`"${input.item}" is not a checklist item a person signs. Use one of: ${MANUAL_ITEMS.join(', ')}.`); + } + if (current.version !== input.version) { + throw new Error(`The sign-off file is for ${current.version}, not ${input.version}.`); + } + const item = input.item as ManualItem; + const report = input.updateReport; + if (report && !report.ok) throw new Error(`The update test did not pass: ${report.detail}`); + const notes = [input.notes?.trim(), report ? `Update test: ${report.detail}` : undefined].filter(Boolean).join(' '); + const entry: Entry = { by: input.by.trim(), date: input.date, ...(notes ? { notes } : {}) }; + if (input.devices?.length) entry.devices = input.devices; + const problem = entryProblem(item, entry, now, types); + if (problem) throw new Error(problem); + return { ...current, items: { ...current.items, [item]: entry } }; +} + +/** Writes the entry to the version's sign-off file and returns its path. */ +export function recordSignoff(root: string, input: SignoffInput, types: readonly string[], now = today()): string { + const path = signoffPath(input.version); + const file = join(root, path); + const current = existsSync(file) ? readSignoff(root, input.version) : signoffTemplate(input.version); + const next = withEntry(current, input, now, types); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, `${JSON.stringify(next, null, 2)}\n`); + return path; +} + +function option(args: string[], name: string): string | undefined { + const at = args.indexOf(name); + return at === -1 ? undefined : args[at + 1]; +} + +function main(): void { + const args = process.argv.slice(2); + const [version, item] = args; + const by = option(args, '--by'); + if (!version || !item || !by) { + throw new Error('Usage: node app/scripts/release/signoff.ts --by [--date YYYY-MM-DD]'); + } + const devices = args.flatMap((arg, at) => (args[at - 1] === '--device' ? [parseDevice(arg)] : [])); + const reportFile = option(args, '--update-report'); + const input: SignoffInput = { + version, + item, + by, + date: option(args, '--date') ?? today(), + notes: option(args, '--notes'), + devices, + updateReport: reportFile ? JSON.parse(readFileSync(reportFile, 'utf8')) : undefined, + }; + const path = recordSignoff(ROOT, input, deviceTypes(readPlan(ROOT))); + console.log(`Recorded ${item} for ${version} in ${path}. Commit it before you tag the release.`); +} + +if (import.meta.main) main(); diff --git a/app/scripts/release/test-support.ts b/app/scripts/release/test-support.ts new file mode 100644 index 00000000..f0e18de6 --- /dev/null +++ b/app/scripts/release/test-support.ts @@ -0,0 +1,81 @@ +// Helpers for the release script tests: a throwaway minisign key that signs the way `tauri signer sign` does, and a +// release folder that looks like the one the workflow makes. Nothing here runs outside the tests. + +import { createHash, generateKeyPairSync, randomBytes, sign as edSign } from 'node:crypto'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { RELEASE_FILES, versionOf, writeManifest } from '../write-manifest.ts'; +import type { PublicKey } from './minisign.ts'; + +export interface TestKey { + publicKey: PublicKey; + /** The `.pub` file as `tauri signer generate` writes it. */ + pubFile: string; + /** The `.sig` file as `tauri signer sign` writes it, for a file and a version. */ + signFile(data: Buffer, file: string, version: string): string; + /** A signature with any trusted comment, for the tests that need a wrong one. */ + signWithComment(data: Buffer, comment: string): string; +} + +/** A new key pair that signs like the Tauri command-line tool, with a random key id. */ +export function newTestKey(): TestKey { + const { publicKey, privateKey } = generateKeyPairSync('ed25519'); + const raw = publicKey.export({ format: 'der', type: 'spki' }).subarray(12); + const id = randomBytes(8); + const keyLine = Buffer.concat([Buffer.from('Ed', 'latin1'), id, raw]).toString('base64'); + const pubText = `untrusted comment: minisign public key: ${id.toString('hex')}\n${keyLine}\n`; + const signWithComment = (data: Buffer, comment: string): string => { + const signature = edSign(null, createHash('blake2b512').update(data).digest(), privateKey); + const global = edSign(null, Buffer.concat([signature, Buffer.from(comment)]), privateKey); + const body = Buffer.concat([Buffer.from('ED', 'latin1'), id, signature]).toString('base64'); + const lines = ['untrusted comment: signature from tauri secret key', body]; + lines.push(`trusted comment: ${comment}`, global.toString('base64')); + return Buffer.from(`${lines.join('\n')}\n`).toString('base64'); + }; + return { + publicKey: { id, key: raw }, + pubFile: Buffer.from(pubText).toString('base64'), + signFile: (data, file, version) => signWithComment(data, `timestamp:1790997565\tfile:${file}\tversion:${version}`), + signWithComment, + }; +} + +/** A minimal PE file: DOS header, PE header, and an optional header with room for a certificate table. */ +export function fakePe(options: { plus?: boolean; signed?: boolean; directories?: number } = {}): Buffer { + const { plus = true, signed = false, directories = 16 } = options; + const pe = 0x80; + const optional = pe + 24; + const buffer = Buffer.alloc(0x400); + buffer.write('MZ', 0, 'latin1'); + buffer.writeUInt32LE(pe, 0x3c); + buffer.write('PE\0\0', pe, 'latin1'); + buffer.writeUInt16LE(plus ? 0x20b : 0x10b, optional); + buffer.writeUInt32LE(directories, optional + (plus ? 108 : 92)); + if (signed) { + const entry = optional + (plus ? 112 : 96) + 4 * 8; + buffer.writeUInt32LE(0x300, entry); + buffer.writeUInt32LE(0x80, entry + 4); + } + return buffer; +} + +/** The bytes of a fake exe. They differ per file, so a swapped file shows up. */ +export function fakeExe(file: string): Buffer { + return Buffer.concat([Buffer.from('MZ'), Buffer.from(`fake ${file}`.repeat(20))]); +} + +/** Writes the three signed exes, their `.sig` files, and the manifest for `tag` into `dir`. */ +export function writeSampleRelease( + dir: string, + tag: string, + key: TestKey, + options: { notes?: string; exe?: (file: string) => Buffer } = {}, +): string { + mkdirSync(dir, { recursive: true }); + for (const { file } of RELEASE_FILES) { + const exe = (options.exe ?? fakeExe)(file); + writeFileSync(join(dir, file), exe); + writeFileSync(join(dir, `${file}.sig`), key.signFile(exe, file, versionOf(tag))); + } + return writeManifest(dir, tag, options.notes ?? `OpenNote ${tag}`, '2026-10-14T18:02:11.000Z').name; +} diff --git a/app/scripts/release/update-test.test.ts b/app/scripts/release/update-test.test.ts new file mode 100644 index 00000000..36ff3810 --- /dev/null +++ b/app/scripts/release/update-test.test.ts @@ -0,0 +1,66 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import type { Manifest } from '../write-manifest.ts'; +import { + appliedIn, + decodePubFile, + earlierBeta, + failureIn, + localManifest, + TRAILER, + withTrailer, +} from './update-test.ts'; + +describe('update-test.ts', () => { + it('finds the last beta before the release', () => { + const tags = ['v0.1.0-beta.3', 'v0.1.0-beta.10', 'v0.1.0-test.1', 'v0.2.0-beta.1', 'channel-manifests', 'v0.0.9']; + expect(earlierBeta(tags, 'v0.2.0-beta.1')).toBe('v0.1.0-beta.10'); + expect(earlierBeta(tags, 'v0.2.0')).toBe('v0.2.0-beta.1'); + expect(earlierBeta(tags, 'v0.1.0-beta.3')).toBeUndefined(); + expect(earlierBeta([], 'v1.0.0')).toBeUndefined(); + }); + + it('points each file of the manifest at the local server and changes nothing the updater checks', () => { + const manifest: Manifest = { + version: '0.2.0-beta.1', + notes: 'n', + pub_date: '2026-10-07T00:00:00Z', + channel: 'beta', + platforms: { + 'windows-x86_64': { + url: 'https://github.com/XrxcGH/OpenNote/releases/download/v0.2.0-beta.1/OpenNote_Windows64.exe', + signature: 'sig', + size: 3, + sha256: 'abc', + }, + }, + }; + const local = localManifest(manifest, 'http://127.0.0.1:5000'); + expect(local.platforms['windows-x86_64']).toEqual({ + url: 'http://127.0.0.1:5000/OpenNote_Windows64.exe', + signature: 'sig', + size: 3, + sha256: 'abc', + }); + expect(local.version).toBe(manifest.version); + }); + + it('marks the test app with its version', () => { + const marked = withTrailer(Buffer.from('MZ'), '0.1.0-beta.4').toString(); + expect(marked).toBe(`MZ${TRAILER}0.1.0-beta.4`); + }); + + it('reads the public key from a .pub file', () => { + const text = 'untrusted comment: minisign public key: 1\nRWQ\n'; + expect(decodePubFile(Buffer.from(text).toString('base64'))).toBe(text); + expect(() => decodePubFile(Buffer.from('nope').toString('base64'))).toThrow(/minisign/); + }); + + it("reads the test app's event log", () => { + const log = 'started 0.1.0-beta.4\napplied 0.1.0-beta.4 0.2.0-beta.1\n'; + expect(appliedIn(log, '0.1.0-beta.4', '0.2.0-beta.1')).toBe(true); + expect(appliedIn(log, '0.1.0-beta.3', '0.2.0-beta.1')).toBe(false); + expect(failureIn('update failed verifyFailed\nupdate failure detail: bad hash')).toBe('update failed verifyFailed'); + expect(failureIn(log)).toBeUndefined(); + }); +}); diff --git a/app/scripts/release/update-test.ts b/app/scripts/release/update-test.ts new file mode 100644 index 00000000..b0456b82 --- /dev/null +++ b/app/scripts/release/update-test.ts @@ -0,0 +1,198 @@ +// The update-from-earlier-beta test. +// A copy of the updater's test app (crates/updater/examples/fake_app.rs) is built from the last beta tag. +// It updates itself to the release in a folder, the way a beta tester's copy will. +// It reads the manifest, downloads its exe, checks the hash and the signature, swaps the exe, and starts the new one. +// The files are served from 127.0.0.1, which only test builds of the updater accept. +// Each URL in the manifest points there. +// The release workflow runs it after the package job. +// A dry run uses the throwaway key, and a tag uses the committed key. +// +// Usage: +// +// node app/scripts/release/update-test.ts earlier +// +// Prints the last beta tag before , or nothing when there is none. +// +// node app/scripts/release/update-test.ts run --fake-app --from +// [--pubkey ] [--report ] +// +// Without --pubkey it trusts the first key committed in app/src-tauri/keys. + +import { spawn, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createServer, type Server } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { RELEASE_FILES, versionOf, type Manifest } from '../write-manifest.ts'; +import { KEYS_DIR } from './verify.ts'; +import { compareVersions } from './versions.ts'; + +/** What the test app appends to a copy of itself to say which version it is. */ +export const TRAILER = '#fake-app-version:'; + +/** How long the test waits for the swap. A cold exe download and signature check take seconds. */ +export const WAIT_MS = 120_000; + +/** The last `v*-beta.*` tag below `tag`, or undefined when there is none. */ +export function earlierBeta(tags: readonly string[], tag: string): string | undefined { + const current = versionOf(tag); + const betas = tags + .map((each) => each.trim()) + .filter((each) => /^v\d+\.\d+\.\d+-beta\.\d+$/.test(each)) + .filter((each) => compareVersions(versionOf(each), current) < 0) + .sort((a, b) => compareVersions(versionOf(b), versionOf(a))); + return betas[0]; +} + +/** The manifest with each file's URL on the local server. Everything the updater checks stays the same. */ +export function localManifest(manifest: Manifest, base: string): Manifest { + const platforms = Object.fromEntries( + Object.entries(manifest.platforms).map(([key, entry]) => { + const file = entry.url.slice(entry.url.lastIndexOf('/') + 1); + return [key, { ...entry, url: `${base}/${encodeURIComponent(file)}` }]; + }), + ); + return { ...manifest, platforms }; +} + +/** A copy of the test app's exe that says it is `version`. */ +export function withTrailer(exe: Buffer, version: string): Buffer { + return Buffer.concat([exe, Buffer.from(`${TRAILER}${version}`)]); +} + +/** The minisign public key text in a `.pub` file as `tauri signer generate` writes it (base64 of the text). */ +export function decodePubFile(contents: string): string { + const text = Buffer.from(contents.trim(), 'base64').toString('utf8'); + if (!text.startsWith('untrusted comment:')) throw new Error('The .pub file does not hold a minisign public key.'); + return text; +} + +/** Whether the test app's event log says it applied `from` to `to`. */ +export function appliedIn(events: string, from: string, to: string): boolean { + return events.split(/\r?\n/).some((line) => line.trim() === `applied ${from} ${to}`); +} + +/** The reason the update failed, from the test app's event log, if it says. */ +export function failureIn(events: string): string | undefined { + return events + .split(/\r?\n/) + .find((line) => line.startsWith('update failure detail:') || line.startsWith('update failed')); +} + +/** Serves the manifest and the release's files, and nothing else, from 127.0.0.1. */ +function serve(dir: string): Promise<{ server: Server; base: string }> { + return new Promise((resolve) => { + const server = createServer((request, response) => { + const name = decodeURIComponent((request.url ?? '/').slice(1)); + if (request.method !== 'GET' || !readdirSync(dir).includes(name)) { + response.writeHead(404).end(); + return; + } + const body = readFileSync(join(dir, name)); + response.writeHead(200, { 'Content-Length': body.length }).end(body); + }); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + const port = typeof address === 'object' && address ? address.port : 0; + resolve({ server, base: `http://127.0.0.1:${port}` }); + }); + }); +} + +export interface RunOptions { + dir: string; + tag: string; + fakeApp: string; + from: string; + pubkeyText: string; +} + +export interface Report { + from: string; + to: string; + file: string; + ok: boolean; + detail: string; +} + +/** Runs the test and returns what happened. The new exe it starts is stopped at the end. */ +export async function runUpdateTest(options: RunOptions): Promise { + const to = versionOf(options.tag); + const release = RELEASE_FILES.find((each) => each.target === 'x86_64-pc-windows-msvc'); + if (!release) throw new Error('No x64 file in app/release-files.json.'); + const manifestName = to.includes('-') ? 'beta.json' : 'latest.json'; + const manifest = JSON.parse(readFileSync(join(options.dir, manifestName), 'utf8')) as Manifest; + const work = mkdtempSync(join(tmpdir(), 'opennote-update-test-')); + const served = join(work, 'served'); + const app = join(work, 'app'); + for (const folder of [served, app]) mkdirSync(folder, { recursive: true }); + copyFileSync(join(options.dir, release.file), join(served, release.file)); + const { server, base } = await serve(served); + writeFileSync(join(served, manifestName), JSON.stringify(localManifest(manifest, base))); + const exe = join(app, 'OpenNote-update-test.exe'); + writeFileSync(exe, withTrailer(readFileSync(options.fakeApp), options.from)); + const env = { + ...process.env, + FAKE_APP_DIR: work, + FAKE_APP_PUBKEY: options.pubkeyText, + // The new exe the test app starts keeps its files in the test folder, not the runner's profile. + OPENNOTE_PROFILE_DIR: join(work, 'profile'), + }; + spawn(exe, ['update', `${base}/${manifestName}`], { env, stdio: 'ignore' }).unref(); + const events = join(work, 'events.log'); + const deadline = Date.now() + WAIT_MS; + let log = ''; + while (Date.now() < deadline) { + log = existsSync(events) ? readFileSync(events, 'utf8') : ''; + if (appliedIn(log, options.from, to) || failureIn(log)) break; + await new Promise((done) => setTimeout(done, 500)); + } + server.close(); + spawnSync('taskkill', ['/F', '/T', '/IM', 'OpenNote-update-test.exe'], { stdio: 'ignore' }); + const want = createHash('sha256') + .update(readFileSync(join(options.dir, release.file))) + .digest('hex'); + const got = existsSync(exe) ? createHash('sha256').update(readFileSync(exe)).digest('hex') : ''; + const applied = appliedIn(log, options.from, to); + const ok = applied && got === want; + const detail = ok + ? `${options.from} updated itself to ${to}, and its exe is now ${release.file}.` + : (failureIn(log) ?? + (applied ? 'The swap reported success, but the exe is not the release file.' : 'The update timed out.')); + return { from: options.from, to, file: release.file, ok, detail }; +} + +function option(args: string[], name: string): string | undefined { + const at = args.indexOf(name); + return at === -1 ? undefined : args[at + 1]; +} + +async function main(): Promise { + const [command, ...args] = process.argv.slice(2); + if (command === 'earlier') { + const tags = spawnSync('git', ['tag', '--list', 'v*'], { encoding: 'utf8' }).stdout.split('\n'); + console.log(earlierBeta(tags, args[0] ?? '') ?? ''); + return; + } + if (command !== 'run' || args.length < 2) { + throw new Error( + 'Usage: node app/scripts/release/update-test.ts run --fake-app --from ', + ); + } + const [dir, tag] = args; + const fakeApp = option(args, '--fake-app'); + const from = option(args, '--from'); + if (!fakeApp || !from) throw new Error('--fake-app and --from are needed.'); + const pubkey = option(args, '--pubkey'); + const pubFile = pubkey ?? readdirSync(KEYS_DIR).find((name) => name.endsWith('.pub')); + if (!pubFile) throw new Error('No --pubkey, and no key committed in app/src-tauri/keys.'); + const pubkeyText = decodePubFile(readFileSync(pubkey ?? join(KEYS_DIR, pubFile), 'utf8')); + const report = await runUpdateTest({ dir, tag, fakeApp, from, pubkeyText }); + console.log(report.detail); + const reportFile = option(args, '--report'); + if (reportFile) writeFileSync(reportFile, `${JSON.stringify(report, null, 2)}\n`); + if (!report.ok) process.exitCode = 1; +} + +if (import.meta.main) await main(); diff --git a/app/scripts/release/verify.test.ts b/app/scripts/release/verify.test.ts new file mode 100644 index 00000000..0ddd5b4d --- /dev/null +++ b/app/scripts/release/verify.test.ts @@ -0,0 +1,190 @@ +// @vitest-environment node +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { RELEASE_FILES, type Manifest } from '../write-manifest.ts'; +import { fakeExe, fakePe, newTestKey, writeSampleRelease, type TestKey } from './test-support.ts'; +import { keysIn, manifestName, verifyRelease } from './verify.ts'; + +const dirs: string[] = []; +afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); + +function release(tag: string, options: { exe?: (file: string) => Buffer } = {}) { + const key = newTestKey(); + const dir = mkdtempSync(join(tmpdir(), 'opennote-verify-')); + dirs.push(dir); + const name = writeSampleRelease(dir, tag, key, options); + const check = (use: { keys?: TestKey[]; authenticode?: boolean; windows?: string } = {}) => + verifyRelease({ + dir, + tag, + keys: (use.keys ?? [key]).map((each) => each.publicKey), + requireAuthenticode: use.authenticode, + windowsStatus: () => use.windows, + }); + const edit = (change: (manifest: Manifest) => void) => { + const manifest = JSON.parse(readFileSync(join(dir, name), 'utf8')) as Manifest; + change(manifest); + writeFileSync(join(dir, name), JSON.stringify(manifest)); + }; + return { dir, name, key, check, edit }; +} + +describe('a good release', () => { + it.each(['v1.0.0', 'v1.0.0-beta.2'])('has no problems (%s)', (tag) => { + expect(release(tag).check()).toEqual([]); + }); + + it('accepts a signature from any one of several trusted keys', () => { + const { check, key } = release('v1.0.0'); + expect(check({ keys: [newTestKey(), key] })).toEqual([]); + }); +}); + +describe('an exe the updater would refuse', () => { + it('is caught when it changed after it was signed', () => { + const { dir, check } = release('v1.0.0'); + const changed = Buffer.concat([fakeExe('OpenNote_Windows32.exe'), Buffer.from('!')]); + writeFileSync(join(dir, 'OpenNote_Windows32.exe'), changed); + const problems = check().join('\n'); + expect(problems).toContain('OpenNote_Windows32.exe: The signature does not match the file.'); + expect(problems).toContain('says OpenNote_Windows32.exe is'); + expect(problems).toContain('SHA-256 for OpenNote_Windows32.exe'); + }); + + it('is caught when another key signed it', () => { + const { check } = release('v1.0.0'); + expect(check({ keys: [newTestKey()] }).join('\n')).toContain('not signed with a key that this release trusts'); + }); + + it('is caught when there is no key to check against', () => { + expect(release('v1.0.0').check({ keys: [] })[0]).toContain('no update key'); + }); + + it('is caught when its signature names another version or file', () => { + const { dir, key, check } = release('v1.0.0'); + const file = 'OpenNote_Windows64.exe'; + writeFileSync(join(dir, `${file}.sig`), key.signFile(fakeExe(file), file, '0.9.0')); + expect(check().join('\n')).toContain(`The signature of ${file} does not name version 1.0.0 exactly once.`); + writeFileSync(join(dir, `${file}.sig`), key.signFile(fakeExe(file), 'other.exe', '1.0.0')); + expect(check().join('\n')).toContain(`does not name the file ${file} exactly once`); + }); + + it('is caught when the signature repeats a field to hide the real one', () => { + const { dir, key, check } = release('v1.0.0'); + const file = 'OpenNote_WindowsARM64.exe'; + const comment = `timestamp:1\tversion:0.9.0\tfile:${file}\tversion:1.0.0`; + writeFileSync(join(dir, `${file}.sig`), key.signWithComment(fakeExe(file), comment)); + expect(check().join('\n')).toContain('does not name version 1.0.0 exactly once'); + }); + + it('is caught when a file or its signature is missing', () => { + const { dir, check } = release('v1.0.0'); + rmSync(join(dir, 'OpenNote_Windows64.exe.sig')); + rmSync(join(dir, 'OpenNote_Windows32.exe')); + const problems = check().join('\n'); + expect(problems).toContain('OpenNote_Windows64.exe.sig is missing'); + expect(problems).toContain('OpenNote_Windows32.exe is missing or empty'); + }); +}); + +describe('the Authenticode requirement', () => { + it('is left alone unless it is asked for', () => { + expect(release('v1.0.0').check({ authenticode: false })).toEqual([]); + }); + + it('catches an exe with no Authenticode signature', () => { + const problems = release('v1.0.0').check({ authenticode: true }); + expect(problems).toHaveLength(RELEASE_FILES.length); + expect(problems[0]).toContain('has no Authenticode signature'); + }); + + it('accepts exes that carry one, and that Windows calls valid', () => { + const signed = (file: string) => Buffer.concat([fakePe({ signed: true }), Buffer.from(file)]); + expect(release('v1.0.0', { exe: signed }).check({ authenticode: true, windows: 'Valid' })).toEqual([]); + }); + + it('accepts them when nothing can answer for Windows, as off Windows', () => { + const signed = (file: string) => Buffer.concat([fakePe({ signed: true }), Buffer.from(file)]); + expect(release('v1.0.0', { exe: signed }).check({ authenticode: true })).toEqual([]); + }); + + it('refuses them when Windows does not trust the certificate', () => { + const signed = (file: string) => Buffer.concat([fakePe({ signed: true }), Buffer.from(file)]); + const problems = release('v1.0.0', { exe: signed }).check({ authenticode: true, windows: 'NotTrusted' }); + expect(problems).toHaveLength(RELEASE_FILES.length); + expect(problems[0]).toBe('Windows says the Authenticode signature of OpenNote_Windows64.exe is NotTrusted.'); + }); +}); + +describe('a manifest the updater would refuse', () => { + it('is caught when it is missing, or the wrong one for the channel is present', () => { + const stable = release('v1.0.0'); + rmSync(join(stable.dir, 'latest.json')); + expect(stable.check()).toContain('latest.json is missing.'); + const beta = release('v1.0.0-beta.1'); + writeFileSync(join(beta.dir, 'latest.json'), '{}'); + expect(beta.check()[0]).toBe('latest.json must not be in a prerelease.'); + }); + + it('is caught when it has the wrong version, channel, date, notes, or platforms', () => { + const { edit, check } = release('v1.0.0'); + edit((manifest) => { + manifest.version = '0.9.0'; + manifest.channel = 'beta'; + manifest.pub_date = 'yesterday'; + manifest.notes = ' '; + delete manifest.platforms['windows-i686']; + }); + const problems = check().join('\n'); + expect(problems).toContain('has version 0.9.0, but the tag is v1.0.0'); + expect(problems).toContain('has channel beta, but 1.0.0 is stable'); + expect(problems).toContain('no valid pub_date'); + expect(problems).toContain('no release notes'); + expect(problems).toContain('lists windows-aarch64, windows-x86_64, but a release has'); + expect(problems).toContain('The manifest has no entry for windows-i686'); + }); + + it('is caught when it sends an exe to another address or holds another signature', () => { + const { edit, check } = release('v1.0.0'); + edit((manifest) => { + manifest.platforms['windows-x86_64'].url = 'https://evil.example/OpenNote_Windows64.exe'; + manifest.platforms['windows-aarch64'].signature = 'AAAA'; + }); + const problems = check().join('\n'); + expect(problems).toContain('sends OpenNote_Windows64.exe to https://evil.example/'); + expect(problems).toContain('signature for OpenNote_WindowsARM64.exe differs from OpenNote_WindowsARM64.exe.sig'); + }); + + it('is caught when the notes are too long for the updater', () => { + const { edit, check } = release('v1.0.0'); + edit((manifest) => { + manifest.notes = 'x'.repeat(5000); + }); + expect(check().join('\n')).toContain('longer than 4096 bytes'); + }); + + it('is caught when it is not JSON', () => { + const { dir, name, check } = release('v1.0.0'); + writeFileSync(join(dir, name), 'not json'); + expect(check()).toContain('latest.json is not valid JSON.'); + }); +}); + +describe('manifestName and keysIn', () => { + it('puts prereleases in beta.json', () => { + expect(manifestName('1.0.0')).toBe('latest.json'); + expect(manifestName('1.0.0-beta.1')).toBe('beta.json'); + }); + + it('reads every .pub file in a folder, and none from a missing folder', () => { + const dir = mkdtempSync(join(tmpdir(), 'opennote-keys-')); + dirs.push(dir); + const key = newTestKey(); + writeFileSync(join(dir, 'update.pub'), key.pubFile); + writeFileSync(join(dir, 'README.md'), 'not a key'); + expect(keysIn(dir)).toEqual([key.publicKey]); + expect(keysIn(join(dir, 'missing'))).toEqual([]); + }); +}); diff --git a/app/scripts/release/verify.ts b/app/scripts/release/verify.ts new file mode 100644 index 00000000..fb79318b --- /dev/null +++ b/app/scripts/release/verify.ts @@ -0,0 +1,179 @@ +// Checks a finished release folder the way a running copy of OpenNote will. It checks the manifest, then each exe's +// size, hash, and signature (docs/RELEASING.md, Verifying a release). The release workflow runs it before anything +// is published, so a release that the updater would refuse never goes out. +// Usage: node app/scripts/release/verify.ts [--pubkey ]... [--require-authenticode] +// Without --pubkey it trusts the keys committed in app/src-tauri/keys, which are the keys built into the app. + +import { createHash } from 'node:crypto'; +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { REPO, RELEASE_FILES, versionOf, type Manifest, type ReleaseFile } from '../write-manifest.ts'; +import { onlyField, parsePublicKey, parseSignature, verifyWithAny, type PublicKey } from './minisign.ts'; +import { hasAuthenticode, windowsSignatureStatus } from './pe.ts'; + +/** The limits the updater enforces (crates/updater/src/config.rs and manifest.rs). */ +export const MAX_MANIFEST_BYTES = 64 * 1024; +export const MAX_EXE_BYTES = 200 * 1024 * 1024; +export const MAX_NOTES_BYTES = 4096; + +export const KEYS_DIR = join(import.meta.dirname, '..', '..', 'src-tauri', 'keys'); + +export interface VerifyOptions { + dir: string; + tag: string; + /** The public keys the app trusts. The signature on every exe must verify with one of them. */ + keys: readonly PublicKey[]; + /** Also require an Authenticode signature on every exe. */ + requireAuthenticode?: boolean; + files?: readonly ReleaseFile[]; + /** Asks the operating system about a signature. Tests replace it, because PowerShell takes seconds to start. */ + windowsStatus?: (path: string) => string | undefined; +} + +/** The keys in the `.pub` files of a folder, in file-name order. A missing folder has none. */ +export function keysIn(dir: string = KEYS_DIR): PublicKey[] { + if (!existsSync(dir)) return []; + const names = readdirSync(dir).filter((name) => name.endsWith('.pub')); + return names.sort().map((name) => parsePublicKey(readFileSync(join(dir, name), 'utf8'))); +} + +/** Which manifest file a version goes in: beta.json for a prerelease, latest.json otherwise. */ +export function manifestName(version: string): string { + return version.includes('-') ? 'beta.json' : 'latest.json'; +} + +function readManifest(dir: string, name: string, problems: string[]): Manifest | undefined { + const path = join(dir, name); + if (!existsSync(path)) { + problems.push(`${name} is missing.`); + return undefined; + } + if (statSync(path).size > MAX_MANIFEST_BYTES) problems.push(`${name} is larger than ${MAX_MANIFEST_BYTES} bytes.`); + try { + return JSON.parse(readFileSync(path, 'utf8')) as Manifest; + } catch { + problems.push(`${name} is not valid JSON.`); + return undefined; + } +} + +function manifestProblems(manifest: Manifest, name: string, tag: string, files: readonly ReleaseFile[]): string[] { + const problems: string[] = []; + const version = versionOf(tag); + if (manifest.version !== version) problems.push(`${name} has version ${manifest.version}, but the tag is ${tag}.`); + const channel = version.includes('-') ? 'beta' : 'stable'; + if (manifest.channel !== channel) + problems.push(`${name} has channel ${manifest.channel}, but ${version} is ${channel}.`); + if (Number.isNaN(Date.parse(manifest.pub_date))) problems.push(`${name} has no valid pub_date.`); + const notes = manifest.notes ?? ''; + if (notes.trim() === '') problems.push(`${name} has no release notes.`); + if (Buffer.byteLength(notes) > MAX_NOTES_BYTES) + problems.push(`The notes in ${name} are longer than ${MAX_NOTES_BYTES} bytes.`); + const expected = files.map(({ platform }) => platform).sort(); + const found = Object.keys(manifest.platforms ?? {}).sort(); + if (expected.join() !== found.join()) + problems.push(`${name} lists ${found.join(', ')}, but a release has ${expected.join(', ')}.`); + return problems; +} + +/** Everything wrong with one exe and its manifest entry; empty when a copy of the app would accept it. */ +function fileProblems( + options: VerifyOptions, + manifest: Manifest | undefined, + { platform, file }: ReleaseFile, +): string[] { + const path = join(options.dir, file); + if (!existsSync(path) || statSync(path).size === 0) return [`${file} is missing or empty.`]; + const exe = readFileSync(path); + const problems: string[] = []; + if (exe.length > MAX_EXE_BYTES) + problems.push(`${file} is larger than ${MAX_EXE_BYTES} bytes, so the app would refuse it.`); + problems.push(...signatureProblems(options, file, exe)); + const entry = manifest?.platforms?.[platform]; + if (manifest && !entry) return [...problems, `The manifest has no entry for ${platform}.`]; + if (entry) problems.push(...entryProblems(options, file, exe, entry)); + if (options.requireAuthenticode) problems.push(...authenticodeProblems(options, path, file, exe)); + return problems; +} + +function entryProblems( + options: VerifyOptions, + file: string, + exe: Buffer, + entry: Manifest['platforms'][string], +): string[] { + const problems: string[] = []; + const url = `https://github.com/${REPO}/releases/download/${options.tag}/${file}`; + if (entry.url !== url) problems.push(`The manifest sends ${file} to ${entry.url}, not ${url}.`); + if (entry.size !== exe.length) + problems.push(`The manifest says ${file} is ${entry.size} bytes, but it is ${exe.length}.`); + if (entry.sha256 !== createHash('sha256').update(exe).digest('hex')) + problems.push(`The manifest's SHA-256 for ${file} does not match the file.`); + const sigPath = join(options.dir, `${file}.sig`); + const sig = existsSync(sigPath) ? readFileSync(sigPath, 'utf8').trim() : ''; + if (entry.signature.trim() !== sig) problems.push(`The manifest's signature for ${file} differs from ${file}.sig.`); + return problems; +} + +function signatureProblems(options: VerifyOptions, file: string, exe: Buffer): string[] { + const sigPath = join(options.dir, `${file}.sig`); + if (!existsSync(sigPath)) return [`${file}.sig is missing.`]; + if (options.keys.length === 0) return ['There is no update key to check the signatures against.']; + try { + const comment = verifyWithAny(exe, parseSignature(readFileSync(sigPath, 'utf8')), options.keys); + const problems: string[] = []; + const version = versionOf(options.tag); + if (onlyField(comment, 'version') !== version) + problems.push(`The signature of ${file} does not name version ${version} exactly once.`); + if (onlyField(comment, 'file') !== file) + problems.push(`The signature of ${file} does not name the file ${file} exactly once.`); + return problems; + } catch (error) { + return [`${file}: ${(error as Error).message}`]; + } +} + +function authenticodeProblems(options: VerifyOptions, path: string, file: string, exe: Buffer): string[] { + if (!hasAuthenticode(exe)) { + return [`${file} has no Authenticode signature. See docs/RELEASING.md, Add code signing later.`]; + } + const status = (options.windowsStatus ?? windowsSignatureStatus)(path); + return status === undefined || status === 'Valid' + ? [] + : [`Windows says the Authenticode signature of ${file} is ${status}.`]; +} + +/** Checks a release folder against a tag. It returns one sentence for each problem, and nothing when all is well. */ +export function verifyRelease(options: VerifyOptions): string[] { + const files = options.files ?? RELEASE_FILES; + const name = manifestName(versionOf(options.tag)); + const other = name === 'beta.json' ? 'latest.json' : 'beta.json'; + const problems: string[] = []; + if (existsSync(join(options.dir, other))) + problems.push(`${other} must not be in a ${name === 'beta.json' ? 'prerelease' : 'stable release'}.`); + const manifest = readManifest(options.dir, name, problems); + if (manifest) problems.push(...manifestProblems(manifest, name, options.tag, files)); + for (const file of files) problems.push(...fileProblems(options, manifest, file)); + return problems; +} + +function main(): void { + const args = process.argv.slice(2); + const [dir, tag] = args.filter((arg, at) => !arg.startsWith('--') && args[at - 1] !== '--pubkey'); + if (!dir || !tag) + throw new Error('Usage: node app/scripts/release/verify.ts [--pubkey ]...'); + const pubkeys = args.flatMap((arg, at) => (args[at - 1] === '--pubkey' ? [arg] : [])); + const keys = pubkeys.length > 0 ? pubkeys.map((path) => parsePublicKey(readFileSync(path, 'utf8'))) : keysIn(); + const requireAuthenticode = args.includes('--require-authenticode'); + const problems = verifyRelease({ dir, tag, keys, requireAuthenticode }); + problems.forEach((problem) => console.log(`::error::${problem}`)); + if (problems.length > 0) { + process.exitCode = 1; + return; + } + console.log( + `The release in ${dir} is what the app's updater accepts for ${tag}: ${RELEASE_FILES.length} signed exes.`, + ); +} + +if (import.meta.main) main(); diff --git a/app/scripts/release/versions.ts b/app/scripts/release/versions.ts new file mode 100644 index 00000000..10b5e6f8 --- /dev/null +++ b/app/scripts/release/versions.ts @@ -0,0 +1,37 @@ +// Compares semantic versions the way semver.org section 11 orders them, so the release scripts can find the +// release before this one. The scripts have no dependencies of their own, so this is small and tested. + +/** Splits `1.2.3-beta.1+build` into its numbers and prerelease identifiers. Build metadata is ignored. */ +function parse(version: string): { numbers: number[]; pre: string[] } { + const [core, ...rest] = version.split('+')[0].split('-'); + const numbers = core.split('.').map(Number); + if (numbers.length !== 3 || numbers.some((part) => !Number.isInteger(part) || part < 0)) { + throw new Error(`"${version}" is not a semantic version.`); + } + return { numbers, pre: rest.length > 0 ? rest.join('-').split('.') : [] }; +} + +function comparePart(a: string, b: string): number { + const numeric = /^\d+$/; + if (numeric.test(a) && numeric.test(b)) return Math.sign(Number(a) - Number(b)); + if (numeric.test(a)) return -1; + if (numeric.test(b)) return 1; + return a < b ? -1 : a > b ? 1 : 0; +} + +/** Negative when `a` comes before `b`, positive when after, zero when they are the same release. */ +export function compareVersions(a: string, b: string): number { + const left = parse(a); + const right = parse(b); + for (let at = 0; at < 3; at++) { + if (left.numbers[at] !== right.numbers[at]) return Math.sign(left.numbers[at] - right.numbers[at]); + } + if (left.pre.length === 0 || right.pre.length === 0) return Math.sign(right.pre.length - left.pre.length); + for (let at = 0; at < Math.max(left.pre.length, right.pre.length); at++) { + if (left.pre[at] === undefined) return -1; + if (right.pre[at] === undefined) return 1; + const order = comparePart(left.pre[at], right.pre[at]); + if (order !== 0) return order; + } + return 0; +} diff --git a/app/scripts/release/winget.test.ts b/app/scripts/release/winget.test.ts new file mode 100644 index 00000000..e7faaa43 --- /dev/null +++ b/app/scripts/release/winget.test.ts @@ -0,0 +1,126 @@ +// @vitest-environment node +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { RELEASE_FILES, type Manifest } from '../write-manifest.ts'; +import { newTestKey, writeSampleRelease } from './test-support.ts'; +import { CONFIG_PATH, configProblems, scalar, wingetFiles, type WingetConfig } from './winget.ts'; + +const dirs: string[] = []; +afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); + +const config = JSON.parse(readFileSync(CONFIG_PATH, 'utf8')) as WingetConfig; + +function releaseManifest(tag: string, notes = 'Typed notes.\nPen drawing.'): Manifest { + const dir = mkdtempSync(join(tmpdir(), 'opennote-winget-')); + dirs.push(dir); + const name = writeSampleRelease(dir, tag, newTestKey(), { notes }); + return JSON.parse(readFileSync(join(dir, name), 'utf8')) as Manifest; +} + +describe('the package details in app/winget.json', () => { + it('follow the rules of the winget manifest schema', () => { + expect(configProblems(config)).toEqual([]); + expect(config.PackageIdentifier).toBe('XrxcGH.OpenNote'); + }); + + it('are checked for the mistakes winget would reject', () => { + const bad = { + ...config, + PackageIdentifier: 'OpenNote', + ShortDescription: 'x'.repeat(257), + Tags: Array.from({ length: 17 }, (_, at) => `tag${at}`), + PackageUrl: 'http://example.com', + }; + expect(configProblems(bad)).toEqual([ + '"OpenNote" is not a valid PackageIdentifier.', + 'ShortDescription is longer than 256 characters.', + 'Tags are limited to 16, of 40 characters each.', + 'PackageUrl must be an https address.', + ]); + }); +}); + +describe('scalar', () => { + it('writes safe text bare and everything else in double quotes', () => { + expect(scalar('OpenNote')).toBe('OpenNote'); + expect(scalar('note-taking')).toBe('note-taking'); + expect(scalar('1.0.0')).toBe('1.0.0'); + expect(scalar('1.0.0-beta.1')).toBe('1.0.0-beta.1'); + expect(scalar('a: b')).toBe('"a: b"'); + expect(scalar('with, comma')).toBe('"with, comma"'); + expect(scalar('say "hi"')).toBe('"say \\"hi\\""'); + expect(scalar('yes')).toBe('"yes"'); + expect(scalar('1.5')).toBe('"1.5"'); + expect(scalar('')).toBe('""'); + }); +}); + +describe('wingetFiles', () => { + const manifest = releaseManifest('v1.0.0'); + const files = wingetFiles(config, manifest, 'v1.0.0'); + const folder = 'manifests/x/XrxcGH/OpenNote/1.0.0'; + + it('makes the version, installer, and locale files where winget-pkgs expects them', () => { + expect(Object.keys(files)).toEqual([ + `${folder}/XrxcGH.OpenNote.yaml`, + `${folder}/XrxcGH.OpenNote.installer.yaml`, + `${folder}/XrxcGH.OpenNote.locale.en-US.yaml`, + ]); + expect(files[`${folder}/XrxcGH.OpenNote.yaml`]).toContain('ManifestType: version'); + }); + + it('lists one portable installer per architecture, with the hash the updater checks', () => { + const text = files[`${folder}/XrxcGH.OpenNote.installer.yaml`]; + expect(text).toContain('InstallerType: portable'); + expect(text).toContain('- Architecture: x64'); + expect(text).toContain('- Architecture: x86'); + expect(text).toContain('- Architecture: arm64'); + for (const { platform, file } of RELEASE_FILES) { + const entry = manifest.platforms[platform]; + expect(text).toContain(`InstallerUrl: https://github.com/XrxcGH/OpenNote/releases/download/v1.0.0/${file}`); + expect(text).toContain(`InstallerSha256: ${entry.sha256.toUpperCase()}`); + } + expect(text).toContain('ReleaseDate: 2026-10-14'); + }); + + it('puts the release notes in a block, so punctuation in them is safe', () => { + const tricky = wingetFiles(config, releaseManifest('v1.0.0', 'Fixed: "a" crash # one.\n\nSecond.'), 'v1.0.0'); + const text = tricky[`${folder}/XrxcGH.OpenNote.locale.en-US.yaml`]; + expect(text).toContain('ReleaseNotes: |-\n Fixed: "a" crash # one.\n\n Second.\n'); + expect(text).toContain('ReleaseNotesUrl: https://github.com/XrxcGH/OpenNote/releases/tag/v1.0.0'); + }); + + it('ends every file with a line break and uses no tabs', () => { + for (const text of Object.values(files)) { + expect(text.endsWith('\n')).toBe(true); + expect(text).not.toContain('\t'); + } + }); + + it('refuses a prerelease, a manifest with a missing architecture, and bad package details', () => { + expect(() => wingetFiles(config, releaseManifest('v1.0.0-beta.1'), 'v1.0.0-beta.1')).toThrow('prerelease'); + const missing = structuredClone(manifest); + delete missing.platforms['windows-i686']; + expect(() => wingetFiles(config, missing, 'v1.0.0')).toThrow('no entry for windows-i686'); + expect(() => wingetFiles({ ...config, PackageIdentifier: 'OpenNote' }, manifest, 'v1.0.0')).toThrow( + 'PackageIdentifier', + ); + }); +}); + +describe.skipIf(!process.env.WINGET_VALIDATE)('winget validate (set WINGET_VALIDATE=1 to run it)', () => { + it('accepts the files', () => { + const out = mkdtempSync(join(tmpdir(), 'opennote-winget-out-')); + dirs.push(out); + const files = wingetFiles(config, releaseManifest('v1.0.0'), 'v1.0.0'); + for (const [path, text] of Object.entries(files)) { + mkdirSync(dirname(join(out, path)), { recursive: true }); + writeFileSync(join(out, path), text); + } + const folder = dirname(join(out, Object.keys(files)[0])); + expect(execFileSync('winget', ['validate', '--manifest', folder], { encoding: 'utf8' })).toContain('succeeded'); + }, 60_000); +}); diff --git a/app/scripts/release/winget.ts b/app/scripts/release/winget.ts new file mode 100644 index 00000000..899f86fa --- /dev/null +++ b/app/scripts/release/winget.ts @@ -0,0 +1,160 @@ +// Writes the three winget manifest files for a stable release, ready to submit to the winget-pkgs repository. The +// exes are plain portable programs, so winget installs each one as a `portable` package. The download addresses, +// sizes, and hashes come from the manifest that write-manifest.ts wrote, so winget and the updater always agree. +// Usage: node app/scripts/release/winget.ts + +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { RELEASE_FILES, versionOf, type Manifest } from '../write-manifest.ts'; +import { manifestName } from './verify.ts'; + +const SCHEMA_VERSION = '1.6.0'; +const ARCHITECTURES: Record = { + 'windows-x86_64': 'x64', + 'windows-i686': 'x86', + 'windows-aarch64': 'arm64', +}; + +/** The package details that app/winget.json holds. */ +export interface WingetConfig { + PackageIdentifier: string; + PackageName: string; + Publisher: string; + PublisherUrl: string; + PackageUrl: string; + License: string; + LicenseUrl: string; + ShortDescription: string; + Moniker: string; + Command: string; + Tags: string[]; +} + +export const CONFIG_PATH = join(import.meta.dirname, '..', '..', 'winget.json'); + +/** A YAML value. A string is written bare when that is safe, and in double quotes when it is not. */ +export function scalar(value: string | number): string { + const text = String(value); + const plain = /^[A-Za-z][A-Za-z0-9 ._()/+-]*$/.test(text) && !/^(true|false|null|yes|no|on|off|y|n)$/i.test(text); + const dotted = /^\d+(\.\d+){2,}(-[A-Za-z0-9.-]+)?$/.test(text); + return plain || dotted ? text : JSON.stringify(text); +} + +/** The text of a multi-line value as a YAML block, so line breaks and colons need no escaping. */ +function block(key: string, text: string): string { + const body = text.split('\n').map((line) => (line === '' ? '' : ` ${line}`)); + return [`${key}: |-`, ...body].join('\n'); +} + +/** Everything wrong with the package details, in a sentence each. The rules follow winget's manifest schema. */ +export function configProblems(config: WingetConfig): string[] { + const problems: string[] = []; + const id = /^[^.\s\\/:*?"<>|]{1,32}(\.[^.\s\\/:*?"<>|]{1,32}){1,7}$/; + const hasControl = [...config.PackageIdentifier].some((char) => char.charCodeAt(0) < 32); + if (!id.test(config.PackageIdentifier) || hasControl) { + problems.push(`"${config.PackageIdentifier}" is not a valid PackageIdentifier.`); + } + if (config.ShortDescription.length > 256) problems.push('ShortDescription is longer than 256 characters.'); + if (config.Tags.length > 16 || config.Tags.some((tag) => tag.length > 40)) + problems.push('Tags are limited to 16, of 40 characters each.'); + for (const key of ['PublisherUrl', 'PackageUrl', 'LicenseUrl'] as const) { + if (!config[key].startsWith('https://')) problems.push(`${key} must be an https address.`); + } + return problems; +} + +function versionFile(config: WingetConfig, version: string): string { + return [ + `PackageIdentifier: ${config.PackageIdentifier}`, + `PackageVersion: ${scalar(version)}`, + 'DefaultLocale: en-US', + 'ManifestType: version', + `ManifestVersion: ${SCHEMA_VERSION}`, + '', + ].join('\n'); +} + +function installerFile(config: WingetConfig, manifest: Manifest, tag: string): string { + const installers = RELEASE_FILES.flatMap(({ platform }) => { + const entry = manifest.platforms[platform]; + if (!entry) throw new Error(`The manifest has no entry for ${platform}.`); + return [ + `- Architecture: ${ARCHITECTURES[platform]}`, + ` InstallerUrl: ${entry.url}`, + ` InstallerSha256: ${entry.sha256.toUpperCase()}`, + ]; + }); + return [ + `PackageIdentifier: ${config.PackageIdentifier}`, + `PackageVersion: ${scalar(versionOf(tag))}`, + 'MinimumOSVersion: 10.0.0.0', + 'InstallerType: portable', + 'Commands:', + `- ${scalar(config.Command)}`, + `ReleaseDate: ${manifest.pub_date.slice(0, 10)}`, + 'Installers:', + ...installers, + 'ManifestType: installer', + `ManifestVersion: ${SCHEMA_VERSION}`, + '', + ].join('\n'); +} + +function localeFile(config: WingetConfig, manifest: Manifest, tag: string): string { + const notesUrl = `${config.PackageUrl}/releases/tag/${tag}`; + return [ + `PackageIdentifier: ${config.PackageIdentifier}`, + `PackageVersion: ${scalar(versionOf(tag))}`, + 'PackageLocale: en-US', + `Publisher: ${scalar(config.Publisher)}`, + `PublisherUrl: ${config.PublisherUrl}`, + `PackageName: ${scalar(config.PackageName)}`, + `PackageUrl: ${config.PackageUrl}`, + `License: ${scalar(config.License)}`, + `LicenseUrl: ${config.LicenseUrl}`, + `ShortDescription: ${scalar(config.ShortDescription)}`, + `Moniker: ${scalar(config.Moniker)}`, + 'Tags:', + ...config.Tags.map((tagName) => `- ${scalar(tagName)}`), + block('ReleaseNotes', manifest.notes.trim()), + `ReleaseNotesUrl: ${notesUrl}`, + 'ManifestType: defaultLocale', + `ManifestVersion: ${SCHEMA_VERSION}`, + '', + ].join('\n'); +} + +/** + * The three manifest files for a stable release, by the path winget-pkgs expects under its `manifests` folder. + * It refuses a prerelease, because winget-pkgs is for stable versions, and a problem in the package details. + */ +export function wingetFiles(config: WingetConfig, manifest: Manifest, tag: string): Record { + const version = versionOf(tag); + if (version.includes('-')) throw new Error(`${tag} is a prerelease. Only stable releases go in winget.`); + const problems = configProblems(config); + if (problems.length > 0) throw new Error(problems.join(' ')); + const id = config.PackageIdentifier; + const folder = ['manifests', id[0].toLowerCase(), ...id.split('.'), version].join('/'); + return { + [`${folder}/${id}.yaml`]: versionFile(config, version), + [`${folder}/${id}.installer.yaml`]: installerFile(config, manifest, tag), + [`${folder}/${id}.locale.en-US.yaml`]: localeFile(config, manifest, tag), + }; +} + +function main(): void { + const [dir, tag, out] = process.argv.slice(2); + if (!dir || !tag || !out) throw new Error('Usage: node app/scripts/release/winget.ts '); + const manifest = JSON.parse(readFileSync(join(dir, manifestName(versionOf(tag))), 'utf8')) as Manifest; + const config = JSON.parse(readFileSync(CONFIG_PATH, 'utf8')) as WingetConfig; + const files = wingetFiles(config, manifest, tag); + for (const [path, text] of Object.entries(files)) { + mkdirSync(join(out, path, '..'), { recursive: true }); + writeFileSync(join(out, path), text); + } + console.log( + `Wrote ${Object.keys(files).length} winget files for ${config.PackageIdentifier} ${versionOf(tag)} into ${out}.`, + ); +} + +if (import.meta.main) main(); diff --git a/app/scripts/release/workflow.test.ts b/app/scripts/release/workflow.test.ts new file mode 100644 index 00000000..79a6de4b --- /dev/null +++ b/app/scripts/release/workflow.test.ts @@ -0,0 +1,365 @@ +// @vitest-environment node +// Checks the safety rules of .github/workflows/release.yml. No YAML parser is installed, so these tests read the +// file by its layout: jobs at 2 spaces, job settings at 4, steps as list items at 6. +// +// They catch the mistakes that matter most for a workflow that signs and publishes. Examples are a secret in the +// wrong step, an action that is not pinned, a dry run that can publish, and untrusted text pasted into a script. +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { RELEASE_FILES, REPO } from '../write-manifest.ts'; +import { SBOM_FILE, hashOutput } from './check-downloads.ts'; + +const workflowFile = (name: string) => + readFileSync(join(import.meta.dirname, '..', '..', '..', '.github', 'workflows', name), 'utf8'); +const WORKFLOW = workflowFile('release.yml'); +const CI = workflowFile('ci.yml'); + +/** The text of each job, by name. */ +function jobsOf(text: string): Record { + const body = text.slice(text.indexOf('\njobs:\n') + 7); + const parts = body.split(/^ {2}(?=[a-z][\w-]*:\s*$)/m).filter(Boolean); + return Object.fromEntries(parts.map((part) => [part.slice(0, part.indexOf(':')), part])); +} + +/** The text of each step of a job, in order. */ +function stepsOf(job: string): string[] { + return job.split(/^ {6}- /m).slice(1); +} + +/** The script of each step of a job that has one. */ +function scriptsOf(job: string): string[] { + return stepsOf(job) + .map((step) => step.split(/^ {8}run: ?/m)[1]?.split(/^ {8}\S/m)[0]) + .filter((script): script is string => script !== undefined) + .map((script) => script.replace(/^[|>][-+]?\n/, '')); +} + +const jobs = jobsOf(WORKFLOW); +const names = Object.keys(jobs); +const stepNamed = (job: string, title: string) => { + const step = stepsOf(jobs[job]).find((candidate) => + candidate.split('\n').some((line) => line.trim().startsWith(`name: ${title}`)), + ); + if (!step) throw new Error(`No step "${title}" in ${job}.`); + return step; +}; + +describe('release.yml triggers', () => { + it('runs for version tags, and by hand as a dry run', () => { + expect(WORKFLOW).toMatch(/^on:\n {2}push:\n {4}tags: \['v\*'\]\n {2}workflow_dispatch:\n/m); + }); + + it('has these jobs', () => { + expect(names).toEqual(['version', 'checklist', 'ci', 'build', 'sbom', 'package', 'update-test', 'publish']); + }); + + it('only waits for jobs that exist, and a job never waits for itself', () => { + for (const [name, job] of Object.entries(jobs)) { + const needs = /^ {4}needs: (.+)$/m.exec(job)?.[1] ?? ''; + for (const needed of needs + .replace(/[[\]]/g, '') + .split(',') + .map((each) => each.trim()) + .filter(Boolean)) { + expect(names, `${name} needs ${needed}`).toContain(needed); + expect(needed).not.toBe(name); + } + } + }); + + it('builds the exes only after the checklist and CI, and packages only after the build', () => { + expect(jobs.build).toMatch(/needs: \[ci, checklist\]/); + expect(jobs.package).toMatch(/needs: \[version, checklist, build, sbom\]/); + expect(jobs.publish).toMatch(/needs: \[version, package, update-test\]/); + expect(jobs['update-test']).toMatch(/needs: \[version, package\]/); + }); +}); + +describe('permissions', () => { + it('start from nothing, and every job asks for its own', () => { + expect(WORKFLOW).toMatch(/^permissions: \{\}$/m); + for (const [name, job] of Object.entries(jobs)) + expect(job, name).toMatch(/^ {4}permissions:\n {6}contents: (read|write)/m); + }); + + it('let only the publish job write', () => { + for (const [name, job] of Object.entries(jobs)) { + expect(/: write\b/.test(job), name).toBe(name === 'publish'); + } + }); + + it('give the checklist job read access to issues and runs, and no other job', () => { + expect(jobs.checklist).toMatch(/actions: read\n {6}issues: read/); + for (const name of names.filter((each) => each !== 'checklist')) + expect(jobs[name], name).not.toMatch(/issues: read/); + }); +}); + +describe('actions', () => { + it('are pinned to a full commit SHA, with the version in a comment', () => { + const used = [...WORKFLOW.matchAll(/^\s+(?:- )?uses: (\S+)(.*)$/gm)].filter((match) => !match[1].startsWith('./')); + expect(used.length).toBeGreaterThan(8); + for (const [, action, rest] of used) { + expect(action, action).toMatch(/@[0-9a-f]{40}$/); + expect(rest, action).toMatch(/# v\d/); + } + }); + + it('are pinned in ci.yml too, which runs in the same release run and can upload artifacts to it', () => { + expect(jobs.ci).toMatch(/^ {4}uses: \.\/\.github\/workflows\/ci\.yml$/m); + const used = [...CI.matchAll(/^\s+(?:- )?uses: (\S+)(.*)$/gm)]; + expect(used.length).toBeGreaterThan(20); + for (const [, action, rest] of used) { + expect(action, `ci.yml uses ${action}`).toMatch(/@[0-9a-f]{40}$/); + expect(rest, `ci.yml uses ${action}`).toMatch(/# v\d/); + } + }); + + it('check out without leaving a token on disk', () => { + const checkouts = [...WORKFLOW.matchAll(/uses: actions\/checkout@[^\n]*\n((?: {8,}[^\n]*\n)+)/g)]; + expect(checkouts).toHaveLength(6); + for (const [, options] of checkouts) expect(options).toContain('persist-credentials: false'); + }); +}); + +describe('secrets', () => { + it('appear only in the step that signs the updates, which runs only for a pushed tag', () => { + const owners = Object.entries(jobs).flatMap(([name, job]) => + stepsOf(job) + .filter((step) => /\$\{\{\s*secrets\./.test(step)) + .map((step) => `${name}: ${step.split('\n')[0]}`), + ); + expect(owners).toEqual(['package: name: Sign the updates']); + const step = stepNamed('package', 'Sign the updates'); + expect(step).toContain("if: github.event_name == 'push'"); + expect(step).toContain('secrets.TAURI_SIGNING_PRIVATE_KEY }}'); + expect(step).toContain('secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}'); + expect(step).not.toContain('--dry-run'); + }); + + it('are not used anywhere outside a step, such as a job or workflow setting', () => { + expect(WORKFLOW.match(/\$\{\{\s*secrets\./g)).toHaveLength(2); + }); + + it('belong to the release environment, which the package job uses for a pushed tag and never for a dry run', () => { + expect(jobs.package).toMatch(/^ {4}environment: \$\{\{ github\.event_name == 'push' && 'release' \|\| '' \}\}$/m); + }); + + it('wait for a second approval before the publish job, and no other job uses an environment', () => { + expect(jobs.publish).toMatch(/^ {4}environment: release$/m); + const withEnvironment = names.filter((name) => /^ {4}environment:/m.test(jobs[name])); + expect(withEnvironment).toEqual(['package', 'publish']); + }); +}); + +describe('a pushed tag', () => { + it('is released only from a commit on main', () => { + const step = stepNamed('version', 'Check that the tag is on main'); + expect(step).toContain("if: github.event_name == 'push'"); + expect(step).toContain('git merge-base --is-ancestor $env:GITHUB_SHA origin/main'); + expect(step).toContain('if ($LASTEXITCODE -ne 0)'); + expect(stepsOf(jobs.version)[0]).toContain('fetch-depth: 0'); + }); +}); + +describe('the dry run', () => { + it('is the only kind of run by hand, with its dry_run input on', () => { + expect(WORKFLOW).toMatch(/workflow_dispatch:\n {4}inputs:\n {6}dry_run:\n(?: {8}.+\n)+/); + expect(WORKFLOW).toContain(' default: true\n'); + const step = stepNamed('version', 'Refuse a run by hand that is not a dry run'); + expect(step).toContain('DRY_RUN: ${{ inputs.dry_run }}'); + expect(step).toContain("if ($env:DRY_RUN -ne 'true')"); + }); + + it('tests an update from the last beta with the throwaway key, and a tag with the committed key', () => { + expect(stepNamed('update-test', 'Update from the last beta (dry run)')).toContain('--pubkey dry-run/update.pub'); + const tagged = stepNamed('update-test', 'Update from the last beta (tag)'); + expect(tagged).toContain("if: github.event_name == 'push'"); + expect(tagged).not.toContain('--pubkey'); + expect(jobs['update-test']).not.toMatch(/\$\{\{\s*secrets\./); + }); + + it('may build without a committed update key, and a release may not', () => { + expect(jobs.build).toContain("OPENNOTE_DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}"); + }); + + it('signs with a throwaway key and never with the secrets', () => { + const step = stepNamed('package', 'Sign with a throwaway key (dry run)'); + expect(step).toContain("if: github.event_name == 'workflow_dispatch'"); + expect(step).toContain('--dry-run --pubkey-out dry-run/update.pub'); + expect(step).not.toContain('secrets.'); + }); + + it('checks the result against the throwaway key', () => { + expect(stepNamed('package', 'Check the dry run the way the app will')).toContain('--pubkey dry-run/update.pub'); + }); + + it('never reaches the publish job', () => { + expect(jobs.publish).toMatch(/^ {4}if: github\.event_name == 'push'$/m); + }); + + it('stays out of the publish job, which only publishes what the package job kept', () => { + expect(stepsOf(jobs.publish).map((step) => step.split('\n')[0])).toEqual([ + 'name: Download the release files', + 'name: Check that the beta channel can be updated', + 'name: Publish the release', + 'name: Update the beta channel', + ]); + expect(jobs.publish).not.toContain('checkout'); + // Its scripts run only the GitHub CLI, on the beta channel's prerelease. + const allowed = /^(\$prerelease = )?gh release (view|upload) channel-manifests |^if \(|^\}$|^exit 1$|^"::error::/; + const lines = scriptsOf(jobs.publish).flatMap((script) => script.split('\n').map((line) => line.trim())); + for (const line of lines.filter(Boolean)) expect(line).toMatch(allowed); + }); +}); + +describe('the beta channel', () => { + const config = readFileSync( + join(import.meta.dirname, '..', '..', '..', 'crates', 'updater', 'src', 'config.rs'), + 'utf8', + ); + const betaUrl = /BETA_MANIFEST_URL: &str = "([^"]+)"/.exec(config)?.[1] ?? ''; + const download = /^https:\/\/github\.com\/([^/]+\/[^/]+)\/releases\/download\/([^/]+)\/([^/]+)$/; + const [, owner, tag, file] = download.exec(betaUrl) ?? ['', '', '', '']; + const forBetasOnly = "if: needs.version.outputs.enforce == 'true' && contains(needs.version.outputs.tag, '-')"; + + it('is read by the updater from a fixed prerelease of this repository', () => { + expect(owner).toBe(REPO); + expect(tag).toBe('channel-manifests'); + expect(file).toBe('beta.json'); + }); + + it("gets each beta's manifest, at the address the updater reads", () => { + const step = stepNamed('publish', 'Update the beta channel'); + expect(step).toContain(`gh release upload ${tag} dist-release/${file} --clobber`); + expect(step).toContain(forBetasOnly); + }); + + it('never gets a test build, whose version sorts above the betas of its release', () => { + expect(jobs.version).toContain( + "$enforce = ($env:GITHUB_EVENT_NAME -eq 'push') -and ($tag -notmatch '-test(\\.|$)')", + ); + expect(stepNamed('publish', 'Update the beta channel')).toContain("needs.version.outputs.enforce == 'true'"); + }); + + it('is checked before the release is published, so a missing prerelease stops it in time', () => { + const order = stepsOf(jobs.publish).map((step) => step.split('\n')[0]); + const check = stepNamed('publish', 'Check that the beta channel can be updated'); + expect(check).toContain(forBetasOnly); + expect(check).toContain(`gh release view ${tag} --json isPrerelease`); + expect(order.indexOf('name: Check that the beta channel can be updated')).toBeLessThan( + order.indexOf('name: Publish the release'), + ); + expect(order.indexOf('name: Update the beta channel')).toBeGreaterThan(order.indexOf('name: Publish the release')); + }); +}); + +describe('the package job', () => { + const order = stepsOf(jobs.package).map((step) => /name: (.+)/.exec(step)?.[1] ?? ''); + const at = (title: string) => order.findIndex((name) => name.startsWith(title)); + + it('signs, then writes the manifest, then checks the release, in that order', () => { + const steps = [ + 'Write the release notes', + 'Sign the updates', + 'Write the update manifest', + 'Write the checksums', + 'Check the release', + ]; + const places = steps.map(at); + expect(places.every((place) => place >= 0)).toBe(true); + expect([...places].sort((a, b) => a - b)).toEqual(places); + }); + + it('runs no install scripts next to the signing key', () => { + expect(stepNamed('package', 'Install tools without install scripts')).toContain('npm ci --ignore-scripts'); + expect(jobs.package).not.toMatch(/run: npm ci\s*$/m); + }); + + it('uses the release notes in the manifest', () => { + expect(stepNamed('package', 'Write the update manifest')).toContain('RELEASE_NOTES_FILE: release-notes.txt'); + }); + + it('requires an Authenticode signature for a release, and not for a test build', () => { + const step = stepNamed('package', 'Check the release the way the app will'); + expect(step).toContain("if: github.event_name == 'push'"); + expect(step).toContain("@('--require-authenticode')"); + }); + + it('writes winget manifests for a stable release only', () => { + expect(stepNamed('package', 'Write the winget manifests')).toContain( + "if: ${{ !contains(needs.version.outputs.tag, '-') }}", + ); + }); +}); + +describe('scripts', () => { + it('never paste a context into the text of a command, where a tag name could change the command', () => { + const risky = /\$\{\{\s*(github\.(head_ref|event\.|ref_name)|needs\.|steps\.|inputs\.)/; + for (const [name, job] of Object.entries(jobs)) { + for (const step of stepsOf(job)) { + const script = step.split(/^ {8}run: ?/m)[1]?.split(/^ {8}\S/m)[0] ?? ''; + expect(script, `${name}: ${step.split('\n')[0]}`).not.toMatch(risky); + } + } + }); + + it('write the version job outputs the other jobs read', () => { + expect(jobs.version).toContain('tag: ${{ steps.tag.outputs.tag }}'); + expect(jobs.version).toContain('enforce: ${{ steps.tag.outputs.enforce }}'); + expect(jobs.version).toContain("-notmatch '-test(\\.|$)'"); + }); + + it('enforce the checklist only for a pushed tag that is not a test build', () => { + const step = stepNamed('checklist', 'Check the release checklist'); + expect(step).toContain("$extra = if ($env:ENFORCE -eq 'true') { @() } else { @('--report') }"); + }); + + it('check each exe before it is signed', () => { + expect(stepNamed('build', 'Check that the exe may ship')).toContain('check-release-exe.ts'); + }); +}); + +describe('the files that get signed', () => { + const order = stepsOf(jobs.package).map((step) => /name: (.+)/.exec(step)?.[1] ?? ''); + const check = 'Check the downloads before signing'; + + it('are downloaded by exact name, never by a pattern that another job of the run could match', () => { + expect(jobs.package).not.toMatch(/^ +(pattern|merge-multiple|artifact-ids):/m); + const artifacts = stepsOf(jobs.package) + .filter((step) => step.includes('uses: actions/download-artifact@')) + .map((step) => /^ {10}name: (.+)$/m.exec(step)?.[1]); + expect(artifacts).toEqual([...RELEASE_FILES.map(({ target }) => `exe-${target}`), 'sbom']); + expect(jobs.build).toContain('name: exe-${{ matrix.target }}\n'); + expect(jobs.sbom).toContain('name: sbom\n'); + }); + + it('have the hash that the build and sbom jobs reported for them', () => { + for (const { target } of RELEASE_FILES) { + const output = hashOutput(target); + expect(jobs.build).toContain(` ${output}: \${{ steps.hash.outputs.${output} }}`); + } + expect(stepNamed('build', 'Report the hash of the exe')).toContain('"sha256-$env:RELEASE_TARGET=$hash"'); + expect(jobs.sbom).toContain(' sha256: ${{ steps.hash.outputs.sha256 }}'); + expect(stepNamed('sbom', 'Report the hash of the bill of materials')).toContain(`dist-sbom/${SBOM_FILE}`); + const step = stepNamed('package', check); + expect(step).toContain('BUILD_SHA256: ${{ toJSON(needs.build.outputs) }}'); + expect(step).toContain('SBOM_SHA256: ${{ needs.sbom.outputs.sha256 }}'); + expect(step).toContain('run: node app/scripts/release/check-downloads.ts dist-release'); + }); + + it('are checked after every download and before anything is signed', () => { + const downloads = order.flatMap((name, place) => (name.startsWith('Download') ? [place] : [])); + expect(downloads).toHaveLength(4); + expect(order.indexOf(check)).toBeGreaterThan(Math.max(...downloads)); + expect(order.indexOf(check)).toBeLessThan(order.indexOf('Sign the updates')); + expect(order.indexOf(check)).toBeLessThan(order.indexOf('Sign with a throwaway key (dry run)')); + }); + + it('are built without a cache that the CI jobs of the same ref can write', () => { + expect(jobs.build).not.toMatch(/^\s+cache:/m); + expect(jobs.build).not.toContain('rust-cache'); + expect(jobs.build).not.toContain('actions/cache'); + }); +}); diff --git a/app/scripts/write-manifest.test.ts b/app/scripts/write-manifest.test.ts index bfcbdac2..3bf3e98c 100644 --- a/app/scripts/write-manifest.test.ts +++ b/app/scripts/write-manifest.test.ts @@ -8,6 +8,7 @@ import { checkSignedComment, manifestFile, manifestFor, + notesFor, platformEntry, readReleaseFile, readSignature, @@ -221,3 +222,23 @@ describe('writeManifest', () => { expect(existsSync(join(dir, 'latest.json'))).toBe(false); }); }); + +describe('notesFor', () => { + it('prefers the notes file, then RELEASE_NOTES, then a one-line default', () => { + const dir = releaseDir('0.5.0'); + const file = join(dir, 'notes.txt'); + writeFileSync(file, ' Faster start-up.\n'); + expect(notesFor('v0.5.0', { RELEASE_NOTES_FILE: file, RELEASE_NOTES: 'from the variable' })).toBe( + 'Faster start-up.', + ); + expect(notesFor('v0.5.0', { RELEASE_NOTES: ' from the variable ' })).toBe('from the variable'); + expect(notesFor('v0.5.0', {})).toBe('OpenNote v0.5.0'); + }); + + it('falls back when the notes file is missing or empty', () => { + const dir = releaseDir('0.5.0'); + writeFileSync(join(dir, 'empty.txt'), '\n'); + expect(notesFor('v0.5.0', { RELEASE_NOTES_FILE: join(dir, 'empty.txt') })).toBe('OpenNote v0.5.0'); + expect(notesFor('v0.5.0', { RELEASE_NOTES_FILE: join(dir, 'missing.txt'), RELEASE_NOTES: 'kept' })).toBe('kept'); + }); +}); diff --git a/app/scripts/write-manifest.ts b/app/scripts/write-manifest.ts index bc098463..54c24de4 100644 --- a/app/scripts/write-manifest.ts +++ b/app/scripts/write-manifest.ts @@ -6,7 +6,8 @@ import { createHash } from 'node:crypto'; import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; -const REPO = 'XrxcGH/OpenNote'; +/** The GitHub repository that hosts the releases. The updater only downloads from github.com. */ +export const REPO = 'XrxcGH/OpenNote'; /** One file a release ships: the Tauri updater's platform key, the Rust target it's built for, and its name. */ export interface ReleaseFile { @@ -141,10 +142,20 @@ export function writeManifest( return { name, manifest }; } +/** + * The manifest's release notes: the text in the file named by RELEASE_NOTES_FILE (release/notes.ts writes it), else + * RELEASE_NOTES, else a one-line default. The app shows at most 8 lines of them in its update notice. + */ +export function notesFor(tag: string, env: Record): string { + const file = env.RELEASE_NOTES_FILE; + const fromFile = file && existsSync(file) ? readFileSync(file, 'utf8').trim() : ''; + return fromFile || env.RELEASE_NOTES?.trim() || `OpenNote ${tag}`; +} + function main(): void { const [dir, tag] = process.argv.slice(2); if (!dir || !tag) throw new Error('Usage: node app/scripts/write-manifest.ts '); - const notes = process.env.RELEASE_NOTES ?? `OpenNote ${tag}`; + const notes = notesFor(tag, process.env); const { name, manifest } = writeManifest(dir, tag, notes, new Date().toISOString()); const files = Object.keys(manifest.platforms).length; console.log(`Wrote ${name} for ${manifest.version}, with ${files} files.`); diff --git a/app/src-tauri/Cargo.toml b/app/src-tauri/Cargo.toml index 76f5f525..8740c037 100644 --- a/app/src-tauri/Cargo.toml +++ b/app/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "opennote" -version = "0.1.0" +version = "0.1.0-beta.5" description = "Open-source note-taking app combining writing, drawing, and recording" edition = "2021" license = "Apache-2.0" @@ -14,49 +14,103 @@ crate-type = ["staticlib", "cdylib", "rlib"] # For CI's end-to-end builds only: test update endpoints on 127.0.0.1 and a test key compiled in from # OPENNOTE_TEST_UPDATE_PUBKEY. The release build job fails for an exe built with it. test-endpoints = ["opennote-updater/test-endpoints"] +# The native engines: whisper.cpp for speech and ONNX Runtime for translation and meaning. Release builds turn it on. +# It needs CMake, a C++ compiler, and libclang. Without it the pure Rust speech reader runs, and translation and +# meaning vectors report that this build does not carry them. +native-engines = ["opennote-intel-engines/whisper-cpp", "opennote-intel-engines/onnx"] +# The native engines plus DirectML for translation and meaning vectors (graphics card or NPU, processor if it cannot start). +native-directml = ["native-engines", "opennote-intel-engines/onnx-directml"] +# Puts the language model program (crates/llm-host, built with its llama feature) inside the exe. The release build sets +# OPENNOTE_LLM_HOST_BIN to the path of the built program. A release is one exe, so the program cannot travel beside it. +embed-llm-host = [] [build-dependencies] base64 = "0.22.1" +serde_json.workspace = true tauri-build = { version = "2", features = [] } [dependencies] +base64 = "0.23.1" +crc32fast = "1.5.2" +flate2 = "1.1.10" +# Connectors: PKCE and sign-in state need the operating system's random generator, the token endpoints take +# form bodies, and the account name is read from the payload of an ID token. All three are already built. +getrandom = "0.3.4" log.workspace = true +opennote-api = { path = "../../crates/api" } opennote-core = { path = "../../crates/core" } +opennote-crashreport = { path = "../../crates/crashreport" } +opennote-diagnostics = { path = "../../crates/diagnostics" } +opennote-intel = { path = "../../crates/intel" } +opennote-intel-engines = { path = "../../crates/intel-engines" } +opennote-interop = { path = "../../crates/interop" } +# Phase 9: recording to Opus needs libopus, which builds with CMake (preinstalled on the Windows runners). +opennote-metafile = { path = "../../crates/third_party/metafile" } +opennote-media = { path = "../../crates/media", features = ["opus"] } +opennote-print = { path = "../../crates/print" } +opennote-search = { path = "../../crates/search" } opennote-updater = { path = "../../crates/updater" } +# Phase 10: Anki packages are zip files that hold a SQLite collection, with the same SQLite build as the search crate. +rusqlite = { version = "0.40.2", features = ["bundled"] } +notify = "8.2.0" semver.workspace = true serde.workspace = true serde_json.workspace = true +sha1 = "0.11.0" sha2.workspace = true tauri = { version = "2", features = [] } # Web images for paste (Phase 4 ARCHITECTURE.md section 12.2), with the same TLS stack and version as the updater. -ureq = { version = "3.4.2", default-features = false, features = ["native-tls-no-default"] } +ureq = { version = "3.4.2", default-features = false, features = ["native-tls"] } +tempfile.workspace = true +url = "2.5.8" +# Protected sections: passwords and Hello signatures sit in buffers that are wiped when dropped. +zeroize = "1.9.0" [target.'cfg(windows)'.dependencies] webview2-com = "0.39.1" +# For `#[implement]`, which the Snap Layouts overlay's UI Automation provider uses. +windows-core = "0.62.2" [target.'cfg(windows)'.dependencies.windows] version = "0.62.2" features = [ + "ApplicationModel_Activation", + "ApplicationModel_DataTransfer_ShareTarget", "Foundation", + "Foundation_Collections", + "Media_ClosedCaptioning", "Networking_Connectivity", + "Storage_FileProperties", + "Security_Credentials", + "Security_Cryptography", + "Storage_Streams", "UI_ViewManagement", "Win32_Foundation", "Win32_Globalization", + "Win32_Media_MediaFoundation", "Win32_Graphics_Dwm", "Win32_Graphics_Gdi", "Win32_Graphics_Imaging", "Win32_Security", + "Win32_Security_Credentials", "Win32_Storage_FileSystem", + "Win32_Storage_Packaging_Appx", "Win32_System_Com", "Win32_System_Com_StructuredStorage", "Win32_System_DataExchange", "Win32_System_LibraryLoader", "Win32_System_Memory", "Win32_System_Ole", + "Win32_System_Power", "Win32_System_Registry", + "Win32_System_RemoteDesktop", + "Win32_System_Time", "Win32_System_Threading", "Win32_System_Variant", + "Win32_System_WindowsProgramming", "Win32_UI_Accessibility", + "Win32_UI_HiDpi", + "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_Shell", "Win32_UI_Shell_Common", "Win32_UI_Shell_PropertiesSystem", @@ -65,5 +119,8 @@ features = [ [dev-dependencies] proptest.workspace = true +# A local HTTPS server for the web image tests (T1-1): a certificate made per run, so no key is ever committed. +rcgen = "0.13.2" +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } tempfile.workspace = true ts-rs = { version = "12.0.1", features = ["no-serde-warnings"] } diff --git a/app/src-tauri/build.rs b/app/src-tauri/build.rs index fa83a0fb..1742854a 100644 --- a/app/src-tauri/build.rs +++ b/app/src-tauri/build.rs @@ -1,6 +1,7 @@ //! The build script. It declares every app command in Tauri's app manifest, so each one gets an `allow-` //! permission that capabilities/default.json grants by name, instead of allowing every command. It also writes -//! `OUT_DIR/update_keys.rs` with the update public keys from `keys/*.pub`, which `src/updater.rs` includes. +//! `OUT_DIR/update_keys.rs` with the update public keys from `keys/*.pub`, which `src/updater.rs` includes. A +//! `release` build without a key stops (`src/build_rules.rs`). use std::{ env, fs, @@ -10,6 +11,7 @@ use std::{ use base64::{engine::general_purpose::STANDARD, Engine as _}; include!("src/command_list.rs"); +include!("src/build_rules.rs"); /// The folder of update public keys, each as `tauri signer generate` writes it. const KEYS_DIR: &str = "keys"; @@ -20,13 +22,117 @@ const TEST_ENDPOINTS_MARKER: &str = "OPENNOTE-TEST-ENDPOINTS-BUILD"; fn main() { println!("cargo:rerun-if-changed=src/command_list.rs"); println!("cargo:rerun-if-changed={KEYS_DIR}"); + println!("cargo:rerun-if-env-changed=OPENNOTE_DRY_RUN"); let test_endpoints = env::var_os("CARGO_FEATURE_TEST_ENDPOINTS").is_some(); - let source = update_keys_source(&public_keys(Path::new(KEYS_DIR)), test_endpoints); + let keys = public_keys(Path::new(KEYS_DIR)); + let source = update_keys_source(&keys, test_endpoints); let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("Cargo sets OUT_DIR")); + let profile = profile_from_out_dir(&out_dir).unwrap_or_default(); + let dry_run = env::var("OPENNOTE_DRY_RUN").is_ok_and(|value| value == "1"); + if let Some(problem) = release_key_problem(&profile, keys.len(), test_endpoints, dry_run) { + panic!("{problem}"); + } write_if_changed(&out_dir.join("update_keys.rs"), &source); + embed_cli(&out_dir); + + connector_clients(); let manifest = tauri_build::AppManifest::new().commands(APP_COMMANDS); - tauri_build::try_build(tauri_build::Attributes::new().app_manifest(manifest)).expect("the Tauri build step failed"); + let windows = embed_windows_manifest(&out_dir); + tauri_build::try_build( + tauri_build::Attributes::new() + .app_manifest(manifest) + .windows_attributes(windows), + ) + .expect("the Tauri build step failed"); +} + +/// Copies the `opennote` command-line tool that the release job built (`OPENNOTE_CLI_EXE`) to +/// `OUT_DIR/opennote-cli.bin`, which `src/install/cli.rs` includes, so the one exe can install the tool. Without the +/// variable the file is empty, and the app says the tool isn't installed. +fn embed_cli(out_dir: &Path) { + println!("cargo:rerun-if-env-changed=OPENNOTE_CLI_EXE"); + let bytes = match env::var_os("OPENNOTE_CLI_EXE").filter(|path| !path.is_empty()) { + Some(path) => { + let path = PathBuf::from(path); + println!("cargo:rerun-if-changed={}", path.display()); + fs::read(&path).unwrap_or_else(|error| panic!("OPENNOTE_CLI_EXE {} can't be read: {error}", path.display())) + } + None => Vec::new(), + }; + let target = out_dir.join("opennote-cli.bin"); + if fs::read(&target).ok().as_deref() != Some(bytes.as_slice()) { + fs::write(&target, bytes).expect("OUT_DIR is writable"); + } +} + +/// Links the Windows manifest into every executable this crate makes, the unit-test runner included, rather than +/// only into the app as Tauri's resource would. The test runner reaches window code that needs Common Controls 6, +/// and without the manifest it doesn't start (STATUS_ENTRYPOINT_NOT_FOUND). +fn embed_windows_manifest(out_dir: &Path) -> tauri_build::WindowsAttributes { + let windows = tauri_build::WindowsAttributes::new_without_app_manifest(); + let msvc = env::var("CARGO_CFG_TARGET_OS").is_ok_and(|os| os == "windows") + && env::var("CARGO_CFG_TARGET_ENV").is_ok_and(|target| target == "msvc"); + if !msvc { + return windows.app_manifest(windows_manifest()); + } + let path = out_dir.join("windows-app.manifest"); + write_if_changed(&path, &windows_manifest()); + println!("cargo:rustc-link-arg=/MANIFEST:EMBED"); + println!("cargo:rustc-link-arg=/MANIFESTINPUT:{}", path.display()); + windows +} + +/// The exe's Windows manifest, with the sparse package's publisher for "Share to OpenNote" (packaging/msix): the +/// certificate subject in `OPENNOTE_MSIX_PUBLISHER`, or the development publisher `register-dev.ps1` uses. +fn windows_manifest() -> String { + println!("cargo:rerun-if-changed=windows-app.manifest"); + println!("cargo:rerun-if-env-changed=OPENNOTE_MSIX_PUBLISHER"); + let publisher = env::var("OPENNOTE_MSIX_PUBLISHER") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "CN=OpenNote Development".to_owned()); + assert!( + publisher.starts_with("CN=") && !publisher.contains(['"', '<', '>', '&']), + "OPENNOTE_MSIX_PUBLISHER must look like CN=Name" + ); + let template = fs::read_to_string("windows-app.manifest").expect("windows-app.manifest is readable"); + template.replace("publisher=\"PUBLISHER\"", &format!("publisher=\"{publisher}\"")) +} + +/// The file at the repository root where the owner of a build keeps the app registrations of the connectors. It is +/// not in the repository. Its shape is the one of `connectors.config.example.json`. +const CONNECTORS_CONFIG: &str = "../../connectors.config.json"; + +/// The services whose client IDs can be built in. Each has `OPENNOTE__CLIENT_ID` and `..._CLIENT_SECRET`. +const CONNECTOR_NAMES: [&str; 6] = ["microsoft", "google", "slack", "dropbox", "box", "vimeo"]; + +/// Hands the client IDs of `connectors.config.json` to the compiler as the same variables a build can set by hand +/// (`src/connectors/files.rs` reads them with `option_env!`). A variable that is already set wins over the file. +fn connector_clients() { + println!("cargo:rerun-if-changed={CONNECTORS_CONFIG}"); + let file = fs::read_to_string(CONNECTORS_CONFIG) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()); + for name in CONNECTOR_NAMES { + let upper = name.to_ascii_uppercase(); + for (field, suffix) in [("clientId", "CLIENT_ID"), ("clientSecret", "CLIENT_SECRET")] { + let variable = format!("OPENNOTE_{upper}_{suffix}"); + println!("cargo:rerun-if-env-changed={variable}"); + if env::var_os(&variable).is_some() { + continue; + } + let value = file + .as_ref() + .and_then(|root| root.pointer(&format!("/clients/{name}/{field}"))) + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|text| !text.is_empty()); + if let Some(value) = value { + println!("cargo:rustc-env={variable}={value}"); + } + } + } } /// The decoded text of every `*.pub` file in `dir`, sorted by file name. A missing folder has no keys. diff --git a/app/src-tauri/capabilities/default.json b/app/src-tauri/capabilities/default.json index 35d3fe2e..e0e9e197 100644 --- a/app/src-tauri/capabilities/default.json +++ b/app/src-tauri/capabilities/default.json @@ -2,7 +2,7 @@ "$schema": "../gen/schemas/desktop-schema.json", "identifier": "default", "description": "Permissions for the main window: Tauri's core defaults and each app command by name. No plugin permissions.", - "windows": ["main"], + "windows": ["main", "tool-*", "page-*", "capture"], "permissions": [ "core:default", "allow-settings-update", @@ -19,8 +19,60 @@ "allow-app-first-paint", "allow-app-ready", "allow-app-exit-ready", + "allow-app-close-anyway", "allow-perf-mark", "allow-log-write", + "allow-audio-assets-dir", + "allow-audio-devices", + "allow-audio-clock", + "allow-audio-prepare", + "allow-audio-begin", + "allow-audio-recording-status", + "allow-audio-pause-recording", + "allow-audio-resume-recording", + "allow-audio-switch-microphone", + "allow-audio-switch-system-audio", + "allow-audio-stop", + "allow-audio-recover", + "allow-audio-open-playback", + "allow-audio-play", + "allow-audio-pause-playback", + "allow-audio-seek", + "allow-audio-skip", + "allow-audio-set-speed", + "allow-audio-set-skip-silence", + "allow-audio-playback-status", + "allow-audio-close-playback", + "allow-audio-trim-silence", + "allow-audio-remove-part", + "allow-audio-delete-files", + "allow-audio-adopt-tracks", + "allow-audio-split", + "allow-audio-enhance", + "allow-audio-compress", + "allow-audio-storage-scan", + "allow-audio-purge-history", + "allow-audio-export", + "allow-audio-export-video", + "allow-audio-import-file", + "allow-audio-meeting-poll", + "allow-audio-snap", + "allow-crash-consent-get", + "allow-crash-consent-set", + "allow-crash-example", + "allow-crash-list", + "allow-crash-prepare", + "allow-crash-send", + "allow-crash-delete", + "allow-crash-delete-all", + "allow-diagnostics-self-check", + "allow-diagnostics-build-feedback", + "allow-diagnostics-save-feedback", + "allow-diagnostics-startup", + "allow-diagnostics-enter-safe-mode", + "allow-diagnostics-restart", + "allow-privacy-get", + "allow-privacy-set-offline", "allow-install-status", "allow-install-pick-folder", "allow-install-check-folder", @@ -35,6 +87,22 @@ "allow-updater-go-back", "allow-notes-snapshot-load", "allow-notes-snapshot-save", + "allow-notes-load-initial", + "allow-notes-list-notebooks", + "allow-notes-list-children", + "allow-notes-get", + "allow-notes-create", + "allow-notes-rename", + "allow-notes-set-color", + "allow-notes-move", + "allow-notes-set-page-level", + "allow-notes-trash", + "allow-notes-restore", + "allow-notes-list-trash", + "allow-notes-restore-from-trash", + "allow-notes-purge", + "allow-notes-status", + "allow-notes-flush", "allow-page-open", "allow-page-apply", "allow-page-undo", @@ -46,6 +114,11 @@ "allow-history-restore", "allow-history-restore-blocks", "allow-history-name", + "allow-history-delete", + "allow-page-add-strokes", + "allow-page-read-strokes", + "allow-search-call", + "allow-shellqol-call", "allow-clipboard-facts", "allow-clipboard-read", "allow-image-import", @@ -57,6 +130,71 @@ "allow-spell-add-word", "allow-spell-remove-word", "allow-speech-voices", - "allow-speech-synthesize" + "allow-speech-synthesize", + "allow-print-prepare", + "allow-print-render", + "allow-print-close", + "allow-export-pick-save", + "allow-export-write", + "allow-export-open", + "allow-export-selection-docx", + "allow-print-printers", + "allow-print-submit", + "allow-tool-window-open", + "allow-tool-windows-reset", + "allow-study-anki-read", + "allow-study-anki-write", + "allow-study-zotero-items", + "allow-study-cite-lookup", + "allow-study-zotero-run", + "allow-study-zotero-cancel", + "allow-study-zotero-reply", + "allow-interop-pick", + "allow-interop-detect", + "allow-interop-local-sources", + "allow-interop-preview", + "allow-interop-import", + "allow-interop-cancel", + "allow-interop-export", + "allow-interop-reveal", + "allow-interop-more", + "allow-intel-settings-get", + "allow-intel-settings-set", + "allow-intel-status", + "allow-intel-ocr-languages", + "allow-intel-ocr-recognize", + "allow-intel-ink-recognize", + "allow-intel-ink-tidy", + "allow-intel-summarize", + "allow-intel-keywords", + "allow-intel-action-items", + "allow-intel-chapters", + "allow-intel-vocabulary-offer", + "allow-intel-speech-voices", + "allow-intel-speech-synthesize", + "allow-intel-read-aloud-start", + "allow-intel-read-aloud-next", + "allow-intel-read-aloud-cancel", + "allow-intel-clip-audio", + "allow-page-extras-caret", + "allow-page-extras-link-title", + "allow-attachment-import", + "allow-attachment-open", + "allow-attachment-thumbnail", + "allow-attachment-applied", + "allow-intel-ext-call", + "allow-intel-transcribe", + "allow-intel-transcribe-cancel", + "allow-intel-stream-start", + "allow-intel-stream-stop", + "allow-intel-caption-style", + "allow-connectors-list", + "allow-connectors-connect", + "allow-connectors-cancel", + "allow-connectors-disconnect", + "allow-connectors-request", + "allow-api-call", + "allow-share-take", + "allow-accounts-call" ] } diff --git a/app/src-tauri/capabilities/print.json b/app/src-tauri/capabilities/print.json new file mode 100644 index 00000000..e1960501 --- /dev/null +++ b/app/src-tauri/capabilities/print.json @@ -0,0 +1,7 @@ +{ + "$schema": "../gen/schemas/desktop-schema.json", + "identifier": "print", + "description": "The hidden print window of a PDF export: it may report the page it prepared, and nothing else.", + "windows": ["print-*"], + "permissions": ["core:event:allow-emit"] +} diff --git a/app/src-tauri/examples/notes_harness.rs b/app/src-tauri/examples/notes_harness.rs new file mode 100644 index 00000000..d3ef76d9 --- /dev/null +++ b/app/src-tauri/examples/notes_harness.rs @@ -0,0 +1,78 @@ +//! Runs the notes commands over stdin and stdout, so the notes contract suite in Vitest can run against the real +//! bridge and core (app/src/services/notes/core/contract.test.ts). Each line in is one JSON request, +//! `{ "id": 1, "cmd": "notes_create", "args": { ... } }`, and each line out is a response, `{ "id": 1, "ok": ... }` +//! or `{ "id": 1, "err": { "code": ..., "message": ... } }`, or a notes event, `{ "event": { ... } }`. The command +//! `reset` starts over with a new core, an empty library, and a new notes folder in a temporary folder. + +use std::{ + io::{self, BufRead, Write}, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; + +use opennote_lib::core_bridge::CoreBridge; +use serde_json::{json, Value}; + +type Out = Arc>; + +fn write_line(out: &Out, value: &Value) { + let mut out = out.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _ = writeln!(out, "{value}"); + let _ = out.flush(); +} + +struct Library { + bridge: CoreBridge, + notes: PathBuf, +} + +fn start(base: &Path, n: u32, out: &Out) -> Library { + let dir = base.join(n.to_string()); + let bridge = CoreBridge::at(dir.join("local")); + let events = out.clone(); + bridge.listen(Box::new(move |name, payload| { + if name == "notes:event" { + write_line(&events, &json!({ "event": payload })); + } + })); + Library { + bridge, + notes: dir.join("Notes"), + } +} + +fn main() { + let base = std::env::temp_dir().join(format!("opennote-notes-harness-{}", std::process::id())); + let out: Out = Arc::new(Mutex::new(io::stdout())); + let mut count = 0; + let mut library = start(&base, count, &out); + for line in io::stdin().lock().lines() { + let Ok(line) = line else { break }; + let Ok(request) = serde_json::from_str::(&line) else { + continue; + }; + let id = request.get("id").cloned().unwrap_or(Value::Null); + let command = request + .get("cmd") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(); + let args = request.get("args").cloned().unwrap_or_else(|| json!({})); + if command == "reset" { + library.bridge.shutdown(); + count += 1; + library = start(&base, count, &out); + write_line(&out, &json!({ "id": id, "ok": null })); + continue; + } + let folder = Some(library.notes.clone()); + let result = library.bridge.notes(folder, |bridge| bridge.dispatch(&command, &args)); + let response = match result { + Ok(value) => json!({ "id": id, "ok": value }), + Err(error) => json!({ "id": id, "err": error }), + }; + write_line(&out, &response); + } + library.bridge.shutdown(); + let _ = std::fs::remove_dir_all(&base); +} diff --git a/app/src-tauri/icons/128x128.png b/app/src-tauri/icons/128x128.png index d2337b23..33da665f 100644 Binary files a/app/src-tauri/icons/128x128.png and b/app/src-tauri/icons/128x128.png differ diff --git a/app/src-tauri/icons/128x128@2x.png b/app/src-tauri/icons/128x128@2x.png index a6a5efac..707a7cda 100644 Binary files a/app/src-tauri/icons/128x128@2x.png and b/app/src-tauri/icons/128x128@2x.png differ diff --git a/app/src-tauri/icons/32x32.png b/app/src-tauri/icons/32x32.png index f9419f48..6830f21f 100644 Binary files a/app/src-tauri/icons/32x32.png and b/app/src-tauri/icons/32x32.png differ diff --git a/app/src-tauri/icons/64x64.png b/app/src-tauri/icons/64x64.png index b1ebbdaf..153d8bd0 100644 Binary files a/app/src-tauri/icons/64x64.png and b/app/src-tauri/icons/64x64.png differ diff --git a/app/src-tauri/icons/Square107x107Logo.png b/app/src-tauri/icons/Square107x107Logo.png index 16e30131..6d5ebc33 100644 Binary files a/app/src-tauri/icons/Square107x107Logo.png and b/app/src-tauri/icons/Square107x107Logo.png differ diff --git a/app/src-tauri/icons/Square142x142Logo.png b/app/src-tauri/icons/Square142x142Logo.png index 96140715..ef681a98 100644 Binary files a/app/src-tauri/icons/Square142x142Logo.png and b/app/src-tauri/icons/Square142x142Logo.png differ diff --git a/app/src-tauri/icons/Square150x150Logo.png b/app/src-tauri/icons/Square150x150Logo.png index 42608a88..15f1bf3b 100644 Binary files a/app/src-tauri/icons/Square150x150Logo.png and b/app/src-tauri/icons/Square150x150Logo.png differ diff --git a/app/src-tauri/icons/Square284x284Logo.png b/app/src-tauri/icons/Square284x284Logo.png index bd9057ca..9ebfd16b 100644 Binary files a/app/src-tauri/icons/Square284x284Logo.png and b/app/src-tauri/icons/Square284x284Logo.png differ diff --git a/app/src-tauri/icons/Square30x30Logo.png b/app/src-tauri/icons/Square30x30Logo.png index 887e0edd..f62da3bd 100644 Binary files a/app/src-tauri/icons/Square30x30Logo.png and b/app/src-tauri/icons/Square30x30Logo.png differ diff --git a/app/src-tauri/icons/Square310x310Logo.png b/app/src-tauri/icons/Square310x310Logo.png index 75cb9f79..f4bfd56e 100644 Binary files a/app/src-tauri/icons/Square310x310Logo.png and b/app/src-tauri/icons/Square310x310Logo.png differ diff --git a/app/src-tauri/icons/Square44x44Logo.png b/app/src-tauri/icons/Square44x44Logo.png index 747f5b5b..936b1a29 100644 Binary files a/app/src-tauri/icons/Square44x44Logo.png and b/app/src-tauri/icons/Square44x44Logo.png differ diff --git a/app/src-tauri/icons/Square71x71Logo.png b/app/src-tauri/icons/Square71x71Logo.png index 74646a75..13567e1a 100644 Binary files a/app/src-tauri/icons/Square71x71Logo.png and b/app/src-tauri/icons/Square71x71Logo.png differ diff --git a/app/src-tauri/icons/Square89x89Logo.png b/app/src-tauri/icons/Square89x89Logo.png index 8f169cf7..c65e833a 100644 Binary files a/app/src-tauri/icons/Square89x89Logo.png and b/app/src-tauri/icons/Square89x89Logo.png differ diff --git a/app/src-tauri/icons/StoreLogo.png b/app/src-tauri/icons/StoreLogo.png index 194b9034..120c2c18 100644 Binary files a/app/src-tauri/icons/StoreLogo.png and b/app/src-tauri/icons/StoreLogo.png differ diff --git a/app/src-tauri/icons/icon.icns b/app/src-tauri/icons/icon.icns index 16c128fd..1643ca26 100644 Binary files a/app/src-tauri/icons/icon.icns and b/app/src-tauri/icons/icon.icns differ diff --git a/app/src-tauri/icons/icon.ico b/app/src-tauri/icons/icon.ico index cab76aa2..e193c1bb 100644 Binary files a/app/src-tauri/icons/icon.ico and b/app/src-tauri/icons/icon.ico differ diff --git a/app/src-tauri/icons/icon.png b/app/src-tauri/icons/icon.png index 64c9e0a1..c43400dc 100644 Binary files a/app/src-tauri/icons/icon.png and b/app/src-tauri/icons/icon.png differ diff --git a/app/src-tauri/src/api/approver.rs b/app/src-tauri/src/api/approver.rs new file mode 100644 index 00000000..7ad0f821 --- /dev/null +++ b/app/src-tauri/src/api/approver.rs @@ -0,0 +1,167 @@ +//! Asks the person in the app. A question goes to the interface as an `api://event` of kind `question`. +//! The request waits until App permissions' dialog answers through `api_call("decide")`. +//! Two minutes with no answer counts as no. Questions that arrive while the window is closed wait the same way and are listed again +//! when the interface asks for `pending`. + +use std::{ + collections::HashMap, + sync::{mpsc, Arc, Mutex, PoisonError}, +}; + +use opennote_api::{ + backend::{ApprovalRequest, Approver, Decision}, + routes::APPROVAL_WAIT, +}; +use serde_json::json; + +/// Where questions go: the interface's event channel in the app, a closure in tests. +pub type Ask = Arc; + +#[derive(Default)] +struct Waiting { + answers: HashMap>, + questions: Vec, +} + +type HideFn = Arc; + +/// The app's approver. +pub struct AppApprover { + waiting: Mutex, + show: Mutex>, + hide: Mutex>, +} + +impl Default for AppApprover { + fn default() -> AppApprover { + AppApprover { + waiting: Mutex::default(), + show: Mutex::new(None), + hide: Mutex::new(None), + } + } +} + +impl AppApprover { + /// Sends questions, and the IDs of questions that no longer wait, through these. + pub fn connect(&self, show: Ask, hide: Arc) { + *self.show.lock().unwrap_or_else(PoisonError::into_inner) = Some(show); + *self.hide.lock().unwrap_or_else(PoisonError::into_inner) = Some(hide); + } + + /// The questions waiting for an answer. + pub fn pending(&self) -> Vec { + self.waiting + .lock() + .unwrap_or_else(PoisonError::into_inner) + .questions + .clone() + } + + /// The person's answer. Returns false when the question no longer waits. + pub fn decide(&self, id: &str, decision: Decision) -> bool { + let sender = self + .waiting + .lock() + .unwrap_or_else(PoisonError::into_inner) + .answers + .remove(id); + sender.is_some_and(|sender| sender.send(decision).is_ok()) + } + + fn forget(&self, id: &str) { + let mut waiting = self.waiting.lock().unwrap_or_else(PoisonError::into_inner); + waiting.answers.remove(id); + waiting.questions.retain(|question| question.id != id); + drop(waiting); + if let Some(hide) = self.hide.lock().unwrap_or_else(PoisonError::into_inner).clone() { + hide(id); + } + } +} + +impl Approver for AppApprover { + fn ask(&self, request: ApprovalRequest) -> Decision { + let (sender, receiver) = mpsc::channel(); + { + let mut waiting = self.waiting.lock().unwrap_or_else(PoisonError::into_inner); + waiting.answers.insert(request.id.clone(), sender); + waiting.questions.push(request.clone()); + } + let show = self.show.lock().unwrap_or_else(PoisonError::into_inner).clone(); + let Some(show) = show else { + self.forget(&request.id); + return Decision::Deny; + }; + show(&request); + let decision = receiver.recv_timeout(APPROVAL_WAIT).unwrap_or(Decision::Deny); + self.forget(&request.id); + decision + } +} + +/// The event that shows a question. +pub fn question_event(request: &ApprovalRequest) -> serde_json::Value { + json!({ "kind": "question", "request": request }) +} + +#[cfg(test)] +mod tests { + use std::thread; + + use opennote_api::{backend::Question, Access, AppKind}; + + use super::*; + + fn question(id: &str) -> ApprovalRequest { + ApprovalRequest { + id: id.into(), + app_name: "opennote".into(), + question: Question::Connect { + app_kind: AppKind::Cli, + wants: Access::Read, + }, + } + } + + #[test] + fn a_question_waits_for_the_persons_answer() { + let approver = Arc::new(AppApprover::default()); + let shown = Arc::new(Mutex::new(Vec::new())); + let hidden = Arc::new(Mutex::new(Vec::new())); + { + let (shown, hidden) = (shown.clone(), hidden.clone()); + approver.connect( + Arc::new(move |request| shown.lock().expect("lock").push(request.id.clone())), + Arc::new(move |id| hidden.lock().expect("lock").push(id.to_owned())), + ); + } + let asking = { + let approver = approver.clone(); + thread::spawn(move || approver.ask(question("q1"))) + }; + while approver.pending().is_empty() { + thread::yield_now(); + } + assert!(!approver.decide("other", Decision::Deny)); + assert!(approver.decide( + "q1", + Decision::Allow { + access: None, + notebooks: None, + always: false + } + )); + assert!(matches!(asking.join().expect("answers"), Decision::Allow { .. })); + assert_eq!(*shown.lock().expect("lock"), ["q1"]); + assert_eq!(*hidden.lock().expect("lock"), ["q1"]); + assert!(approver.pending().is_empty()); + } + + #[test] + fn without_a_window_the_answer_is_no() { + let approver = AppApprover::default(); + assert_eq!(approver.ask(question("q2")), Decision::Deny); + assert!(approver.pending().is_empty()); + } +} diff --git a/app/src-tauri/src/api/backend.rs b/app/src-tauri/src/api/backend.rs new file mode 100644 index 00000000..1379f924 --- /dev/null +++ b/app/src-tauri/src/api/backend.rs @@ -0,0 +1,617 @@ +//! The local API's view of the notes, over the same core the interface uses. Reads go through the notebooks' +//! trees and a page's saved state; changes go through a page session of the `api` client, as one transaction each, +//! then a save whose version in page history is named "Added via " or "Changed via ". A page in an +//! encrypted (locked) section is never opened. + +#[cfg(test)] +use std::{path::PathBuf, sync::Arc}; + +use opennote_api::{ + backend::{ + block_summary, headings_of, word_count, Backend, BackendError, HistoryEntry, Location, NewPage, NotebookInfo, + PageInfo, PageInspection, PageText, SearchHit, SectionInfo, + }, + render::Polyline, +}; +use opennote_core::{ + model::notebook::NotebookTree, + ops::resolve::{Edit, TxnRequest}, + session::page::PageHandle, + store::assets::AssetSource, + BlockId, SystemClock, +}; +use serde_json::{json, Value}; +use tauri::{AppHandle, Manager}; + +use super::markdown; +use crate::{ + core_bridge::{Bridge, CoreBridge}, + ipc::{IpcError, IpcResult}, +}; + +/// The page client the API edits as. The interface's own clients are `main-` and the tool windows'. +pub const API_CLIENT: &str = "api"; + +/// The section daily notes live in, as the interface names it (`qolSearch.daily.sectionTitle`). +pub const DAILY_SECTION: &str = "Daily notes"; + +/// The section a share or a clip link lands in when no page is open to take it. +pub const QUICK_SECTION: &str = "Quick notes"; + +/// Where the core is: the app's, or a test's. +#[derive(Clone)] +pub enum Notes { + App(AppHandle), + #[cfg(test)] + Test(Arc, PathBuf), +} + +/// The notes behind the local API. +#[derive(Clone)] +pub struct CoreBackend { + notes: Notes, +} + +fn from_ipc(error: IpcError) -> BackendError { + match error.code.as_str() { + "notFound" => BackendError::NotFound, + "readOnly" => BackendError::ReadOnly, + "locked" => BackendError::Locked, + _ => BackendError::Failed(error.message), + } +} + +impl CoreBackend { + pub fn new(notes: Notes) -> CoreBackend { + CoreBackend { notes } + } + + fn run(&self, work: impl FnOnce(&mut Bridge) -> IpcResult) -> Result { + match &self.notes { + Notes::App(app) => crate::core_bridge::run_notes(app, &app.state::(), work), + #[cfg(test)] + Notes::Test(bridge, folder) => bridge.notes(Some(folder.clone()), work), + } + .map_err(from_ipc) + } + + /// A page's Markdown and its page JSON, for webhooks, which also need its tags and when it was made. The caller + /// checks that the page isn't locked first. + pub fn page_with_json(&self, page: &str) -> Result<(String, Value), BackendError> { + let place = self.locate(page)?; + if place.locked { + return Err(BackendError::Locked); + } + let json = self.run(|bridge| with_page(bridge, page, page_json))?; + Ok((markdown::page_markdown(&place.title, &json), json)) + } + + /// Where a page is, when it is a page and isn't locked. + fn page_place(&self, page: &str) -> Result { + let place = self.locate(page)?; + if place.locked { + return Err(BackendError::Locked); + } + if place.section_id.is_none() || place.notebook_id == place.id || place.section_id.as_deref() == Some(page) { + return Err(BackendError::NotFound); + } + Ok(place) + } + + fn trees(bridge: &Bridge) -> Vec { + bridge + .core + .notebooks() + .into_iter() + .filter(|notebook| !notebook.is_backup()) + .map(|notebook| notebook.tree()) + .collect() + } +} + +/// Where an ID is in the trees. +fn locate_in(trees: &[NotebookTree], id: &str) -> Option { + for tree in trees { + let notebook = tree.notebook.to_string(); + if notebook == id { + return Some(Location { + id: id.to_owned(), + notebook_id: notebook, + section_id: None, + locked: false, + title: tree.title.clone(), + }); + } + for section in &tree.sections { + let section_id = section.id.to_string(); + if section_id == id { + return Some(Location { + id: id.to_owned(), + notebook_id: notebook, + section_id: Some(section_id), + locked: section.encrypted, + title: section.title.clone(), + }); + } + if let Some(page) = section.pages.iter().find(|page| page.id.to_string() == id) { + return Some(Location { + id: id.to_owned(), + notebook_id: notebook, + section_id: Some(section_id), + locked: section.encrypted, + title: page.title.clone(), + }); + } + } + } + None +} + +fn page_info(location: &Location) -> PageInfo { + PageInfo { + id: location.id.clone(), + notebook_id: location.notebook_id.clone(), + section_id: location.section_id.clone().unwrap_or_default(), + title: location.title.clone(), + modified: None, + } +} + +/// The page JSON of an open session. +fn page_json(handle: &PageHandle) -> IpcResult { + let envelope = handle + .envelope(None) + .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string()))?; + let decoded = opennote_core::wire::envelope::decode(&envelope.bytes) + .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string()))?; + serde_json::from_slice(decoded.page_json) + .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string())) +} + +/// Runs `work` on the API's session of a page, and closes the session after. +fn with_page(bridge: &mut Bridge, page: &str, work: impl FnOnce(&PageHandle) -> IpcResult) -> IpcResult { + let handle = bridge.handle(page, API_CLIENT)?; + let result = work(&handle); + if let Err(error) = bridge.close_handle(page, API_CLIENT) { + ::log::warn!("Couldn't close a page session: {}", error.message); + } + result +} + +fn edit_error(error: opennote_core::EditError) -> IpcError { + IpcError::new(error.code(), error.to_string()) +} + +/// Applies `edits` as one transaction, saves, and names the new version in page history. +fn apply_named(handle: &PageHandle, edits: Vec, label: &str) -> IpcResult<()> { + if let Some(reason) = handle.read_only() { + return Err(IpcError::new("readOnly", format!("{reason:?}"))); + } + let request = TxnRequest { + page: handle.id(), + client: handle.client().clone(), + // Each edit has its own session, which starts at sequence 1 and closes after. + client_seq: 1, + coalesce: None, + ui: None, + edits, + }; + handle.apply(request).map_err(edit_error)?; + let saved = handle + .save_now() + .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string()))?; + if let Err(error) = handle.name_version(saved.revision, Some(label.to_owned()), false) { + ::log::warn!("Couldn't name the API's version in page history: {error}"); + } + Ok(()) +} + +fn new_block_id() -> String { + BlockId::generate(&SystemClock::new()).to_string() +} + +/// The edits that add `markdown` and `attachments` after `after`. +fn content_edits(handle: &PageHandle, page: &NewPage, after: Option) -> IpcResult> { + let mut edits = Vec::new(); + let mut after = after; + let mut text = page.markdown.clone(); + if let Some(url) = &page.source_url { + // The address stays a plain link, so it can't run anything when clicked. + text = format!( + "Source: <{url}> + +{text}" + ); + } + if !text.trim().is_empty() { + let id = new_block_id(); + let mut insert = + json!({ "edit": "insertBlock", "block": { "id": id, "type": "text", "data": { "markdown": text } } }); + if let Some(previous) = after.as_ref() { + insert["after"] = json!(previous); + } + edits.push(insert); + after = Some(id); + } + for attachment in &page.attachments { + let image = matches!( + attachment.mime.as_str(), + "image/png" | "image/jpeg" | "image/gif" | "image/webp" + ); + let asset = handle + .import_asset(AssetSource::bytes( + attachment.name.clone(), + attachment.mime.clone(), + attachment.bytes.clone(), + )) + .map_err(|error| IpcError::new(crate::ipc::codes::IO, error.to_string()))?; + let id = new_block_id(); + let width = asset.width.map_or(624.0, |width| f64::from(width).min(624.0)); + let block = if image { + json!({ "id": id, "type": "image", "frame": { "w": width }, "data": { "asset": asset.id.to_string(), "alt": attachment.name } }) + } else { + json!({ "id": id, "type": "file", "data": { "asset": asset.id.to_string() } }) + }; + edits.push(json!({ "edit": "addAsset", "asset": asset.id.to_string() })); + let mut insert = json!({ "edit": "insertBlock", "block": block }); + if let Some(previous) = after.as_ref() { + insert["after"] = json!(previous); + } + edits.push(insert); + after = Some(id); + } + serde_json::from_value(Value::Array(edits)) + .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string())) +} + +/// The ID of the section with this title in the first notebook, made when it isn't there. Refused when it is locked. +fn section_named(bridge: &mut Bridge, trees: &[NotebookTree], title: &str) -> IpcResult { + let tree = trees + .first() + .ok_or_else(|| IpcError::new("notFound", "There are no notebooks yet."))?; + match tree.sections.iter().find(|section| section.title == title) { + Some(section) if section.encrypted => Err(IpcError::new("locked", format!("{title} is locked."))), + Some(section) => Ok(section.id.to_string()), + None => { + let input = json!({ + "kind": "section", + "placement": { "parentId": tree.notebook.to_string(), "beforeId": null }, + "title": title, + }); + let node = bridge.dispatch("notes_create", &json!({ "input": input }))?; + Ok(node["id"].as_str().unwrap_or_default().to_owned()) + } + } +} + +impl CoreBackend { + /// Makes a page in Quick notes, in the first notebook: where a share or a clip link goes when no page can + /// take it. The page history names `author`. + pub fn quick_note(&self, page: NewPage, author: &str) -> Result { + let section = self.run(|bridge| { + let trees = Self::trees(bridge); + section_named(bridge, &trees, QUICK_SECTION) + })?; + self.create_page(§ion, page, author) + } +} + +/// Today in local time, as the daily note's title `YYYY-MM-DD`. +pub fn local_today() -> String { + #[cfg(windows)] + { + use windows::Win32::{ + Foundation::{FILETIME, SYSTEMTIME}, + System::Time::{FileTimeToSystemTime, SystemTimeToTzSpecificLocalTime}, + }; + let since = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + // FILETIME counts 100 ns ticks since 1601. + let ticks = since.as_nanos() / 100 + 116_444_736_000_000_000; + let ticks = u64::try_from(ticks).unwrap_or(0); + let file_time = FILETIME { + dwLowDateTime: (ticks & 0xffff_ffff) as u32, + dwHighDateTime: (ticks >> 32) as u32, + }; + let (mut utc, mut local) = (SYSTEMTIME::default(), SYSTEMTIME::default()); + // SAFETY: both structures are valid for the calls, which only write them. + let ok = unsafe { + FileTimeToSystemTime(&file_time, &mut utc).is_ok() + && SystemTimeToTzSpecificLocalTime(None, &utc, &mut local).is_ok() + }; + if ok { + return format!("{:04}-{:02}-{:02}", local.wYear, local.wMonth, local.wDay); + } + } + let days = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_secs() / 86_400) + .unwrap_or(0); + let (year, month, day) = civil(i64::try_from(days).unwrap_or(0)); + format!("{year:04}-{month:02}-{day:02}") +} + +fn civil(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + (yoe + era * 400 + i64::from(month <= 2), month, day) +} + +impl Backend for CoreBackend { + fn notebooks(&self) -> Result, BackendError> { + self.run(|bridge| { + Ok(Self::trees(bridge) + .into_iter() + .map(|tree| NotebookInfo { + id: tree.notebook.to_string(), + title: tree.title, + }) + .collect()) + }) + } + + fn sections(&self, notebook: &str) -> Result, BackendError> { + let trees = self.run(|bridge| Ok(Self::trees(bridge)))?; + let tree = trees + .iter() + .find(|tree| tree.notebook.to_string() == notebook) + .ok_or(BackendError::NotFound)?; + Ok(tree + .sections + .iter() + .filter(|section| !section.archived) + .map(|section| SectionInfo { + id: section.id.to_string(), + notebook_id: notebook.to_owned(), + title: section.title.clone(), + locked: section.encrypted, + }) + .collect()) + } + + fn pages(&self, section: &str) -> Result, BackendError> { + let trees = self.run(|bridge| Ok(Self::trees(bridge)))?; + for tree in &trees { + if let Some(found) = tree.sections.iter().find(|each| each.id.to_string() == section) { + if found.encrypted { + return Err(BackendError::Locked); + } + return Ok(found + .pages + .iter() + .map(|page| PageInfo { + id: page.id.to_string(), + notebook_id: tree.notebook.to_string(), + section_id: section.to_owned(), + title: page.title.clone(), + modified: None, + }) + .collect()); + } + } + Err(BackendError::NotFound) + } + + fn locate(&self, id: &str) -> Result { + let trees = self.run(|bridge| Ok(Self::trees(bridge)))?; + locate_in(&trees, id).ok_or(BackendError::NotFound) + } + + fn read_page(&self, page: &str) -> Result { + let place = self.page_place(page)?; + let json = self.run(|bridge| with_page(bridge, page, page_json))?; + Ok(PageText { + markdown: markdown::page_markdown(&place.title, &json), + page: PageInfo { + modified: json["modified"].as_str().map(str::to_owned), + ..page_info(&place) + }, + }) + } + + fn inspect(&self, page: &str) -> Result { + let place = self.page_place(page)?; + let (json, history) = self.run(|bridge| { + with_page(bridge, page, |handle| { + let json = page_json(handle)?; + let history = handle + .history() + .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string()))?; + Ok((json, history)) + }) + })?; + let markdown = markdown::page_markdown(&place.title, &json); + let blocks = json["blocks"] + .as_array() + .into_iter() + .flatten() + .map(|block| { + let data = &block["data"]; + let text = data["markdown"] + .as_str() + .or_else(|| data["alt"].as_str()) + .unwrap_or_default(); + block_summary( + block["id"].as_str().unwrap_or_default(), + block["type"].as_str().unwrap_or_default(), + text, + ) + }) + .collect(); + Ok(PageInspection { + blocks, + headings: headings_of(&markdown), + words: word_count(&markdown), + history: history + .into_iter() + .take(10) + .map(|entry| HistoryEntry { + label: entry.name.unwrap_or_default(), + saved: entry.saved_at.to_rfc3339(), + }) + .collect(), + page: PageInfo { + modified: json["modified"].as_str().map(str::to_owned), + ..page_info(&place) + }, + }) + } + + fn ink(&self, page: &str) -> Result, BackendError> { + self.page_place(page)?; + let lines = self.run(|bridge| with_page(bridge, page, |handle| Ok(handle.ink_lines())))?; + Ok(lines + .into_iter() + .map(|line| Polyline { + points: line.points, + rgba: line.color, + width: line.width, + }) + .collect()) + } + + fn create_page(&self, section: &str, page: NewPage, author: &str) -> Result { + let place = self.locate(section)?; + if place.locked { + return Err(BackendError::Locked); + } + if place.section_id.as_deref() != Some(section) || place.id != section { + return Err(BackendError::Invalid("Pages go into a section.".into())); + } + let label = format!("Added via {author}"); + let id = self.run(|bridge| { + let input = + json!({ "kind": "page", "placement": { "parentId": section, "beforeId": null }, "title": page.title }); + let node = bridge.dispatch("notes_create", &json!({ "input": input }))?; + let id = node["id"].as_str().unwrap_or_default().to_owned(); + with_page(bridge, &id, |handle| { + let edits = content_edits(handle, &page, None)?; + if edits.is_empty() { + let saved = handle + .save_now() + .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string()))?; + let _ = handle.name_version(saved.revision, Some(label.clone()), false); + return Ok(()); + } + apply_named(handle, edits, &label) + })?; + Ok(id) + })?; + self.locate(&id).map(|place| page_info(&place)) + } + + fn append(&self, page: &str, markdown: &str, author: &str) -> Result { + let place = self.page_place(page)?; + let label = format!("Changed via {author}"); + self.run(|bridge| { + with_page(bridge, page, |handle| { + let after = markdown::last_block(&page_json(handle)?); + let content = NewPage { + markdown: markdown.to_owned(), + ..NewPage::default() + }; + apply_named(handle, content_edits(handle, &content, after)?, &label) + }) + })?; + Ok(page_info(&place)) + } + + fn append_daily(&self, markdown: &str, author: &str) -> Result { + let title = local_today(); + let page = self.run(|bridge| { + let trees = Self::trees(bridge); + let section = section_named(bridge, &trees, DAILY_SECTION)?; + let existing = trees + .iter() + .flat_map(|tree| tree.sections.iter()) + .find(|each| each.id.to_string() == section) + .and_then(|each| each.pages.iter().find(|page| page.title == title)) + .map(|page| page.id.to_string()); + match existing { + Some(id) => Ok(id), + None => { + let input = json!({ "kind": "page", "placement": { "parentId": section, "beforeId": null }, "title": title }); + let node = bridge.dispatch("notes_create", &json!({ "input": input }))?; + Ok(node["id"].as_str().unwrap_or_default().to_owned()) + } + } + }); + self.append(&page?, markdown, author) + } + + fn daily_target(&self) -> Result { + let trees = self.run(|bridge| Ok(Self::trees(bridge)))?; + let tree = trees.first().ok_or(BackendError::NotFound)?; + let section = tree.sections.iter().find(|section| section.title == DAILY_SECTION); + Ok(Location { + id: section.map_or_else(|| tree.notebook.to_string(), |section| section.id.to_string()), + notebook_id: tree.notebook.to_string(), + section_id: section.map(|section| section.id.to_string()), + locked: section.is_some_and(|section| section.encrypted), + title: format!("{DAILY_SECTION}, {}", local_today()), + }) + } + + fn search(&self, text: &str, notebooks: Option<&[String]>, limit: usize) -> Result, BackendError> { + #[allow(irrefutable_let_patterns)] // the test notes are a second kind in test builds only + let Notes::App(app) = &self.notes + else { + return Ok(Vec::new()); + }; + let bridge = app.state::(); + let found = bridge + .search_query(match notebooks { + Some(ids) => json!({ "text": text, "notebooks": ids, "limit": limit.min(100) }), + None => json!({ "text": text, "limit": limit.min(100) }), + }) + .map_err(from_ipc)?; + let trees = self.run(|bridge| Ok(Self::trees(bridge)))?; + let hits = found["hits"].as_array().cloned().unwrap_or_default(); + Ok(hits + .iter() + .filter_map(|hit| { + let id = hit["page"].as_str()?; + let place = locate_in(&trees, id)?; + (!place.locked).then(|| SearchHit { + page: page_info(&place), + snippet: hit["snippet"]["text"] + .as_str() + .unwrap_or_default() + .chars() + .take(300) + .collect(), + }) + }) + .collect()) + } + + fn backup(&self) -> Result { + #[allow(irrefutable_let_patterns)] // the test notes are a second kind in test builds only + let Notes::App(app) = &self.notes + else { + return Err(BackendError::Failed("Backups need the app.".into())); + }; + let record = crate::shellqol::backup::run(app).map_err(from_ipc)?; + if record["running"].as_bool() == Some(true) { + return Err(BackendError::Invalid("A backup is already running.".into())); + } + if let Some(error) = record["error"].as_str() { + return Err(BackendError::Failed(error.to_owned())); + } + Ok(format!( + "Backed up {} notebooks to the backup folder in Settings.", + record["notebooks"].as_u64().unwrap_or(0) + )) + } +} + +#[cfg(test)] +#[path = "backend_tests.rs"] +mod tests; diff --git a/app/src-tauri/src/api/backend_tests.rs b/app/src-tauri/src/api/backend_tests.rs new file mode 100644 index 00000000..67e8b945 --- /dev/null +++ b/app/src-tauri/src/api/backend_tests.rs @@ -0,0 +1,202 @@ +//! The local API's backend over a real core in a temporary folder: listing, reading as Markdown, adding a page with +//! its source link, adding to a page, and the "via " names in page history. + +use std::sync::Arc; + +use serde_json::json; + +use super::*; +use crate::core_bridge::CoreBridge; + +struct Fixture { + _dir: tempfile::TempDir, + bridge: Arc, + notes: PathBuf, + notebook: String, + section: String, + page: String, +} + +fn fixture() -> Fixture { + let dir = tempfile::tempdir().expect("a temp folder"); + let (local, notes) = (dir.path().join("local"), dir.path().join("Notes")); + let bridge = Arc::new(CoreBridge::at(local)); + let (notebook, section, page) = bridge + .notes(Some(notes.clone()), |bridge| { + let create = |bridge: &mut Bridge, kind: &str, parent: Option, title: &str| { + let input = + json!({ "kind": kind, "placement": { "parentId": parent, "beforeId": null }, "title": title }); + let node = bridge.dispatch("notes_create", &json!({ "input": input }))?; + Ok::<_, IpcError>(node["id"].as_str().unwrap_or_default().to_owned()) + }; + let notebook = create(bridge, "notebook", None, "Biology")?; + let section = create(bridge, "section", Some(notebook.clone()), "Cells")?; + let page = create(bridge, "page", Some(section.clone()), "Membranes")?; + Ok((notebook, section, page)) + }) + .expect("a notebook"); + Fixture { + _dir: dir, + bridge, + notes, + notebook, + section, + page, + } +} + +fn backend(fixture: &Fixture) -> CoreBackend { + CoreBackend::new(Notes::Test(fixture.bridge.clone(), fixture.notes.clone())) +} + +fn history_names(fixture: &Fixture, page: &str) -> Vec { + fixture + .bridge + .notes(Some(fixture.notes.clone()), |bridge| { + let handle = bridge.handle(page, "history-check")?; + let names = handle + .history() + .map_err(|error| IpcError::new("io", error.to_string()))? + .into_iter() + .filter_map(|entry| entry.name) + .collect(); + bridge.close_handle(page, "history-check").expect("closes"); + Ok(names) + }) + .expect("history") +} + +#[test] +fn lists_the_notebook_its_sections_and_pages() { + let fixture = fixture(); + let api = backend(&fixture); + let notebooks = api.notebooks().expect("notebooks"); + assert!(notebooks + .iter() + .any(|notebook| notebook.id == fixture.notebook && notebook.title == "Biology")); + let sections = api.sections(&fixture.notebook).expect("sections"); + assert_eq!(sections.len(), 1); + assert_eq!(sections[0].title, "Cells"); + assert!(!sections[0].locked); + let pages = api.pages(&fixture.section).expect("pages"); + assert_eq!( + pages.iter().map(|page| page.title.as_str()).collect::>(), + ["Membranes"] + ); + let place = api.locate(&fixture.page).expect("located"); + assert_eq!(place.notebook_id, fixture.notebook); + assert_eq!(place.section_id.as_deref(), Some(fixture.section.as_str())); + assert_eq!( + api.locate("01k6f00000000000000000zzzz").unwrap_err(), + BackendError::NotFound + ); +} + +#[test] +fn adds_a_page_with_its_source_and_names_it_in_page_history() { + let fixture = fixture(); + let api = backend(&fixture); + let created = api + .create_page( + &fixture.section, + NewPage { + title: "Clipped".into(), + markdown: "Cell walls are rigid.".into(), + source_url: Some("https://example.org/cells".into()), + attachments: Vec::new(), + }, + "Web clipper", + ) + .expect("created"); + assert_eq!(created.title, "Clipped"); + let text = api.read_page(&created.id).expect("read"); + assert!(text.markdown.starts_with("# Clipped"), "{}", text.markdown); + assert!( + text.markdown.contains("Source: "), + "{}", + text.markdown + ); + assert!(text.markdown.contains("Cell walls are rigid.")); + assert_eq!(history_names(&fixture, &created.id), ["Added via Web clipper"]); +} + +#[test] +fn adds_to_the_end_of_a_page_as_one_named_step() { + let fixture = fixture(); + let api = backend(&fixture); + api.append(&fixture.page, "First line.", "opennote").expect("appended"); + api.append(&fixture.page, "Second line.", "opennote") + .expect("appended again"); + let text = api.read_page(&fixture.page).expect("read").markdown; + let (first, second) = ( + text.find("First line.").expect("first"), + text.find("Second line.").expect("second"), + ); + assert!(first < second, "{text}"); + assert_eq!( + history_names(&fixture, &fixture.page), + ["Changed via opennote", "Changed via opennote"] + ); +} + +#[test] +fn refuses_to_put_a_page_anywhere_but_a_section() { + let fixture = fixture(); + let api = backend(&fixture); + let page = NewPage { + title: "Nope".into(), + ..NewPage::default() + }; + assert!(matches!( + api.create_page(&fixture.notebook, page.clone(), "x"), + Err(BackendError::Invalid(_)) + )); + assert!(matches!( + api.create_page(&fixture.page, page, "x"), + Err(BackendError::Invalid(_)) + )); +} + +#[test] +fn the_daily_note_is_made_once_and_added_to() { + let fixture = fixture(); + let api = backend(&fixture); + let first = api.append_daily("Met with Sam.", "opennote").expect("daily"); + let again = api.append_daily("Then lunch.", "opennote").expect("daily again"); + assert_eq!(first.id, again.id); + assert_eq!(first.title, local_today()); + let text = api.read_page(&first.id).expect("read").markdown; + assert!(text.contains("Met with Sam.") && text.contains("Then lunch."), "{text}"); + let target = api.daily_target().expect("target"); + assert_eq!(target.section_id.as_deref(), Some(first.section_id.as_str())); +} + +#[test] +fn inspect_and_ink_read_back_a_change_and_who_made_it() { + let fixture = fixture(); + let api = backend(&fixture); + api.append( + &fixture.page, + "## Cristae\n\nFolds inside.", + "opennote (Claude (copyedit))", + ) + .expect("appended"); + let inspected = api.inspect(&fixture.page).expect("inspected"); + assert!( + inspected.headings.iter().any(|heading| heading == "## Cristae"), + "{:?}", + inspected.headings + ); + assert!(inspected + .blocks + .iter() + .any(|block| block.kind == "text" && block.first_line == "## Cristae")); + assert!(inspected.words >= 3); + assert_eq!( + inspected.history.first().map(|entry| entry.label.as_str()), + Some("Changed via opennote (Claude (copyedit))") + ); + assert!(api.ink(&fixture.page).expect("ink").is_empty(), "no strokes yet"); + assert!(matches!(api.inspect(&fixture.section), Err(BackendError::NotFound))); + assert!(matches!(api.ink(&fixture.notebook), Err(BackendError::NotFound))); +} diff --git a/app/src-tauri/src/api/markdown.rs b/app/src-tauri/src/api/markdown.rs new file mode 100644 index 00000000..bcb17f72 --- /dev/null +++ b/app/src-tauri/src/api/markdown.rs @@ -0,0 +1,161 @@ +//! A page as Markdown, for the local API: the text blocks as they are, and a short line in italics for each part +//! that has no text of its own, such as a picture or a drawing. A block of a newer type uses its fallback Markdown. + +use serde_json::Value; + +/// The page JSON's blocks in page order. +fn blocks_in_order(page: &Value) -> Vec<&Value> { + let mut blocks: Vec<&Value> = page["blocks"] + .as_array() + .map(|blocks| blocks.iter().collect()) + .unwrap_or_default(); + blocks.sort_by(|a, b| a["order"].as_str().unwrap_or("").cmp(b["order"].as_str().unwrap_or(""))); + blocks +} + +/// One block as Markdown, or `None` when it adds nothing to read. +fn block_markdown(block: &Value) -> Option { + let data = &block["data"]; + let text = match block["type"].as_str().unwrap_or("") { + "text" => data["markdown"].as_str().unwrap_or("").to_owned(), + "image" => match data["alt"].as_str().filter(|alt| !alt.trim().is_empty()) { + Some(alt) => format!("*[Picture: {}]*", alt.trim()), + None => "*[Picture]*".to_owned(), + }, + "file" => "*[Attached file]*".to_owned(), + "table" => table_markdown(data)?, + "ink" => { + let strokes = data["strokeCount"].as_u64().unwrap_or(0); + if strokes == 0 { + return None; + } + "*[Drawing]*".to_owned() + } + other => match block["fallback"]["markdown"].as_str() { + Some(fallback) => fallback.to_owned(), + None => format!("*[{}]*", other.trim_start_matches("ext:")), + }, + }; + let text = text.trim_end().to_owned(); + (!text.trim().is_empty()).then_some(text) +} + +/// A table as a Markdown pipe table, so a reader through the API sees its cells. Without a header row, the +/// table gets an empty one, because a pipe table needs one. `None` when it has no cells. +fn table_markdown(data: &Value) -> Option { + let columns: Vec<&str> = data["columns"] + .as_array()? + .iter() + .filter_map(|column| column["id"].as_str()) + .collect(); + let rows: Vec> = data["rows"] + .as_array()? + .iter() + .map(|row| { + columns + .iter() + .map(|column| { + let text = row["cells"][column]["markdown"].as_str().unwrap_or(""); + text.trim() + .replace('|', r"\|") + .replace("\r\n", "\n") + .replace('\n', "
") + }) + .collect() + }) + .collect(); + if columns.is_empty() || rows.is_empty() { + return None; + } + let line = |cells: &[String]| format!("| {} |", cells.join(" | ")); + let mut rows = rows.into_iter(); + let header = if data["header"] == true { + rows.next().unwrap_or_default() + } else { + vec![String::new(); columns.len()] + }; + let mut lines = vec![line(&header), line(&vec!["---".to_owned(); columns.len()])]; + lines.extend(rows.map(|row| line(&row))); + Some(lines.join("\n")) +} + +/// The page's title as a heading, then its blocks. +pub fn page_markdown(title: &str, page: &Value) -> String { + let mut parts = vec![format!("# {}", title.trim())]; + parts.extend(blocks_in_order(page).into_iter().filter_map(block_markdown)); + parts.join("\n\n") +} + +/// The ID of the page's last block, for adding after it. +pub fn last_block(page: &Value) -> Option { + blocks_in_order(page) + .into_iter() + .rev() + .find(|block| block["type"].as_str() != Some("ink")) + .and_then(|block| block["id"].as_str().map(str::to_owned)) +} + +/// The page's tags. +pub fn tags(page: &Value) -> Vec { + page["tags"] + .as_array() + .map(|tags| tags.iter().filter_map(|tag| tag.as_str().map(str::to_owned)).collect()) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn reads_text_and_names_the_rest() { + let page = json!({ + "tags": ["biology"], + "blocks": [ + { "id": "b3", "type": "image", "order": "a2", "data": { "alt": "A leaf" } }, + { "id": "b1", "type": "text", "order": "a0", "data": { "markdown": "## Light\n\nWater splits.\n" } }, + { "id": "b2", "type": "ink", "order": "a1", "data": { "role": "layer", "strokeCount": 0 } }, + { "id": "b4", "type": "ext:org.example/kanban", "order": "a3", "data": {}, + "fallback": { "markdown": "**Kanban board**" } }, + { "id": "b5", "type": "ext:org.example/thing", "order": "a4", "data": {} }, + ] + }); + assert_eq!( + page_markdown("Photosynthesis", &page), + "# Photosynthesis\n\n## Light\n\nWater splits.\n\n*[Picture: A leaf]*\n\n**Kanban board**\n\n*[org.example/thing]*" + ); + assert_eq!(last_block(&page).as_deref(), Some("b5")); + assert_eq!(tags(&page), ["biology"]); + } + + #[test] + fn tables_come_out_as_pipe_tables() { + let cell = |text: &str| json!({ "markdown": text }); + let page = json!({ "blocks": [{ "id": "t", "type": "table", "order": "a0", "data": { + "header": true, + "columns": [{ "id": "c1" }, { "id": "c2" }], + "rows": [ + { "id": "r1", "cells": { "c1": cell("Item"), "c2": cell("Cost") } }, + { "id": "r2", "cells": { "c1": cell("Pens | ink"), "c2": cell("4") } }, + { "id": "r3", "cells": { "c1": cell("Paper") } }, + ], + } }] }); + assert_eq!( + page_markdown("Budget", &page), + "# Budget\n\n| Item | Cost |\n| --- | --- |\n| Pens \\| ink | 4 |\n| Paper | |" + ); + let plain = json!({ "blocks": [{ "id": "t", "type": "table", "order": "a0", "data": { + "columns": [{ "id": "c1" }], + "rows": [{ "id": "r1", "cells": { "c1": cell("a") } }], + } }] }); + assert_eq!(page_markdown("T", &plain), "# T\n\n| |\n| --- |\n| a |"); + } + + #[test] + fn an_empty_page_is_its_title() { + assert_eq!(page_markdown(" Empty ", &json!({ "blocks": [] })), "# Empty"); + assert_eq!(last_block(&json!({})), None); + } +} diff --git a/app/src-tauri/src/api/mod.rs b/app/src-tauri/src/api/mod.rs new file mode 100644 index 00000000..58ad6268 --- /dev/null +++ b/app/src-tauri/src/api/mod.rs @@ -0,0 +1,609 @@ +//! The local API in the app (crates/api; docs/help/local-api.md). With `api.local` on and "Let apps on this PC +//! connect" on in Settings, Privacy, App permissions, it listens on a port of `127.0.0.1` and on a named pipe of this +//! profile, and nowhere else. Off means no listener at all. +//! +//! - Grants and webhooks are in `api.json` beside the settings. +//! Token hashes and webhook secrets are in Windows Credential Manager, through the connectors' credential store. +//! - Where it listens goes to `api-endpoint.json` in the local data folder, and the key that proves the listener is +//! OpenNote's goes to the credential store, for the `opennote` tool. Both are removed when it stops. +//! - Questions for the person go to the interface as `api://event` events; App permissions answers them with +//! `api_call("decide")`. +//! - Every access is in `api-access.log` in the log folder. +//! +//! One command, `api_call`, carries every method, like `shellqol_call`. + +pub mod approver; +pub mod backend; +pub mod markdown; +pub mod phone; +pub mod webhooks; + +use std::{ + path::PathBuf, + sync::{Arc, Mutex, OnceLock, PoisonError}, + time::Instant, +}; + +use opennote_api::{ + access_log::AccessLog, + backend::Decision, + client::{write_discovery, write_key_file, Discovery, DISCOVERY_FILE, KEY_FILE, PROOF_TARGET}, + grants::{random_bytes, Access, ConfigFile, Grants, Scope, SecretStore}, + hmac::hex, + routes::{Api, Changed}, + server::Server, + webhooks::{clean_hook, secret_target, Dispatcher, Webhook, BACKOFF, MAX_HOOKS}, +}; +use serde::Deserialize; +use serde_json::{json, Value}; +use tauri::{AppHandle, Emitter, Manager, State}; + +use self::{ + approver::{question_event, AppApprover}, + backend::{CoreBackend, Notes}, +}; +use crate::{ + connectors::Secret, + ipc::{IpcError, IpcResult}, + paths::Paths, + platform_flags, +}; + +/// The event channel of the local API. +pub const EVENT: &str = "api://event"; + +/// The connectors' credential store, as the API crate's [`SecretStore`]. +pub struct Credentials(pub Arc); + +#[allow(clippy::result_unit_err)] +impl SecretStore for Credentials { + fn put(&self, target: &str, secret: &str) -> Result<(), ()> { + self.0.put(target, &Secret::new(secret)).map_err(|_| ()) + } + + fn get(&self, target: &str) -> Result, ()> { + self.0 + .get(target) + .map(|found| found.map(|secret| secret.expose().to_owned())) + .map_err(|_| ()) + } + + fn delete(&self, target: &str) -> Result<(), ()> { + self.0.delete(target).map_err(|_| ()) + } +} + +/// The managed state of the local API. +pub struct ApiState { + pub grants: Arc, + pub log: Arc, + pub approver: Arc, + secrets: Arc, + api: OnceLock>, + server: Mutex>, + discovery: PathBuf, + /// The per-instance key file (`api-instance.key`), for clients without the credential store's copy. + key_file: PathBuf, + pipe: String, + /// The webhook sender and the watcher of saves, made at start-up when the flag is on. + hooks: OnceLock<(Arc, Mutex>)>, + /// Insert > From phone (flag integrations.phoneScan). + pub phone: phone::PhoneState, +} + +impl ApiState { + pub fn new(paths: &Paths) -> ApiState { + let secrets: Arc = Arc::new(Credentials(crate::connectors::platform_store())); + ApiState { + grants: Arc::new(Grants::new( + ConfigFile::at(&paths.roaming.join("api.json")), + secrets.clone(), + )), + log: Arc::new(AccessLog::at(&paths.logs.join("api-access.log"))), + approver: Arc::default(), + secrets, + api: OnceLock::new(), + server: Mutex::new(None), + discovery: paths.local.join(DISCOVERY_FILE), + key_file: paths.local.join(KEY_FILE), + pipe: format!("OpenNote-api-{}", paths.profile_key()), + hooks: OnceLock::new(), + phone: phone::PhoneState::default(), + } + } + + fn api(&self, app: &AppHandle) -> Arc { + self.api + .get_or_init(|| { + let notify_app = app.clone(); + let api = Api::new( + self.grants.clone(), + Arc::new(CoreBackend::new(Notes::App(app.clone()))), + self.approver.clone(), + self.log.clone(), + Vec::new(), + ) + .on_change(move |what| { + let what = match what { + Changed::Grants => "grants", + Changed::Log => "log", + }; + let _ = notify_app.emit(EVENT, json!({ "kind": "changed", "what": what })); + }); + Arc::new(api) + }) + .clone() + } + + fn running(&self) -> Option<(u16, Option)> { + self.server + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .map(|server| (server.port(), server.pipe().map(str::to_owned))) + } + + /// Starts the listeners, unless they run already. + fn start(&self, app: &AppHandle) -> IpcResult<()> { + let mut server = self.server.lock().unwrap_or_else(PoisonError::into_inner); + if server.is_some() { + return Ok(()); + } + // A new key for each start: a client checks it before it sends a token. + let key = random_bytes(32); + let api = self.api(app); + api.set_proof_key(key.clone()); + let port = self.grants.config().port; + let started = + Server::start(api, port, Some(&self.pipe)).map_err(|error| IpcError::new("io", error.to_string()))?; + if started.port() != port { + let chosen = started.port(); + let _ = self.grants.change(|config| config.port = chosen); + } + if self.secrets.put(PROOF_TARGET, &hex(&key)).is_err() { + ::log::warn!("Couldn't store the local API's key in the credential store."); + } + if let Err(error) = write_key_file(&self.key_file, &key) { + ::log::warn!("Couldn't write the local API's key file: {error}"); + } + let discovery = Discovery { + port: started.port(), + pipe: started.pipe().map(str::to_owned), + }; + if let Err(error) = write_discovery(&self.discovery, &discovery) { + ::log::warn!("Couldn't write where the local API listens: {error}"); + } + ::log::info!("The local API listens on 127.0.0.1:{}.", started.port()); + *server = Some(started); + Ok(()) + } + + /// Starts the webhook sender and the watcher of saves. + fn start_hooks(&self, app: &AppHandle) { + let notify_app = app.clone(); + let dispatcher = Arc::new(Dispatcher::start( + self.grants.clone(), + self.secrets.clone(), + self.log.clone(), + Arc::new(webhooks::HttpsPoster), + BACKOFF.to_vec(), + move || { + let _ = notify_app.emit(EVENT, json!({ "kind": "changed", "what": "log" })); + }, + )); + let watch = webhooks::start_watcher(CoreBackend::new(Notes::App(app.clone())), dispatcher.clone()); + if self.hooks.set((dispatcher, Mutex::new(watch))).is_err() { + return; + } + let state = app.clone(); + app.state::() + .on_saved(Box::new(move |page| { + let api = state.state::(); + // Pages are read only when a hook would hear about them. + if !api.grants.config().webhooks.iter().any(|hook| hook.enabled) { + return; + } + if let Some((_, watch)) = api.hooks.get() { + let _ = watch + .lock() + .unwrap_or_else(PoisonError::into_inner) + .send(webhooks::Watch::Saved(page.to_owned())); + } + })); + } + + fn dispatcher(&self) -> IpcResult> { + self.hooks + .get() + .map(|(dispatcher, _)| dispatcher.clone()) + .ok_or_else(|| IpcError::new("notRunning", "Webhooks aren't part of this build.")) + } + + /// Stops the listeners and removes what tells clients where they were. + pub fn stop(&self) { + let server = self.server.lock().unwrap_or_else(PoisonError::into_inner).take(); + if let Some(server) = server { + server.stop(); + let _ = std::fs::remove_file(&self.discovery); + let _ = std::fs::remove_file(&self.key_file); + let _ = self.secrets.delete(PROOF_TARGET); + ::log::info!("The local API stopped."); + } + } +} + +/// Whether the API may run: the flag, and the person's choice. +fn wanted(app: &AppHandle, state: &ApiState) -> bool { + platform_flags::is_on_for(app, platform_flags::API_LOCAL) && state.grants.config().enabled +} + +/// Connects the approver to the interface and starts the listeners when they're wanted. Runs at start-up. +pub fn start(app: &AppHandle) { + let state = app.state::(); + let (show, hide) = (app.clone(), app.clone()); + state.approver.connect( + Arc::new(move |request| { + let _ = show.emit(EVENT, question_event(request)); + }), + Arc::new(move |id| { + let _ = hide.emit(EVENT, json!({ "kind": "answered", "id": id })); + }), + ); + if platform_flags::is_on_for(app, platform_flags::API_LOCAL) { + state.start_hooks(app); + } + if !wanted(app, &state) { + return; + } + if let Err(error) = state.start(app) { + ::log::warn!("The local API didn't start: {}", error.message); + } +} + +/// The command `opennote mcp` runs as, for "Copy MCP config": the tool in the app's `bin` folder. +fn tool_path() -> PathBuf { + let exe = std::env::current_exe().unwrap_or_default(); + let folder = exe.parent().map(PathBuf::from).unwrap_or_default(); + folder + .join(crate::install::path_entry::CLI_DIR) + .join(crate::install::path_entry::CLI_EXE) +} + +/// The MCP configuration an assistant such as Claude Desktop reads, with the tool's full path. An author, when +/// given, is passed as `--author` so page history says which assistant made each change. It is one line of at +/// most 60 characters; `opennote mcp` and the local API clean it again before it reaches a page. +pub fn mcp_config(tool: &std::path::Path, author: Option<&str>) -> String { + let mut args = vec!["mcp".to_owned()]; + let author: String = author + .unwrap_or_default() + .chars() + .filter(|c| !c.is_control()) + .take(60) + .collect(); + if !author.trim().is_empty() { + args.push("--author".to_owned()); + args.push(author.trim().to_owned()); + } + let config = json!({ + "mcpServers": { + "opennote": { + "command": tool.display().to_string(), + "args": args + } + } + }); + serde_json::to_string_pretty(&config).unwrap_or_default() +} + +fn arg(args: &Value, name: &str) -> IpcResult { + serde_json::from_value(args.get(name).cloned().unwrap_or(Value::Null)) + .map_err(|error| IpcError::invalid(name, &error.to_string())) +} + +fn io(error: std::io::Error) -> IpcError { + IpcError::new("io", error.to_string()) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct AppChange { + id: String, + access: Access, + notebooks: Scope, + ask_before_writes: bool, +} + +/// One method of App permissions. +// checks-disable-next-line modifiability: one match over the command names, whose arms share its state; split it when it grows again +pub fn call(app: &AppHandle, state: &ApiState, method: &str, args: &Value) -> IpcResult { + let changed = |what: &str| { + let _ = app.emit(EVENT, json!({ "kind": "changed", "what": what })); + }; + match method { + "status" => { + let running = state.running(); + Ok(json!({ + "available": platform_flags::is_on_for(app, platform_flags::API_LOCAL), + "enabled": state.grants.config().enabled, + "running": running.is_some(), + "port": running.as_ref().map(|(port, _)| *port), + "pipe": running.and_then(|(_, pipe)| pipe), + })) + } + "setEnabled" => { + let enabled: bool = arg(args, "enabled")?; + state.grants.change(|config| config.enabled = enabled).map_err(io)?; + if enabled && wanted(app, state) { + state.start(app)?; + } else { + state.stop(); + } + changed("status"); + call(app, state, "status", &Value::Null) + } + "apps" => Ok(json!(state.grants.apps())), + "updateApp" => { + let change: AppChange = + serde_json::from_value(args.clone()).map_err(|error| IpcError::invalid("app", &error.to_string()))?; + let found = state + .grants + .update(&change.id, change.access, change.notebooks, change.ask_before_writes) + .map_err(io)?; + changed("grants"); + Ok(json!(found)) + } + "revoke" => { + let id: String = arg(args, "id")?; + let found = state.grants.revoke(&id).map_err(io)?; + state.log.add(opennote_api::Entry::new( + &id, + "", + "revoke", + opennote_api::Outcome::Allowed, + )); + changed("grants"); + Ok(json!(found)) + } + "log" => { + let limit: Option = arg(args, "limit")?; + Ok(json!(state.log.recent(limit.unwrap_or(200).min(500)))) + } + "clearLog" => { + state.log.clear(); + changed("log"); + Ok(Value::Null) + } + "pending" => Ok(json!(state.approver.pending())), + "decide" => { + let id: String = arg(args, "id")?; + let decision: Decision = arg(args, "decision")?; + Ok(json!(state.approver.decide(&id, decision))) + } + "pairCode" => { + let access: Access = arg(args, "access")?; + let notebooks: Scope = arg(args, "notebooks")?; + let Some((port, _)) = state.running() else { + return Err(IpcError::new( + "notRunning", + "Turn on Let apps on this PC connect first.", + )); + }; + let now = Instant::now(); + let (code, expires) = state.api(app).pairing.new_code(access, notebooks, now); + let left = expires.saturating_duration_since(now).as_millis() as f64; + Ok(json!({ "code": code, "port": port, "expiresAt": crate::boot::now_epoch_ms() + left })) + } + "cancelCode" => { + state.api(app).pairing.cancel_code(); + Ok(Value::Null) + } + "webhooks" => Ok(json!(state.grants.config().webhooks)), + "saveWebhook" => { + let hook: Webhook = arg(args, "hook")?; + let secret: Option = arg(args, "secret")?; + save_webhook(state, hook, secret).inspect(|_| changed("webhooks")) + } + "deleteWebhook" => { + let id: String = arg(args, "id")?; + let found = state + .grants + .change(|config| { + let before = config.webhooks.len(); + config.webhooks.retain(|hook| hook.id != id); + before != config.webhooks.len() + }) + .map_err(io)?; + let _ = state.secrets.delete(&secret_target(&id)); + changed("webhooks"); + Ok(json!(found)) + } + "testWebhook" => { + let id: String = arg(args, "id")?; + match state.dispatcher()?.test(&id) { + Ok(status) => Ok(json!({ "status": status })), + Err(_) => Ok(json!({ "status": null })), + } + } + "cliStatus" => { + let tool = tool_path(); + let folder = tool.parent().map(PathBuf::from).unwrap_or_default(); + Ok(json!({ + "installed": tool.is_file(), + "onPath": tool.is_file() && crate::install::path_entry::contains(&folder), + "folder": folder.display().to_string(), + })) + } + "setCliPath" => { + let on: bool = arg(args, "on")?; + let after_move: Option = arg(args, "afterMove")?; + if after_move == Some(true) { + // Setup's choice, made just before the app moves to the Programs folder and copies the command + // there (install::prepare_move). The folder isn't there yet, so this doesn't look for the tool. + let paths = app.state::(); + let folder = crate::install::programs_dir(&paths).join(crate::install::path_entry::CLI_DIR); + let done = if on { + crate::install::path_entry::add(&folder) + } else { + crate::install::path_entry::remove(&folder) + }; + return Ok(json!({ "onPath": on && done })); + } + let tool = tool_path(); + let folder = tool.parent().map(PathBuf::from).unwrap_or_default(); + if on && !tool.is_file() { + return Err(IpcError::new( + "notInstalled", + "The opennote command isn't installed yet.", + )); + } + if on { + crate::install::path_entry::add(&folder); + } else { + crate::install::path_entry::remove(&folder); + } + call(app, state, "cliStatus", &Value::Null) + } + "quickNote" => quick_note(app, args), + "phoneStart" => state.phone.start(app), + "phoneStop" => { + state.phone.stop(); + Ok(Value::Null) + } + "phoneState" => Ok(json!({ "state": state.phone.state() })), + "phoneTake" => Ok(state.phone.take()), + "mcpConfig" => { + let author = args.get("author").and_then(Value::as_str); + Ok(json!({ "config": mcp_config(&tool_path(), author), "toolInstalled": tool_path().is_file() })) + } + _ => Err(IpcError::invalid("method", "isn't a local API method")), + } +} + +/// Saves a hook the person typed. A new hook without a secret gets a random one, returned this once so the person +/// can give it to the receiver; after that it is only in the credential store. +fn save_webhook(state: &ApiState, hook: Webhook, secret: Option) -> IpcResult { + let mut hook = clean_hook(hook).map_err(|why| IpcError::invalid("hook", why))?; + let existing = state.grants.config().webhooks; + let is_new = !existing.iter().any(|one| one.id == hook.id); + if is_new { + if existing.len() >= MAX_HOOKS { + return Err(IpcError::invalid("hook", "can't be added: 20 webhooks is the most")); + } + hook.id = opennote_api::grants::random_hex(8); + } + let typed = secret + .map(|secret| secret.trim().to_owned()) + .filter(|secret| !secret.is_empty()); + if typed + .as_ref() + .is_some_and(|secret| secret.len() > 256 || secret.chars().any(char::is_control)) + { + return Err(IpcError::invalid( + "secret", + "must be one line of at most 256 characters", + )); + } + let made = (is_new && typed.is_none()).then(|| opennote_api::grants::random_hex(32)); + if let Some(secret) = typed.as_ref().or(made.as_ref()) { + state + .secrets + .put(&secret_target(&hook.id), secret) + .map_err(|()| IpcError::new("store", "The credential store refused the secret."))?; + } + let saved = hook.clone(); + state + .grants + .change(|config| { + config.webhooks.retain(|one| one.id != saved.id); + config.webhooks.push(saved); + }) + .map_err(io)?; + Ok(json!({ "hook": hook, "newSecret": made })) +} + +/// A page in Quick notes for a share or a clip link that no open page took. Only the app's own window calls this, +/// with what the person shared or agreed to add; the title and text are bounded here all the same. +fn quick_note(app: &AppHandle, args: &Value) -> IpcResult { + let from: String = arg(args, "from")?; + let author = match from.as_str() { + "share" => "Share to OpenNote", + "clip" => "a clip link", + _ => return Err(IpcError::invalid("from", "must be share or clip")), + }; + let title: String = arg(args, "title")?; + let markdown: String = arg(args, "markdown")?; + let source_url: Option = arg(args, "sourceUrl")?; + let title: String = title.chars().filter(|c| !c.is_control()).take(200).collect(); + if markdown.len() > 2 * 1024 * 1024 { + return Err(IpcError::invalid("markdown", "is too long")); + } + let source_url = source_url.filter(|url| { + url.len() <= 2048 + && (url.starts_with("https://") || url.starts_with("http://")) + && !url.chars().any(char::is_whitespace) + }); + let page = opennote_api::backend::NewPage { + title: if title.trim().is_empty() { + "Shared".to_owned() + } else { + title.trim().to_owned() + }, + markdown, + source_url, + attachments: Vec::new(), + }; + CoreBackend::new(Notes::App(app.clone())) + .quick_note(page, author) + .map(|info| json!(info)) + .map_err(|error| match error { + opennote_api::BackendError::Locked => IpcError::new("locked", "Quick notes is locked."), + opennote_api::BackendError::NotFound => IpcError::new("notFound", "There are no notebooks yet."), + opennote_api::BackendError::Failed(why) => { + ::log::warn!("Couldn't make a quick note: {why}"); + IpcError::new("failed", "The page couldn't be made.") + } + _ => IpcError::new("refused", "The page couldn't be made."), + }) +} + +/// Every method of App permissions, in one command. +#[tauri::command] +pub async fn api_call(app: AppHandle, state: State<'_, ApiState>, method: String, args: Value) -> IpcResult { + let _ = state; + // Starting, stopping, and revoking touch the credential store and the disk, so they run off the command thread. + tauri::async_runtime::spawn_blocking(move || { + let state = app.state::(); + call(&app, &state, &method, &args) + }) + .await + .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string()))? +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_mcp_config_names_the_tool_and_its_mcp_command() { + let config = mcp_config( + std::path::Path::new(r"C:\Users\Ada\AppData\Local\Programs\OpenNote\bin\opennote.exe"), + None, + ); + let json: Value = serde_json::from_str(&config).expect("json"); + assert_eq!(json["mcpServers"]["opennote"]["args"], json!(["mcp"])); + assert!(json["mcpServers"]["opennote"]["command"] + .as_str() + .expect("a path") + .ends_with(r"bin\opennote.exe")); + + let named = mcp_config(std::path::Path::new("opennote"), Some(" Claude (copyedit)\n ")); + let json: Value = serde_json::from_str(&named).expect("json"); + assert_eq!( + json["mcpServers"]["opennote"]["args"], + json!(["mcp", "--author", "Claude (copyedit)"]) + ); + let blank = mcp_config(std::path::Path::new("opennote"), Some(" ")); + let json: Value = serde_json::from_str(&blank).expect("json"); + assert_eq!(json["mcpServers"]["opennote"]["args"], json!(["mcp"])); + } +} diff --git a/app/src-tauri/src/api/phone.rs b/app/src-tauri/src/api/phone.rs new file mode 100644 index 00000000..be2beb02 --- /dev/null +++ b/app/src-tauri/src/api/phone.rs @@ -0,0 +1,157 @@ +//! Insert > From phone in the shell (flag `integrations.phoneScan`; docs/help/phone-scan.md). The dialog starts a +//! session with `phoneStart` and ends it with `phoneStop` when it closes; the listener itself is in +//! `opennote_api::phone`. Photos wait here until the window takes them with `phoneTake` and pastes them onto the +//! open page. Nothing waits past the session: stopping it drops photos the window never took. + +use std::{ + collections::VecDeque, + net::IpAddr, + sync::{Arc, Mutex, PoisonError}, + time::Instant, +}; + +use base64::Engine; +use opennote_api::phone::{lan_address, qr_rows, Closed, PhoneLimits, PhoneScan, Photo}; +use serde_json::{json, Value}; +use tauri::{AppHandle, Emitter}; + +use crate::{ + ipc::{IpcError, IpcResult}, + platform_flags, +}; + +/// Photos waiting for the window, at most. +const MAX_WAITING: usize = 30; + +#[derive(Default)] +pub struct PhoneState { + scan: Mutex>, + photos: Arc>>, +} + +impl PhoneState { + /// Starts a session, ending any earlier one. + pub fn start(&self, app: &AppHandle) -> IpcResult { + if !platform_flags::is_on_for(app, platform_flags::PHONE_SCAN) { + return Err(IpcError::new("off", "Insert from phone is off.")); + } + self.stop(); + let Some(ip) = lan_address() else { + return Err(IpcError::new( + "noNetwork", + "This PC isn't on a home or work network that a phone can reach.", + )); + }; + let (photos, notify) = (self.photos.clone(), app.clone()); + let scan = PhoneScan::start(IpAddr::V4(ip), PhoneLimits::default(), move |photo| { + let mut waiting = photos.lock().unwrap_or_else(PoisonError::into_inner); + if waiting.len() < MAX_WAITING { + waiting.push_back(photo); + } + drop(waiting); + let _ = notify.emit(super::EVENT, json!({ "kind": "phone", "what": "photo" })); + }) + .map_err(|error| IpcError::new("io", error.to_string()))?; + let url = scan.url(); + let left = scan.deadline().saturating_duration_since(Instant::now()).as_millis() as f64; + let started = json!({ + "url": url, + "qr": qr_rows(&url).unwrap_or_default(), + "fingerprint": scan.fingerprint(), + "expiresAt": crate::boot::now_epoch_ms() + left, + }); + // The address is the PC's own on its network; the token is never logged. + ::log::info!( + "Insert from phone listens on {} until its dialog closes.", + scan.address() + ); + *self.scan.lock().unwrap_or_else(PoisonError::into_inner) = Some(scan); + Ok(started) + } + + /// Ends the session and drops photos nobody took. + pub fn stop(&self) { + let scan = self.scan.lock().unwrap_or_else(PoisonError::into_inner).take(); + if let Some(scan) = scan { + scan.stop(); + ::log::info!("Insert from phone stopped listening."); + } + self.photos.lock().unwrap_or_else(PoisonError::into_inner).clear(); + } + + /// `open`, or why the session ended: `stopped`, `expired`, `tooManyTries`, `full`, or `none`. + pub fn state(&self) -> &'static str { + let mut scan = self.scan.lock().unwrap_or_else(PoisonError::into_inner); + let Some(running) = scan.as_ref() else { + return "none"; + }; + let state = running.state(); + if state != Closed::Open { + // The listener is closed already; this lets the thread go. + if let Some(done) = scan.take() { + done.stop(); + } + } + state_name(state) + } + + /// The photos that arrived since the last take, as base64. + pub fn take(&self) -> Value { + let photos: Vec = self + .photos + .lock() + .unwrap_or_else(PoisonError::into_inner) + .drain(..) + .collect(); + Value::Array( + photos + .into_iter() + .map(|photo| { + json!({ + "mime": photo.mime, + "width": photo.width, + "height": photo.height, + "data": base64::engine::general_purpose::STANDARD.encode(&photo.bytes), + }) + }) + .collect(), + ) + } +} + +pub fn state_name(state: Closed) -> &'static str { + match state { + Closed::Open => "open", + Closed::Stopped => "stopped", + Closed::Expired => "expired", + Closed::TooManyTries => "tooManyTries", + Closed::Full => "full", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_every_state_for_the_window() { + let names: Vec<_> = [ + Closed::Open, + Closed::Stopped, + Closed::Expired, + Closed::TooManyTries, + Closed::Full, + ] + .into_iter() + .map(state_name) + .collect(); + assert_eq!(names, ["open", "stopped", "expired", "tooManyTries", "full"]); + } + + #[test] + fn taking_with_nothing_waiting_gives_an_empty_list() { + let state = PhoneState::default(); + assert_eq!(state.take(), json!([])); + assert_eq!(state.state(), "none"); + } +} diff --git a/app/src-tauri/src/api/webhooks.rs b/app/src-tauri/src/api/webhooks.rs new file mode 100644 index 00000000..cdb77053 --- /dev/null +++ b/app/src-tauri/src/api/webhooks.rs @@ -0,0 +1,160 @@ +//! Outgoing webhooks in the app (crates/api/src/webhooks.rs; docs/help/local-api.md). The core's saves arrive +//! through the bridge's relay as page IDs; a worker thread reads each page, turns the saves into "added", "changed" +//! and "tag added" events, and hands them to the dispatcher, which posts them over HTTPS with Windows' TLS stack. +//! Nothing is sent while Work offline is on, and a page in a locked section is never read. + +use std::{ + sync::{mpsc, Arc}, + thread, + time::{Duration, Instant}, +}; + +use opennote_api::{ + backend::Backend, + webhooks::{check_url, ChangeWatcher, Dispatcher, Poster, Saved, QUIET}, +}; +use ureq::Agent; + +use super::backend::CoreBackend; + +/// How long one delivery may take. +const TIMEOUT: Duration = Duration::from_secs(15); + +/// The most a receiver's answer is read, since only its status matters. +const MAX_ANSWER: u64 = 64 * 1024; + +/// Posts with `ureq`: HTTPS only, no proxy, no cookies, no redirects, and only to public addresses, so a delivery +/// goes only to the address the person typed and never into the local network. +pub struct HttpsPoster; + +fn agent() -> Agent { + use ureq::tls::{RootCerts, TlsConfig, TlsProvider}; + let tls = TlsConfig::builder() + .provider(TlsProvider::NativeTls) + .root_certs(RootCerts::PlatformVerifier) + .build(); + let config = Agent::config_builder() + .tls_config(tls) + .https_only(true) + .proxy(None) + .max_redirects(0) + .max_redirects_will_error(false) + .http_status_as_error(false) + .timeout_global(Some(TIMEOUT)) + .build(); + // A public host name can still point at a PC or router here: only public addresses are kept when it resolves. + Agent::with_parts( + config, + ureq::unversioned::transport::DefaultConnector::new(), + crate::images::web::public_resolver(), + ) +} + +impl Poster for HttpsPoster { + fn post(&self, url: &str, headers: &[(String, String)], body: &[u8]) -> Result { + if crate::hardening::offline() { + return Err("Work offline is on.".to_owned()); + } + check_url(url).map_err(str::to_owned)?; + let mut request = agent().post(url); + for (name, value) in headers { + request = request.header(name.as_str(), value.as_str()); + } + // The message never repeats the address or the body. + let mut response = request + .send(body) + .map_err(|_| "The address couldn't be reached.".to_owned())?; + let status = response.status().as_u16(); + let _ = response.body_mut().with_config().limit(MAX_ANSWER).read_to_vec(); + Ok(status) + } +} + +/// The page IDs of saves, and when to stop. +pub enum Watch { + Saved(String), + Stop, +} + +/// Reads saved pages and publishes their events, until it hears [`Watch::Stop`]. +pub fn start_watcher(backend: CoreBackend, dispatcher: Arc) -> mpsc::Sender { + let (sender, receiver) = mpsc::channel::(); + let spawned = thread::Builder::new() + .name("opennote-webhook-watch".into()) + .spawn(move || { + let mut watcher = ChangeWatcher::new(QUIET); + loop { + let wait = watcher.next_due().map_or(Duration::from_secs(3600), |at| { + at.saturating_duration_since(Instant::now()) + }); + match receiver.recv_timeout(wait) { + Ok(Watch::Saved(page)) => { + if let Some(saved) = describe(&backend, &page) { + for event in watcher.saved(saved, Instant::now()) { + dispatcher.publish(&event); + } + } + } + Ok(Watch::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => return, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + for event in watcher.due(Instant::now()) { + dispatcher.publish(&event); + } + } + }); + if spawned.is_err() { + ::log::warn!("Webhooks couldn't start their watcher."); + } + sender +} + +/// A page made within this long before its save counts as new. +const NEW_FOR_MS: i64 = 2 * 60 * 1000; + +/// What a webhook needs to know about a saved page. A locked page is described without being read. +fn describe(backend: &CoreBackend, page: &str) -> Option { + let place = backend.locate(page).ok()?; + let section = place.section_id.clone()?; + let info = opennote_api::backend::PageInfo { + id: place.id.clone(), + notebook_id: place.notebook_id.clone(), + section_id: section, + title: place.title.clone(), + modified: None, + }; + if place.locked { + return Some(Saved { + page: info, + locked: true, + new: false, + tags: Vec::new(), + text: None, + }); + } + let (markdown, json) = backend.page_with_json(page).ok()?; + let created = json["created"] + .as_str() + .and_then(|text| opennote_core::Timestamp::parse(text).ok()) + .map(opennote_core::Timestamp::unix_ms); + let now = crate::boot::now_epoch_ms() as i64; + Some(Saved { + page: info, + locked: false, + new: created.is_some_and(|created| now - created < NEW_FOR_MS), + tags: super::markdown::tags(&json), + text: Some(markdown), + }) +} + +#[cfg(test)] +mod tests { + use super::agent; + + #[test] + fn a_delivery_never_connects_to_a_local_address() { + // localhost resolves to loopback only, so the resolver refuses it before any connection is tried. + let error = agent().post("https://localhost:9/hook").send(&b"{}"[..]).unwrap_err(); + assert!(matches!(error, ureq::Error::BadUri(_)), "{error:?}"); + } +} diff --git a/app/src-tauri/src/appearance.rs b/app/src-tauri/src/appearance.rs index 8aa0205c..bdb629e7 100644 --- a/app/src-tauri/src/appearance.rs +++ b/app/src-tauri/src/appearance.rs @@ -155,7 +155,7 @@ pub fn read_raw() -> RawAppearance { /// A `REG_DWORD` under `HKEY_CURRENT_USER`, or `None` when it isn't there. #[cfg(windows)] -fn registry_dword(subkey: &str, value: &str) -> Option { +pub(crate) fn registry_dword(subkey: &str, value: &str) -> Option { use windows::{ core::PCWSTR, Win32::System::Registry::{RegGetValueW, HKEY_CURRENT_USER, RRF_RT_REG_DWORD}, diff --git a/app/src-tauri/src/audio.rs b/app/src-tauri/src/audio.rs new file mode 100644 index 00000000..1ccffcb3 --- /dev/null +++ b/app/src-tauri/src/audio.rs @@ -0,0 +1,883 @@ +// checks-disable-file modifiability: one module whose parts share its state; split it when it grows again + +//! Audio recording and playback for the interface (Phase 9; ADR 0007). The media crate does the work; this module +//! is the Tauri side of its service: one [`AudioService`] in the app's state, one command for each of its methods +//! (the interface's `hostOver` names them `audio_`), and a few commands that edit a finished recording. +//! +//! Recordings are assets of the page, in the page's own folder in the notes folder (`/
// +//! assets/-mic.ogg`, with a timeline file beside it), so they travel with the notes. The page keeps each +//! recording's entry in its view, under `recordings`, with a block of type `ext:org.opennote/recording` to say where +//! the recording sits. The core can't edit a block of a type it doesn't know, but it edits and keeps the view. +//! +//! The media crate writes the files, and the core owns the page's asset table. Before the interface saves an entry +//! that makes or changes tracks, it calls `audio_adopt_tracks`. That hands the open page an entry for each track +//! (`adopt_asset`), and the interface's `addAsset` edit puts them in the table. A track that is still growing is +//! marked `state: recording`, and the core's size and hash checks skip it. The entry written when the recording +//! stops replaces it. Without a table entry, the core's garbage collection would delete the file at 30 days. +//! +//! The interface asks for a page's folder with `audio_assets_dir`. Every command that takes a folder checks that it +//! is the `assets` folder of a page in an open notebook. + +use std::{ + collections::HashMap, + fs, + path::{Component, Path, PathBuf}, + sync::{Arc, Mutex, PoisonError}, +}; + +use opennote_core::{ + store::layout::{NotebookLayout, ASSETS_DIR}, + PageId, SystemClock, +}; +use opennote_media::{ + audio::{AudioError, DeviceInfo, RecordingSummary, Resolution}, + edit::{self, Range}, + layout::{asset_entry, RecordingEntry, RecordingPlan}, + service::{ + AudioService, ClockReading, Finished, PlaybackInfo, PlaybackStatus, Prepared, RecordingStatus, Services, + StartRequest, + }, + storage, +}; +use serde::Serialize; +use tauri::{AppHandle, Manager, State, WebviewWindow}; + +use crate::{ + core_bridge::{Bridge, CoreBridge}, + ipc::{codes, IpcError, IpcResult}, + paths::Paths, +}; + +/// The error codes the interface maps to messages (`audio.errors` in the strings). +pub mod error_codes { + pub const DEVICE: &str = "audioDevice"; + pub const ENCODER: &str = "audioEncoder"; + pub const FORMAT: &str = "audioFormat"; + pub const CORRUPT: &str = "audioCorrupt"; + /// None of the recording's audio files are there. The only answer recovery gives up on: a damaged file + /// (`audioCorrupt`) is still on disk. + pub const MISSING: &str = "audioMissing"; + pub const WRITER: &str = "audioWriter"; + /// The recording asked to be recovered is the one running now, perhaps in another window. + pub const RUNNING: &str = "audioRunning"; +} + +struct Inner { + services: Services, + service: Mutex, + /// The recordings that stopped in this run, by the window that stopped them, which writes their final entry. + stopped: Mutex>, +} + +/// The managed state behind the audio commands. +#[derive(Clone)] +pub struct AudioState(Arc); + +impl AudioState { + /// The real devices. + pub fn new(_paths: &Paths) -> AudioState { + Self::with(Services::system(Arc::new(SystemClock::new()))) + } + + /// Any services, for tests. + pub fn with(services: Services) -> AudioState { + AudioState(Arc::new(Inner { + service: Mutex::new(AudioService::new(services.clone())), + services, + stopped: Mutex::default(), + })) + } + + /// The devices, clocks, and codecs the commands run on, for the commands in `audio_more`. + pub(crate) fn services(&self) -> &Services { + &self.0.services + } + + fn lock(&self) -> std::sync::MutexGuard<'_, AudioService> { + self.0.service.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn stopped(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.0.stopped.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Stops the running recording for the window labeled `window`, which then writes its final entry. + fn stop_for(&self, window: &str) -> IpcResult { + let mut service = self.lock(); + let finished = service.stop().map_err(audio_error)?; + // Marked while the recorder is still held, so no recovery comes between the stop and the mark. + self.stopped().insert(finished.entry.id.clone(), window.to_owned()); + Ok(finished) + } + + /// A window went: the recordings it stopped may be recovered from any window now, as nothing else writes + /// their entry. + pub fn forget_window(&self, window: &str) { + self.stopped().retain(|_, stopper| stopper != window); + } + + /// Stops a recording that is still running, so its files close cleanly. The app calls it on exit. + pub fn shutdown(&self) { + let mut service = self.lock(); + if service.recording_status().is_ok() { + if let Err(error) = service.stop() { + ::log::error!("Couldn't close the recording on exit: {error}"); + } + } + service.close_playback(); + } +} + +/// The `assets` folder of a page of an open notebook, as the core lays it out. It may not exist yet. +pub(crate) fn page_assets_dir(bridge: &Bridge, page: PageId) -> Option { + let (notebook, _) = bridge.core.find_node(page.0)?; + let tree = notebook.tree(); + let (section, _) = tree.find_page(page)?; + // Recordings write straight to disk, so a protected section has none: they would be plain (ADR 0042). + if section.encrypted { + return None; + } + Some( + NotebookLayout::new(notebook.path()) + .page_dir(section.id, page) + .join(ASSETS_DIR), + ) +} + +/// The folder of a page's recordings: the page's own `assets` folder, made if it is not there. +fn assets_dir(bridge: &Bridge, page: &str) -> IpcResult { + let id = PageId::parse(page).map_err(|_| IpcError::invalid("page", "That isn't a page ID."))?; + let dir = page_assets_dir(bridge, id).ok_or_else(|| IpcError::new("notFound", "That page isn't in a notebook."))?; + fs::create_dir_all(&dir)?; + Ok(dir) +} + +/// A folder from the interface, if it is the `assets` folder of a page of an open notebook, with the page. +pub(crate) fn checked_dir(bridge: &Bridge, dir: &str) -> IpcResult<(PathBuf, PageId)> { + let path = PathBuf::from(dir); + let plain = path + .components() + .all(|part| !matches!(part, Component::ParentDir | Component::CurDir)); + let page = path + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + .and_then(|name| PageId::parse(name).ok()); + let wrong = || IpcError::invalid("assetsDir", "That isn't a folder of recordings."); + let page = page.filter(|_| plain).ok_or_else(wrong)?; + match page_assets_dir(bridge, page) { + Some(expected) if crate::notes::same_path(&expected, &path) => Ok((expected, page)), + _ => Err(wrong()), + } +} + +/// An asset table entry for each track of `entry`, as the files in `dir` are now. `growing` says the files are +/// still being written. It reads the finished files to hash them, so it runs off the async runtime's threads. +pub(crate) fn track_assets( + services: &Services, + dir: &Path, + entry: &RecordingEntry, + growing: bool, +) -> IpcResult> { + let created = services.session.now(); + RecordingPlan::from_entry(entry) + .tracks + .iter() + .map(|track| asset_entry(dir, track, created, growing).map_err(audio_error)) + .collect() +} + +/// Hands the open page its assets, for the interface's next `addAsset` edit. +fn adopt(bridge: &Bridge, page: PageId, assets: Vec) -> IpcResult<()> { + let handle = bridge + .open + .iter() + .find(|((id, _), _)| *id == page) + .map(|(_, handle)| handle.clone()) + .ok_or_else(|| IpcError::new("notFound", "This page isn't open."))?; + for asset in assets { + handle.adopt_asset(asset); + } + Ok(()) +} + +/// The audio error as the interface's error, with a code it has a message for. +pub fn audio_error(error: AudioError) -> IpcError { + let code = match &error { + AudioError::Io(_) => codes::IO, + AudioError::Device(_) => error_codes::DEVICE, + AudioError::Encoder(_) => error_codes::ENCODER, + AudioError::Format(_) => error_codes::FORMAT, + AudioError::Corrupt(_) => error_codes::CORRUPT, + AudioError::Missing(_) => error_codes::MISSING, + AudioError::Writer(_) => error_codes::WRITER, + }; + IpcError::new(code, error.to_string()) +} + +pub(crate) fn blocking_failed(error: impl std::fmt::Display) -> IpcError { + IpcError::new(codes::INTERNAL, error.to_string()) +} + +/// Runs slow work, such as opening devices or copying a long recording, off the async runtime's threads. +pub(crate) async fn blocking( + state: &AudioState, + work: impl FnOnce(&AudioState) -> IpcResult + Send + 'static, +) -> IpcResult { + let state = state.clone(); + tauri::async_runtime::spawn_blocking(move || work(&state)) + .await + .map_err(blocking_failed)? +} + +#[tauri::command] +pub async fn audio_assets_dir(bridge: State<'_, CoreBridge>, page: String) -> IpcResult { + let dir = bridge + .run(move |bridge| bridge.with(|bridge| assets_dir(bridge, &page))) + .await?; + Ok(dir.to_string_lossy().into_owned()) +} + +#[tauri::command] +pub async fn audio_devices(state: State<'_, AudioState>) -> IpcResult> { + blocking(&state, |state| state.lock().devices().map_err(audio_error)).await +} + +#[tauri::command] +pub async fn audio_clock(state: State<'_, AudioState>) -> IpcResult { + Ok(state.lock().clock()) +} + +#[tauri::command] +pub async fn audio_prepare( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + mut request: StartRequest, +) -> IpcResult { + let assets_dir = request.assets_dir.to_string_lossy().into_owned(); + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + request.assets_dir = dir; + blocking(&state, move |state| state.lock().prepare(request).map_err(audio_error)).await +} + +/// Hands the open page the tracks of an entry, as the files in its `assets` folder are now: growing, or finished. +/// The interface calls it just before the `addAsset` edits that put them in the page's table. +#[tauri::command] +pub async fn audio_adopt_tracks( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + assets_dir: String, + entry: RecordingEntry, + growing: bool, +) -> IpcResult<()> { + let (dir, page) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + let assets = blocking(&state, move |state| { + track_assets(&state.0.services, &dir, &entry, growing) + }) + .await?; + bridge + .run(move |bridge| bridge.with(|bridge| adopt(bridge, page, assets))) + .await +} + +#[tauri::command] +pub async fn audio_begin(state: State<'_, AudioState>) -> IpcResult { + blocking(&state, |state| { + state.lock().begin().map_err(|failed| audio_error(failed.error)) + }) + .await +} + +#[tauri::command] +pub async fn audio_recording_status(state: State<'_, AudioState>) -> IpcResult { + state.lock().recording_status().map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_pause_recording(state: State<'_, AudioState>) -> IpcResult<()> { + state.lock().pause_recording().map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_resume_recording(state: State<'_, AudioState>) -> IpcResult<()> { + state.lock().resume_recording().map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_switch_microphone(state: State<'_, AudioState>, id: Option) -> IpcResult { + blocking(&state, move |state| { + state.lock().switch_microphone(id).map_err(audio_error) + }) + .await +} + +#[tauri::command] +pub async fn audio_switch_system_audio(state: State<'_, AudioState>, id: Option) -> IpcResult { + blocking(&state, move |state| { + state.lock().switch_system_audio(id).map_err(audio_error) + }) + .await +} + +#[tauri::command] +pub async fn audio_stop(window: WebviewWindow, state: State<'_, AudioState>) -> IpcResult { + let label = window.label().to_owned(); + blocking(&state, move |state| state.stop_for(&label)).await +} + +#[tauri::command] +pub async fn audio_recover( + window: WebviewWindow, + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + assets_dir: String, + entry: RecordingEntry, +) -> IpcResult { + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + let label = window.label().to_owned(); + blocking(&state, move |state| recover_unless_running(state, &label, &dir, &entry)).await +} + +/// Recovers a recording a crash cut off for the window labeled `window`, or refuses with `audioRunning` when +/// another window owns its entry: it is the recording running now, or one that window stopped and is writing the +/// final entry of. Every window shares this process's recorder but keeps its own idea of what records, so a page +/// opened in a second window while the first records it must learn that from here and leave the entry alone. The +/// window that stopped a recording may recover it, as when an undo takes its entry back to `recording`. +fn recover_unless_running(state: &AudioState, window: &str, dir: &Path, entry: &RecordingEntry) -> IpcResult { + let service = state.lock(); + if service.is_running(&entry.id) { + return Err(IpcError::new( + error_codes::RUNNING, + "This recording is still running. Its entry is saved when it stops.", + )); + } + if state.stopped().get(&entry.id).is_some_and(|stopper| stopper != window) { + return Err(IpcError::new( + error_codes::RUNNING, + "This recording stopped in another window, which saves its entry.", + )); + } + service.recover(dir, entry).map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_open_playback( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + assets_dir: String, + entry: RecordingEntry, + device: Option, +) -> IpcResult { + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + blocking(&state, move |state| { + state + .lock() + .open_playback(&dir, &entry, device.as_deref()) + .map_err(audio_error) + }) + .await +} + +#[tauri::command] +pub async fn audio_play(state: State<'_, AudioState>) -> IpcResult<()> { + state.lock().play().map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_pause_playback(state: State<'_, AudioState>) -> IpcResult<()> { + state.lock().pause_playback().map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_seek(state: State<'_, AudioState>, position_ns: u64) -> IpcResult<()> { + state.lock().seek(position_ns).map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_skip(state: State<'_, AudioState>, delta_ns: i64) -> IpcResult<()> { + state.lock().skip(delta_ns).map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_set_speed(state: State<'_, AudioState>, speed: f32) -> IpcResult<()> { + state.lock().set_speed(speed).map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_set_skip_silence(state: State<'_, AudioState>, on: bool) -> IpcResult<()> { + state.lock().set_skip_silence(on).map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_playback_status(state: State<'_, AudioState>) -> IpcResult { + state.lock().playback_status().map_err(audio_error) +} + +#[tauri::command] +pub async fn audio_close_playback(state: State<'_, AudioState>) -> IpcResult<()> { + state.lock().close_playback(); + Ok(()) +} + +/// A finished edit: the entry that takes the old one's place in the page, and the length it now has. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Edited { + pub entry: RecordingEntry, + pub duration_ns: u64, +} + +pub(crate) fn edited(summary: &RecordingSummary, previous: &RecordingEntry) -> Edited { + Edited { + entry: RecordingEntry::from_summary(summary, Some(previous)), + duration_ns: opennote_media::positions::PositionMap::from_summary(summary).duration_ns(), + } +} + +/// Trims the silence at the start and end. The answer is none when there is nothing worth trimming. The old +/// files stay until the page has saved the new entry and the interface calls `audio_delete_files`. +#[tauri::command] +pub async fn audio_trim_silence( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + assets_dir: String, + entry: RecordingEntry, +) -> IpcResult> { + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + blocking(&state, move |state| { + let services = &state.0.services; + let summary = entry.summary().map_err(audio_error)?; + let plan = RecordingPlan::replacing(&summary, services.session.as_ref()); + let trimmed = edit::trim_silence(&dir, &summary, &plan, &services.decoder).map_err(audio_error)?; + Ok(trimmed.map(|summary| edited(&summary, &entry))) + }) + .await +} + +/// Removes the part between two positions of the audio, in nanoseconds. Notes keep their timing. +#[tauri::command] +pub async fn audio_remove_part( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + assets_dir: String, + entry: RecordingEntry, + start_ns: u64, + end_ns: u64, +) -> IpcResult { + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + if end_ns <= start_ns { + return Err(IpcError::invalid("endNs", "The part to remove ends before it starts.")); + } + blocking(&state, move |state| { + let services = &state.0.services; + let summary = entry.summary().map_err(audio_error)?; + let plan = RecordingPlan::replacing(&summary, services.session.as_ref()); + let range: Range = (start_ns, end_ns); + let removed = edit::remove(&dir, &summary, &[range], &plan).map_err(audio_error)?; + Ok(edited(&removed, &entry)) + }) + .await +} + +/// Deletes the audio and timeline files of a recording the page no longer lists, such as the one an edit +/// replaced. It answers with the bytes freed. +#[tauri::command] +pub async fn audio_delete_files( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + assets_dir: String, + entry: RecordingEntry, +) -> IpcResult { + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + blocking(&state, move |_| { + let summary = entry.summary().map_err(audio_error)?; + storage::delete_audio(&dir, &summary).map_err(audio_error) + }) + .await +} + +/// Closes a recording that is still running when the app exits. +pub fn shutdown(app: &AppHandle) { + if let Some(state) = app.try_state::() { + state.shutdown(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use opennote_core::ops::resolve::{Edit, TxnRequest}; + use opennote_media::audio::{AudioSource, FixedCatalog, TrackKind}; + use opennote_media::service::DeviceFactory; + use serde_json::{json, Value}; + + struct NoDevices; + + impl DeviceFactory for NoDevices { + fn microphone(&self, _id: Option<&str>) -> opennote_media::audio::Result> { + Err(AudioError::Device("There is no microphone.".into())) + } + fn system_audio(&self, _id: Option<&str>) -> opennote_media::audio::Result> { + Err(AudioError::Device("There is no output.".into())) + } + fn output( + &self, + _id: Option<&str>, + ) -> opennote_media::audio::Result> { + Err(AudioError::Device("There is no output.".into())) + } + } + + fn services() -> Services { + let mut services = Services::system(Arc::new(SystemClock::new())); + services.devices = Arc::new(NoDevices); + services.catalog = Arc::new(FixedCatalog(Vec::new())); + services + } + + fn state() -> AudioState { + AudioState::with(services()) + } + + /// A bridge over a temporary notes folder with one notebook, section, and page; the page's ID. + fn a_page(dir: &Path) -> (CoreBridge, PathBuf, String) { + let notes = dir.join("Notes"); + let bridge = CoreBridge::at(dir.join("local")); + let page = bridge + .notes(Some(notes.clone()), |bridge| { + let create = |bridge: &mut Bridge, kind: &str, parent: Option| { + let input = json!({ "kind": kind, "placement": { "parentId": parent, "beforeId": null } }); + let node = bridge.dispatch("notes_create", &json!({ "input": input }))?; + Ok::<_, IpcError>(node["id"].as_str().unwrap_or_default().to_owned()) + }; + let notebook = create(bridge, "notebook", None)?; + let section = create(bridge, "section", Some(notebook))?; + let page = create(bridge, "page", Some(section))?; + bridge.handle(&page, "main-1")?; + Ok(page) + }) + .expect("a page"); + (bridge, notes, page) + } + + fn apply(bridge: &CoreBridge, notes: &Path, page: &str, seq: u64, edits: Value) -> IpcResult<()> { + bridge.notes(Some(notes.to_path_buf()), |bridge| { + let handle = bridge.open_handle(page, "main-1")?; + let request = TxnRequest { + page: handle.id(), + client: handle.client().clone(), + client_seq: seq, + coalesce: None, + ui: None, + edits: serde_json::from_value::>(edits).expect("edits"), + }; + handle + .apply(request) + .map(|_| ()) + .map_err(|error| IpcError::new(error.code(), error.to_string()))?; + handle + .save_now() + .map(|_| ()) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string())) + }) + } + + fn assets_of(bridge: &CoreBridge, notes: &Path, page: &str) -> Value { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + let envelope = bridge + .open_handle(page, "main-1")? + .envelope(None) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))?; + let decoded = opennote_core::wire::envelope::decode(&envelope.bytes) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))?; + let page: Value = serde_json::from_slice(decoded.page_json).expect("page JSON"); + Ok(page["assets"].clone()) + }) + .expect("the page's assets") + } + + #[test] + fn the_folder_is_the_pages_own_assets_folder_in_the_notes_folder() { + let dir = tempfile::tempdir().unwrap(); + let (bridge, notes, page) = a_page(dir.path()); + let folder = bridge.with(|bridge| assets_dir(bridge, &page)).unwrap(); + assert!(folder.is_dir()); + assert!(folder.starts_with(¬es), "{folder:?} is in the notes folder"); + assert_eq!(folder.file_name().unwrap(), "assets"); + assert_eq!(folder.parent().unwrap().file_name().unwrap().to_str().unwrap(), page); + bridge.shutdown(); + } + + #[test] + fn only_the_assets_folder_of_a_page_of_an_open_notebook_is_accepted() { + let dir = tempfile::tempdir().unwrap(); + let (bridge, _notes, page) = a_page(dir.path()); + let folder = bridge.with(|bridge| assets_dir(bridge, &page)).unwrap(); + let accepted = bridge + .with(|bridge| checked_dir(bridge, folder.to_str().unwrap())) + .unwrap(); + assert_eq!(accepted.0, folder); + assert_eq!(accepted.1.to_string(), page); + let outside = tempfile::tempdir().unwrap(); + let escaping = format!("{}/../..", folder.display()); + let page_dir = folder.parent().unwrap().to_path_buf(); + let nested = folder.join("deeper"); + let elsewhere = outside.path().join(&page).join("assets"); + for bad in [ + page_dir.as_path(), + nested.as_path(), + outside.path(), + elsewhere.as_path(), + Path::new(&escaping), + Path::new(""), + ] { + let error = bridge + .with(|bridge| checked_dir(bridge, bad.to_str().unwrap())) + .unwrap_err(); + assert_eq!(error.code, codes::INVALID, "{bad:?}"); + } + for bad in ["", "..", "../x", "a/b", "a\\b", "C:\\Windows", "not-a-page"] { + let error = bridge.with(|bridge| assets_dir(bridge, bad)).unwrap_err(); + assert!( + error.code == codes::INVALID || error.code == "notFound", + "{bad:?}: {}", + error.code + ); + } + bridge.shutdown(); + } + + /// The page's table lists a recording from its first save, and the finished entry replaces the growing one. + #[test] + fn a_recording_is_listed_in_the_page_table_while_it_grows_and_when_it_ends() { + let dir = tempfile::tempdir().unwrap(); + let (bridge, notes, page) = a_page(dir.path()); + let audio = state(); + let folder = bridge.with(|bridge| assets_dir(bridge, &page)).unwrap(); + let checked = bridge + .with(|bridge| checked_dir(bridge, folder.to_str().unwrap())) + .unwrap(); + let plan = RecordingPlan::generate(&[TrackKind::Microphone], audio.0.services.session.as_ref()); + let entry = RecordingEntry::starting(&plan, audio.0.services.session.now()); + let asset = entry.tracks[0].asset.clone(); + let file = folder.join(format!("{asset}-mic.ogg")); + + // Before the file exists: the page saves with a growing entry. + let growing = track_assets(&audio.0.services, &checked.0, &entry, true).unwrap(); + bridge.with(|bridge| adopt(bridge, checked.1, growing)).unwrap(); + apply( + &bridge, + ¬es, + &page, + 1, + json!([{ "edit": "addAsset", "asset": asset }]), + ) + .unwrap(); + let listed = assets_of(&bridge, ¬es, &page); + assert_eq!(listed[&asset]["state"], "recording"); + + // The file grows, and the page still saves. + fs::write(&file, vec![7u8; 100]).unwrap(); + apply( + &bridge, + ¬es, + &page, + 2, + json!([{ "edit": "addAsset", "asset": asset }]), + ) + .unwrap(); + + // It ends: the finished entry has the real size and hash and no state. + fs::write(&file, vec![7u8; 4096]).unwrap(); + let finished = track_assets(&audio.0.services, &checked.0, &entry, false).unwrap(); + bridge.with(|bridge| adopt(bridge, checked.1, finished)).unwrap(); + apply( + &bridge, + ¬es, + &page, + 3, + json!([{ "edit": "addAsset", "asset": asset }]), + ) + .unwrap(); + let done = assets_of(&bridge, ¬es, &page); + assert_eq!(done[&asset]["bytes"], 4096); + assert!(done[&asset].get("state").is_none(), "{done}"); + assert!(file.is_file()); + bridge.shutdown(); + } + + /// Generated sound in place of a microphone. Nothing here opens a real device. + #[derive(Default)] + struct Generated { + handles: Mutex>, + } + + impl DeviceFactory for Generated { + fn microphone(&self, _id: Option<&str>) -> opennote_media::audio::Result> { + let format = opennote_media::audio::SourceFormat { + rate: 48_000, + channels: 1, + }; + let signal: opennote_media::audio::synthetic::Signal = + Arc::new(|frame| ((frame % 97) as f32 - 48.0) / 200.0); + let (source, handle) = opennote_media::audio::synthetic::synthetic(format, signal, 1_000_000_000); + self.handles.lock().unwrap().push(handle); + Ok(Box::new(source)) + } + fn system_audio(&self, _id: Option<&str>) -> opennote_media::audio::Result> { + Err(AudioError::Device("There is no output.".into())) + } + fn output( + &self, + _id: Option<&str>, + ) -> opennote_media::audio::Result> { + Err(AudioError::Device("There is no output.".into())) + } + } + + /// F5-5: a page open in a second window while the main window recorded it saw the live entry and recovered + /// it. The host refuses the recording that runs, or that is prepared to, with `audioRunning`; then, once the + /// main window stopped it, refuses the other window until the main window, which writes the final entry, has + /// gone. A recording with no files answers `audioMissing`, the only answer the interface gives up on. + #[test] + fn only_the_window_that_owns_a_recording_may_recover_it() { + let dir = tempfile::tempdir().unwrap(); + let generated = Arc::new(Generated::default()); + let mut services = services(); + services.devices = generated.clone(); + services.catalog = Arc::new(FixedCatalog(vec![opennote_media::audio::DeviceInfo { + id: "mic".into(), + name: "Generated".into(), + direction: opennote_media::audio::Direction::Input, + is_default: true, + sample_rate: Some(48_000), + channels: Some(1), + }])); + services.clock = opennote_media::audio::synthetic::ManualClock::new(1_000_000_000); + services.encoder = opennote_media::pcm_codec::pcm_encoder_factory(); + services.decoder = opennote_media::pcm_codec::pcm_decoder_factory(); + let state = AudioState::with(services); + let running = |result: IpcResult| result.map(|_| ()).unwrap_err().code == error_codes::RUNNING; + + let prepared = state + .lock() + .prepare(StartRequest { + assets_dir: dir.path().to_path_buf(), + microphone: None, + system_audio: false, + system_device: None, + }) + .expect("prepares"); + let entry = prepared.entry; + assert!( + running(recover_unless_running(&state, "page-x", dir.path(), &entry)), + "a prepared recording is running" + ); + state.lock().begin().expect("begins"); + generated.handles.lock().unwrap()[0].produce_ms(500); + assert!( + running(recover_unless_running(&state, "page-x", dir.path(), &entry)), + "the running recording is refused in another window" + ); + assert!(running(recover_unless_running(&state, "main", dir.path(), &entry))); + + let finished = state.stop_for("main").expect("stops"); + assert_eq!(finished.entry.id, entry.id); + assert!( + running(recover_unless_running(&state, "page-x", dir.path(), &entry)), + "the main window writes the final entry, so the other window leaves it alone" + ); + let undone = recover_unless_running(&state, "main", dir.path(), &entry); + assert!( + undone.is_ok(), + "the window that stopped it recovers it, as after an undo: {undone:?}" + ); + state.forget_window("main"); + let orphaned = recover_unless_running(&state, "page-x", dir.path(), &entry); + assert!( + orphaned.is_ok(), + "once that window has gone, any window recovers it: {orphaned:?}" + ); + + let none: RecordingEntry = serde_json::from_value(json!({ + "id": "r1", + "state": "recording", + "started": "2026-10-07T12:00:00Z", + "tracks": [], + })) + .expect("an entry"); + let error = recover_unless_running(&state, "page-x", dir.path(), &none) + .map(|_| ()) + .unwrap_err(); + assert_eq!(error.code, error_codes::MISSING, "{}", error.message); + } + + #[test] + fn audio_errors_get_codes_the_interface_has_messages_for() { + let code = |error: AudioError| audio_error(error).code; + assert_eq!(code(AudioError::Device(String::new())), error_codes::DEVICE); + assert_eq!(code(AudioError::Encoder(String::new())), error_codes::ENCODER); + assert_eq!(code(AudioError::Format(String::new())), error_codes::FORMAT); + assert_eq!(code(AudioError::Corrupt(String::new())), error_codes::CORRUPT); + assert_eq!(code(AudioError::Missing(String::new())), error_codes::MISSING); + assert_eq!(code(AudioError::Writer(String::new())), error_codes::WRITER); + assert_eq!(code(AudioError::Io(std::io::Error::other("x"))), codes::IO); + } + + /// The whole path on this PC's devices: record two seconds from the default microphone, stop, and play it. + /// Run it with `cargo test -p opennote --lib real_microphone -- --ignored` on a PC with a microphone. + #[test] + #[ignore = "opens the real microphone and speakers"] + fn the_real_microphone_records_and_plays_back() { + let dir = tempfile::tempdir().unwrap(); + let (bridge, _notes, page) = a_page(dir.path()); + let state = AudioState::with(Services::system(Arc::new(SystemClock::new()))); + let assets = bridge.with(|bridge| assets_dir(bridge, &page)).unwrap(); + let mut service = state.lock(); + let request = StartRequest { + assets_dir: assets.clone(), + microphone: None, + system_audio: false, + system_device: None, + }; + service.prepare(request).expect("a microphone to prepare"); + service.begin().expect("the microphone to open"); + std::thread::sleep(std::time::Duration::from_secs(2)); + let status = service.recording_status().unwrap(); + assert!(status.bytes > 0, "the recording grew"); + let finished = service.stop().unwrap(); + assert_eq!(finished.entry.state, opennote_media::layout::RecordingState::Complete); + let info = service + .open_playback(&assets, &finished.entry, None) + .expect("the speakers to open"); + assert!(info.duration_ns > 1_500_000_000, "about two seconds were recorded"); + service.play().unwrap(); + std::thread::sleep(std::time::Duration::from_millis(500)); + let playing = service.playback_status().unwrap(); + assert!(playing.position_ns > 0, "playback moved"); + service.close_playback(); + } + + #[test] + fn nothing_runs_until_a_recording_begins() { + let state = state(); + assert!(state.lock().recording_status().is_err()); + assert!(state.lock().begin().is_err()); + state.shutdown(); + } +} diff --git a/app/src-tauri/src/audio_more/decode.rs b/app/src-tauri/src/audio_more/decode.rs new file mode 100644 index 00000000..7171b97a --- /dev/null +++ b/app/src-tauri/src/audio_more/decode.rs @@ -0,0 +1,143 @@ +//! Decoding an audio or video file to mono samples at 48 kHz, with the codecs Windows has: MP3, AAC in M4A and +//! MP4, WAV, and the others Media Foundation opens. Nothing is added to the app for this. A PC without the codec +//! for a file (the "N" editions of Windows lack the media pack) says so, and nothing is made. + +use std::path::Path; + +use crate::ipc::{IpcError, IpcResult}; + +/// The format the decoded samples have, which is the format of every recording track. +const RATE: u32 = 48_000; + +#[cfg(not(windows))] +pub fn decode_file(_path: &Path, _max_ns: u64) -> IpcResult> { + Err(IpcError::not_implemented("Importing audio files")) +} + +#[cfg(windows)] +fn unreadable(message: &str) -> IpcError { + IpcError::new("audioFormat", message) +} + +/// Decodes the first audio stream of the file. The answer holds at most `max_ns` of sound. +#[cfg(windows)] +pub fn decode_file(path: &Path, max_ns: u64) -> IpcResult> { + use windows::{ + core::{Error, HSTRING}, + Win32::{ + Media::MediaFoundation::{ + IMFSample, MFAudioFormat_Float, MFCreateMediaType, MFCreateSourceReaderFromURL, MFMediaType_Audio, + MFShutdown, MFStartup, MFSTARTUP_FULL, MF_MT_AUDIO_AVG_BYTES_PER_SECOND, MF_MT_AUDIO_BITS_PER_SAMPLE, + MF_MT_AUDIO_BLOCK_ALIGNMENT, MF_MT_AUDIO_NUM_CHANNELS, MF_MT_AUDIO_SAMPLES_PER_SECOND, + MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_VERSION, + }, + System::Com::{CoInitializeEx, CoUninitialize, COINIT_MULTITHREADED}, + }, + }; + + /// The source reader's names for "the first audio stream" and "every stream". + const FIRST_AUDIO: u32 = 0xFFFF_FFFD; + const ALL_STREAMS: u32 = 0xFFFF_FFFE; + const END_OF_STREAM: u32 = 0x2; + + let max_samples = (max_ns / 1_000_000_000 * u64::from(RATE)) as usize; + // SAFETY: COM and Media Foundation start and stop on this thread around the work, and every object below is + // dropped before they stop. Each sample's buffer is locked only while its floats are copied out. + unsafe { + let apartment = CoInitializeEx(None, COINIT_MULTITHREADED); + let started = MFStartup(MF_VERSION, MFSTARTUP_FULL); + let result = (|| -> windows::core::Result> { + started.clone()?; + let reader = MFCreateSourceReaderFromURL(&HSTRING::from(path.as_os_str()), None)?; + reader.SetStreamSelection(ALL_STREAMS, false)?; + reader.SetStreamSelection(FIRST_AUDIO, true)?; + let wanted = MFCreateMediaType()?; + wanted.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Audio)?; + wanted.SetGUID(&MF_MT_SUBTYPE, &MFAudioFormat_Float)?; + wanted.SetUINT32(&MF_MT_AUDIO_NUM_CHANNELS, 1)?; + wanted.SetUINT32(&MF_MT_AUDIO_SAMPLES_PER_SECOND, RATE)?; + wanted.SetUINT32(&MF_MT_AUDIO_BITS_PER_SAMPLE, 32)?; + wanted.SetUINT32(&MF_MT_AUDIO_BLOCK_ALIGNMENT, 4)?; + wanted.SetUINT32(&MF_MT_AUDIO_AVG_BYTES_PER_SECOND, RATE * 4)?; + reader.SetCurrentMediaType(FIRST_AUDIO, None, &wanted)?; + let mut samples: Vec = Vec::new(); + loop { + let (mut flags, mut sample): (u32, Option) = (0, None); + reader.ReadSample(FIRST_AUDIO, 0, None, Some(&mut flags), None, Some(&mut sample))?; + if let Some(sample) = sample { + let buffer = sample.ConvertToContiguousBuffer()?; + let (mut data, mut length) = (std::ptr::null_mut::(), 0u32); + buffer.Lock(&mut data, None, Some(&mut length))?; + let floats = std::slice::from_raw_parts(data as *const f32, length as usize / 4); + samples.extend_from_slice(floats); + buffer.Unlock()?; + } + if flags & END_OF_STREAM != 0 || samples.len() >= max_samples { + break; + } + } + samples.truncate(max_samples); + Ok(samples) + })(); + if started.is_ok() { + let _ = MFShutdown(); + } + if apartment.is_ok() { + CoUninitialize(); + } + result.map_err(|error: Error| { + unreadable(&format!( + "Windows can't read the sound in this file ({}). The codec may be missing.", + error.message() + )) + }) + } +} + +#[cfg(all(test, windows))] +mod tests { + use super::*; + + /// A mono 16-bit WAV file of a tone, `seconds` long, at `rate`. + fn wav(rate: u32, seconds: u32) -> Vec { + let frames = rate * seconds; + let data = frames * 2; + let mut out = Vec::new(); + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&(36 + data).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16u32.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&rate.to_le_bytes()); + out.extend_from_slice(&(rate * 2).to_le_bytes()); + out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&data.to_le_bytes()); + for i in 0..frames { + let value = ((i as f32 * 0.05).sin() * 8000.0) as i16; + out.extend_from_slice(&value.to_le_bytes()); + } + out + } + + #[test] + fn a_wav_file_comes_back_as_48_khz_mono_samples() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("tone.wav"); + std::fs::write(&path, wav(16_000, 2)).unwrap(); + let samples = decode_file(&path, 60 * 1_000_000_000).unwrap(); + let seconds = samples.len() as f32 / RATE as f32; + assert!((seconds - 2.0).abs() < 0.1, "got {seconds} seconds"); + assert!(samples.iter().any(|sample| sample.abs() > 0.1)); + } + + #[test] + fn a_file_that_is_not_audio_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("notes.wav"); + std::fs::write(&path, b"this is not a sound").unwrap(); + assert!(decode_file(&path, 1_000_000_000).is_err()); + } +} diff --git a/app/src-tauri/src/audio_more/meeting.rs b/app/src-tauri/src/audio_more/meeting.rs new file mode 100644 index 00000000..9bcadab9 --- /dev/null +++ b/app/src-tauri/src/audio_more/meeting.rs @@ -0,0 +1,80 @@ +//! The watcher behind the meeting prompt. The interface asks every few seconds, passing the person's choices, and the +//! watcher answers with the app that has just started a call, once per call. It reads only which apps use the +//! microphone, never what is said, and reads nothing at all while the setting is off. + +use std::{ + sync::{Mutex, OnceLock, PoisonError}, + time::Instant, +}; + +use opennote_media::meeting::{MeetingSettings, MeetingWatcher, MicrophoneUsers}; +use serde::Serialize; + +use crate::ipc::{codes, IpcError, IpcResult}; + +/// The offer: the app whose call it is. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MeetingPrompt { + pub app: String, +} + +#[cfg(windows)] +fn users() -> Box { + Box::::default() +} + +/// Other systems have no record to read yet, so nothing is ever offered. +#[cfg(not(windows))] +fn users() -> Box { + struct Nobody; + impl MicrophoneUsers for Nobody { + fn current(&mut self) -> opennote_media::audio::Result> { + Ok(Vec::new()) + } + } + Box::new(Nobody) +} + +struct Watch { + watcher: MeetingWatcher, + started: Instant, +} + +fn watch() -> &'static Mutex { + static WATCH: OnceLock> = OnceLock::new(); + WATCH.get_or_init(|| { + let exe = std::env::current_exe() + .ok() + .and_then(|path| path.file_stem().map(|stem| stem.to_string_lossy().into_owned())) + .unwrap_or_default(); + let own = ["opennote", exe.as_str()]; + Mutex::new(Watch { + watcher: MeetingWatcher::new(MeetingSettings::default(), users(), &own), + started: Instant::now(), + }) + }) +} + +/// Looks at who uses the microphone. `enabled` and `never_for` are the person's choices, and `recording` says +/// whether a recording runs, which keeps the prompt quiet. +#[tauri::command] +pub async fn audio_meeting_poll( + enabled: bool, + never_for: Vec, + recording: bool, +) -> IpcResult> { + tauri::async_runtime::spawn_blocking(move || { + let mut guard = watch().lock().unwrap_or_else(PoisonError::into_inner); + guard.watcher.apply(MeetingSettings { enabled, never_for }); + guard.watcher.set_recording(recording); + let now = u64::try_from(guard.started.elapsed().as_nanos()).unwrap_or(u64::MAX); + guard + .watcher + .poll(now) + .map(|prompt| prompt.map(|prompt| MeetingPrompt { app: prompt.app })) + .map_err(crate::audio::audio_error) + }) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))? +} diff --git a/app/src-tauri/src/audio_more/mod.rs b/app/src-tauri/src/audio_more/mod.rs new file mode 100644 index 00000000..33b609d2 --- /dev/null +++ b/app/src-tauri/src/audio_more/mod.rs @@ -0,0 +1,465 @@ +//! The commands of the audio lane's later features (quality of life after Phase 9). +//! +//! They split a recording, make a voice-enhanced copy, and compress a recording. They list what recordings take and +//! export a recording as audio. They turn a dropped audio or video file into a recording. They snap the screen and +//! run the meeting prompt's watcher. `audio.rs` has the recording and playback commands. The media crate pieces +//! that these use are described in `crates/media/src`. +//! +//! An edit writes new files and leaves the old ones, as in `audio.rs`: the interface saves the page with the new entry +//! and then calls `audio_delete_files` for the old one. + +mod decode; +pub mod meeting; +pub mod snap; +mod video; + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use opennote_core::{ + session::notebook::HistoryScope, store::layout::NotebookLayout, store::layout::ASSETS_DIR, PageId, +}; +use opennote_media::{ + audio::{ClockAnchor, TrackKind}, + convert::{self, Quality, Settings}, + edit, + layout::{RecordingEntry, RecordingPlan}, + positions::PositionMap, + storage, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tauri::{ipc::InvokeBody, State}; + +use crate::{ + audio::{audio_error, blocking, checked_dir, edited, AudioState, Edited}, + core_bridge::CoreBridge, + images::import::percent_decode, + ipc::{codes, IpcError, IpcResult}, + page_export::ExportGrants, +}; + +fn quality_of(name: &str) -> Quality { + if name == "smallest" { + Quality::Smallest + } else { + Quality::Smaller + } +} + +/// A recording split in two. The first half keeps the recording's ID, and the second is a new recording. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Split { + pub first: Edited, + pub second: Edited, +} + +/// Splits a recording at a position, in nanoseconds into the audio. +#[tauri::command] +pub async fn audio_split( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + assets_dir: String, + entry: RecordingEntry, + at_ns: u64, +) -> IpcResult { + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + blocking(&state, move |state| { + let services = state.services(); + let summary = entry.summary().map_err(audio_error)?; + let kinds: Vec = summary.tracks.iter().map(|track| track.kind).collect(); + let first = RecordingPlan::replacing(&summary, services.session.as_ref()); + let second = RecordingPlan::generate(&kinds, services.session.as_ref()); + let [a, b] = edit::split(&dir, &summary, at_ns, [&first, &second]).map_err(audio_error)?; + Ok(Split { + first: edited(&a, &entry), + second: edited(&b, &entry), + }) + }) + .await +} + +/// A copy of the recording with the noise reduced and the voice leveled. The tracks of the answer are the copy's; +/// the interface keeps them beside the original's, and the original stays. +#[tauri::command] +pub async fn audio_enhance( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + assets_dir: String, + entry: RecordingEntry, +) -> IpcResult { + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + blocking(&state, move |state| { + let services = state.services(); + let summary = entry.summary().map_err(audio_error)?; + let plan = RecordingPlan::replacing(&summary, services.session.as_ref()); + let copy = convert::enhance( + &dir, + &summary, + &plan, + &services.decoder, + &services.encoder, + Settings::default(), + ) + .map_err(audio_error)?; + Ok(edited(©, &entry)) + }) + .await +} + +/// A smaller copy of the recording that takes the original's place. +#[tauri::command] +pub async fn audio_compress( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + assets_dir: String, + entry: RecordingEntry, + quality: String, +) -> IpcResult { + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + blocking(&state, move |state| { + let services = state.services(); + let summary = entry.summary().map_err(audio_error)?; + let plan = RecordingPlan::replacing(&summary, services.session.as_ref()); + let copy = + convert::compress(&dir, &summary, &plan, &services.decoder, quality_of(&quality)).map_err(audio_error)?; + Ok(edited(©, &entry)) + }) + .await +} + +/// One recording in the storage list. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Stored { + pub notebook: String, + pub section: String, + pub page: String, + pub page_title: String, + pub assets_dir: String, + pub entry: RecordingEntry, + pub duration_ns: u64, + /// The audio and timeline files together. + pub bytes: u64, + pub frees_smaller: u64, + pub frees_smallest: u64, +} + +/// What every recording of every open notebook takes, largest first. Pages in locked sections are left out. +#[tauri::command] +pub async fn audio_storage_scan(state: State<'_, AudioState>, bridge: State<'_, CoreBridge>) -> IpcResult> { + let notebooks = bridge.run(|bridge| Ok(bridge.notebooks())).await?; + blocking(&state, move |_| { + let mut found = Vec::new(); + for notebook in notebooks { + let tree = notebook.tree(); + let layout = NotebookLayout::new(notebook.path()); + for section in tree.sections.iter().filter(|section| !section.encrypted) { + for page in §ion.pages { + let page_dir = layout.page_dir(section.id, page.id); + let assets = page_dir.join(ASSETS_DIR); + if !assets.is_dir() { + continue; + } + for entry in recordings_of(&page_dir) { + let Ok(summary) = entry.summary() else { continue }; + let usage = storage::usage(&assets, &summary); + found.push(Stored { + notebook: tree.title.clone(), + section: section.title.clone(), + page: page.id.to_string(), + page_title: page.title.clone(), + assets_dir: assets.to_string_lossy().into_owned(), + duration_ns: usage.duration_ns, + bytes: usage.total_bytes, + frees_smaller: storage::space_freed_by_compressing(&assets, &summary, Quality::Smaller), + frees_smallest: storage::space_freed_by_compressing(&assets, &summary, Quality::Smallest), + entry, + }); + } + } + } + } + found.sort_by_key(|stored| std::cmp::Reverse(stored.bytes)); + Ok(found) + }) + .await +} + +/// The finished recordings in a page's `page.json`, which keeps them in its view. +fn recordings_of(page_dir: &Path) -> Vec { + let Ok(text) = fs::read_to_string(NotebookLayout::page_json(page_dir)) else { + return Vec::new(); + }; + let Ok(json) = serde_json::from_str::(&text) else { + return Vec::new(); + }; + json.pointer("/view/recordings") + .and_then(Value::as_object) + .map(|held| { + held.values() + .filter_map(|value| serde_json::from_value::(value.clone()).ok()) + .filter(|entry| !entry.tracks.is_empty()) + .collect() + }) + .unwrap_or_default() +} + +/// Deletes a page's saved versions, so audio taken out of the page can't be brought back from them. Named versions go +/// too, because one of them may hold the part that was removed. +#[tauri::command] +pub async fn audio_purge_history(bridge: State<'_, CoreBridge>, page: String) -> IpcResult { + let id = PageId::parse(&page).map_err(|_| IpcError::invalid("page", "That isn't a page ID."))?; + let notebook = bridge + .run(move |bridge| { + bridge.with(|bridge| { + bridge + .core + .find_node(id.0) + .map(|(notebook, _)| notebook) + .ok_or_else(|| IpcError::new("notFound", "That page isn't in a notebook.")) + }) + }) + .await?; + tauri::async_runtime::spawn_blocking(move || { + notebook + .delete_history(HistoryScope::Page(id), false) + .map(|deleted| deleted.versions) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string())) + }) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))? +} + +/// Saves a recording as one audio file at a path that `export_pick_save` chose. The format is `wav` or `opus`. It +/// answers with the size of the file. +#[tauri::command] +pub async fn audio_export( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + grants: State<'_, ExportGrants>, + assets_dir: String, + entry: RecordingEntry, + dest: String, + format: String, +) -> IpcResult { + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + let dest = PathBuf::from(dest); + if !grants.take(&dest) { + return Err(IpcError::invalid("dest", "Choose where to save before exporting.")); + } + let size = blocking(&state, move |state| { + let services = state.services(); + let summary = entry.summary().map_err(audio_error)?; + // The file is written beside the chosen one and moved over it, so a failure leaves the old file alone. + let temp = dest.with_extension("opennote-part"); + let _ = fs::remove_file(&temp); + let written = match format.as_str() { + "wav" => convert::export_wav(&dir, &summary, &services.decoder, &temp), + "opus" => convert::export_opus(&dir, &summary, &services.decoder, &services.encoder, &temp), + _ => { + return Err(IpcError::invalid( + "format", + "That isn't an audio format OpenNote writes.", + )) + } + } + .map_err(audio_error)?; + fs::rename(&temp, &dest).inspect_err(|_| { + let _ = fs::remove_file(&temp); + })?; + Ok((written, dest)) + }) + .await?; + grants.mark_written(size.1); + Ok(size.0) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct VideoHeader { + assets_dir: String, + entry: RecordingEntry, + /// Where to save it, which `export_pick_save` chose. Empty when `transfer` is set. + #[serde(default)] + dest: String, + /// A file name in the accounts transfer folder, for a video that is uploaded and not saved by the person. + #[serde(default)] + transfer: Option, + width: u32, + height: u32, + /// How many bytes of the body are the strokes' JSON. The picture's RGBA rows follow. + strokes_len: usize, +} + +/// The longest recording that is made into a video: the picture is drawn for every frame. +const MAX_VIDEO_NS: u64 = 3 * 3_600 * 1_000_000_000; + +/// Saves a recording as a narrated video at a path that `export_pick_save` chose: the page as a picture, the ink drawn +/// again as it was drawn, and the recording's sound, as an MP4 file. The body is the strokes' JSON and then the page's +/// RGBA rows; the `x-opennote-video` header (percent-encoded JSON) says where they divide and what to save. It answers +/// with the size of the file. +#[tauri::command] +pub async fn audio_export_video( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + grants: State<'_, ExportGrants>, + paths: State<'_, crate::paths::Paths>, + request: tauri::ipc::Request<'_>, +) -> IpcResult { + let header = request + .headers() + .get("x-opennote-video") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| IpcError::invalid("x-opennote-video", "The video header is missing."))?; + let header: VideoHeader = serde_json::from_str(&percent_decode(header)) + .map_err(|_| IpcError::invalid("x-opennote-video", "The video header isn't valid JSON."))?; + let body = match request.body() { + InvokeBody::Raw(bytes) => bytes.clone(), + InvokeBody::Json(_) => return Err(IpcError::invalid("body", "The page must come as raw bytes.")), + }; + let (dir, _) = bridge.with(|bridge| checked_dir(bridge, &header.assets_dir))?; + let transfer = header.transfer.is_some(); + let dest = match &header.transfer { + Some(name) => { + let plain = !name.is_empty() + && name.len() <= 80 + && !name.starts_with('.') + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')); + if !plain { + return Err(IpcError::invalid("transfer", "That isn't a file name.")); + } + let folder = paths.local.join(crate::connectors::FOLDER).join("tmp"); + fs::create_dir_all(&folder)?; + folder.join(name) + } + None => { + let dest = PathBuf::from(&header.dest); + if !grants.take(&dest) { + return Err(IpcError::invalid("dest", "Choose where to save before exporting.")); + } + dest + } + }; + if body.len() < header.strokes_len { + return Err(IpcError::invalid("body", "The page's strokes are cut short.")); + } + let (strokes, rgba) = body.split_at(header.strokes_len); + let strokes: Vec = serde_json::from_slice(strokes) + .map_err(|_| IpcError::invalid("strokes", "The page's strokes aren't valid."))?; + let mut scene = + opennote_media::video::Scene::from_rgba(header.width, header.height, rgba, strokes).map_err(audio_error)?; + let size = blocking(&state, move |state| { + let services = state.services(); + let summary = header.entry.summary().map_err(audio_error)?; + let map = PositionMap::from_summary(&summary); + let duration = map.duration_ns(); + if duration > MAX_VIDEO_NS { + return Err(IpcError::invalid( + "entry", + "That recording is too long to make into a video.", + )); + } + scene.place(&map); + let player = + opennote_media::playback::open_recording(&dir, &summary, &services.decoder).map_err(audio_error)?; + let mut pcm = opennote_media::transcribe::MixedPcm::new(player); + let temp = dest.with_extension("opennote-part.mp4"); + let _ = fs::remove_file(&temp); + video::write_mp4(&temp, &scene, &mut pcm, duration, &mut |_| {})?; + fs::rename(&temp, &dest).inspect_err(|_| { + let _ = fs::remove_file(&temp); + })?; + Ok((fs::metadata(&dest)?.len(), dest)) + }) + .await?; + if !transfer { + grants.mark_written(size.1); + } + Ok(size.0) +} + +/// What an imported file made. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Imported { + pub entry: RecordingEntry, + pub duration_ns: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ImportHeader { + assets_dir: String, + name: String, +} + +/// The longest file that is turned into a recording: four hours of 48 kHz samples take 2.7 GB as floats. +const MAX_IMPORT_NS: u64 = 4 * 3_600 * 1_000_000_000; + +/// Makes a recording of an audio or video file's sound. The body is the file's bytes and the `x-opennote-import` +/// header (percent-encoded JSON) names the page's `assets` folder and the file. The file is written beside the page's +/// recordings only while it is decoded. +#[tauri::command] +pub async fn audio_import_file( + state: State<'_, AudioState>, + bridge: State<'_, CoreBridge>, + request: tauri::ipc::Request<'_>, +) -> IpcResult { + let header = request + .headers() + .get("x-opennote-import") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| IpcError::invalid("x-opennote-import", "The import header is missing."))?; + let header: ImportHeader = serde_json::from_str(&percent_decode(header)) + .map_err(|_| IpcError::invalid("x-opennote-import", "The import header isn't valid JSON."))?; + let body = match request.body() { + InvokeBody::Raw(bytes) => bytes.clone(), + InvokeBody::Json(_) => return Err(IpcError::invalid("body", "The file must come as raw bytes.")), + }; + let assets_dir = header.assets_dir.clone(); + let (dir, _) = bridge + .run(move |bridge| bridge.with(|bridge| checked_dir(bridge, &assets_dir))) + .await?; + blocking(&state, move |state| { + let services = state.services(); + let extension: String = Path::new(&header.name) + .extension() + .map(|ext| { + ext.to_string_lossy() + .chars() + .filter(char::is_ascii_alphanumeric) + .take(5) + .collect() + }) + .unwrap_or_default(); + let temp = dir.join(format!(".import-{}.{extension}", std::process::id())); + fs::write(&temp, &body)?; + let decoded = decode::decode_file(&temp, MAX_IMPORT_NS); + let _ = fs::remove_file(&temp); + let samples = decoded?; + let plan = RecordingPlan::generate(&[TrackKind::Microphone], services.session.as_ref()); + let wall = opennote_media::layout::wall_clock(services.session.clone()); + let clock = ClockAnchor::now(services.clock.as_ref(), wall.as_ref()); + let summary = convert::import_pcm(&dir, &plan, samples, clock, clock.capture_ns, &services.encoder) + .map_err(audio_error)?; + Ok(Imported { + entry: RecordingEntry::from_summary(&summary, None), + duration_ns: PositionMap::from_summary(&summary).duration_ns(), + }) + }) + .await +} diff --git a/app/src-tauri/src/audio_more/snap.rs b/app/src-tauri/src/audio_more/snap.rs new file mode 100644 index 00000000..21de7e42 --- /dev/null +++ b/app/src-tauri/src/audio_more/snap.rs @@ -0,0 +1,206 @@ +//! Snapping the screen while recording. The interface asks for a capture when the person presses the shortcut, and +//! nothing is captured before that. OpenNote steps out of the way for a moment, so the picture is of what was behind +//! it, such as a slide or a whiteboard in a call, and then comes back. The picture is a PNG. + +use tauri::{ipc::Response, WebviewWindow}; + +use crate::ipc::{codes, IpcError, IpcResult}; + +/// How long OpenNote waits after it minimizes, so Windows has drawn what was behind it. +#[cfg(windows)] +const SETTLE: std::time::Duration = std::time::Duration::from_millis(350); + +/// Captures the whole screen (`screen`, which a region is cut from) or the window that was behind OpenNote (`window`). +/// The answer is the PNG's bytes. +#[tauri::command] +pub async fn audio_snap(window: WebviewWindow, kind: String) -> IpcResult { + if kind != "screen" && kind != "window" { + return Err(IpcError::invalid("kind", "A snap is of the screen or of a window.")); + } + let _ = window.minimize(); + let taken = tauri::async_runtime::spawn_blocking(move || { + #[cfg(windows)] + std::thread::sleep(SETTLE); + capture(&kind) + }) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string())); + let _ = window.unminimize(); + let _ = window.set_focus(); + Ok(Response::new(taken??)) +} + +#[cfg(not(windows))] +fn capture(_kind: &str) -> IpcResult> { + Err(IpcError::not_implemented("Snapping the screen")) +} + +#[cfg(windows)] +fn capture(kind: &str) -> IpcResult> { + use windows::Win32::{ + Foundation::{HWND, RECT}, + Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS}, + UI::WindowsAndMessaging::{ + GetForegroundWindow, GetSystemMetrics, GetWindowRect, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, + SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, + }, + }; + + // SAFETY: the metrics and the window rectangle are plain reads into values owned here. + let area = unsafe { + let screen = RECT { + left: GetSystemMetrics(SM_XVIRTUALSCREEN), + top: GetSystemMetrics(SM_YVIRTUALSCREEN), + right: GetSystemMetrics(SM_XVIRTUALSCREEN) + GetSystemMetrics(SM_CXVIRTUALSCREEN), + bottom: GetSystemMetrics(SM_YVIRTUALSCREEN) + GetSystemMetrics(SM_CYVIRTUALSCREEN), + }; + let behind: HWND = GetForegroundWindow(); + if kind == "window" && !behind.0.is_null() { + let mut frame = RECT::default(); + let bounds = DwmGetWindowAttribute( + behind, + DWMWA_EXTENDED_FRAME_BOUNDS, + std::ptr::from_mut(&mut frame).cast(), + std::mem::size_of::() as u32, + ); + if bounds.is_err() { + let _ = GetWindowRect(behind, &mut frame); + } + // Only what is on the screen. + RECT { + left: frame.left.max(screen.left), + top: frame.top.max(screen.top), + right: frame.right.min(screen.right), + bottom: frame.bottom.min(screen.bottom), + } + } else { + screen + } + }; + let (width, height) = (area.right - area.left, area.bottom - area.top); + if width <= 0 || height <= 0 { + return Err(IpcError::new("audioSnap", "There is nothing on the screen to capture.")); + } + let pixels = grab(area.left, area.top, width, height).map_err(snap_failed)?; + encode_png(&pixels, width as u32, height as u32).map_err(snap_failed) +} + +#[cfg(windows)] +fn snap_failed(error: windows::core::Error) -> IpcError { + IpcError::new( + "audioSnap", + format!("The screen couldn’t be captured: {}", error.message()), + ) +} + +/// The pixels of a rectangle of the screen, as top-down BGRA with every pixel opaque. +#[cfg(windows)] +fn grab(x: i32, y: i32, width: i32, height: i32) -> windows::core::Result> { + use windows::Win32::Graphics::Gdi::{ + BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDC, GetDIBits, ReleaseDC, + SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, CAPTUREBLT, DIB_RGB_COLORS, SRCCOPY, + }; + + // SAFETY: the device contexts and the bitmap are made, used, and released within this call, in reverse order. + // GetDIBits fills `pixels`, which holds exactly width * height * 4 bytes. + unsafe { + let screen = GetDC(None); + let memory = CreateCompatibleDC(Some(screen)); + let bitmap = CreateCompatibleBitmap(screen, width, height); + let previous = SelectObject(memory, bitmap.into()); + let copied = BitBlt(memory, 0, 0, width, height, Some(screen), x, y, SRCCOPY | CAPTUREBLT); + let mut info = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: std::mem::size_of::() as u32, + biWidth: width, + biHeight: -height, + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + ..Default::default() + }, + ..Default::default() + }; + let mut pixels = vec![0u8; width as usize * height as usize * 4]; + let lines = GetDIBits( + memory, + bitmap, + 0, + height as u32, + Some(pixels.as_mut_ptr().cast()), + &mut info, + DIB_RGB_COLORS, + ); + SelectObject(memory, previous); + let _ = DeleteObject(bitmap.into()); + let _ = DeleteDC(memory); + ReleaseDC(None, screen); + copied?; + if lines == 0 { + return Err(windows::core::Error::from_thread()); + } + for pixel in pixels.as_chunks_mut::<4>().0 { + pixel[3] = 255; + } + Ok(pixels) + } +} + +/// A PNG of BGRA pixels, made by Windows Imaging Component. +#[cfg(windows)] +pub(crate) fn encode_png(pixels: &[u8], width: u32, height: u32) -> windows::core::Result> { + use windows::Win32::{ + Foundation::HGLOBAL, + Graphics::Imaging::{ + CLSID_WICImagingFactory, GUID_ContainerFormatPng, GUID_WICPixelFormat32bppBGRA, IWICImagingFactory, + WICBitmapEncoderNoCache, + }, + System::{ + Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, + StructuredStorage::{CreateStreamOnHGlobal, GetHGlobalFromStream}, + CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, STREAM_SEEK_SET, + }, + Memory::{GlobalLock, GlobalSize, GlobalUnlock}, + }, + }; + + // SAFETY: COM starts and stops around the work. The bitmap copies `pixels` before it returns, and the output + // memory is read only while it is locked and before its stream is released. + unsafe { + let apartment = CoInitializeEx(None, COINIT_MULTITHREADED); + let result = (|| -> windows::core::Result> { + let factory: IWICImagingFactory = CoCreateInstance(&CLSID_WICImagingFactory, None, CLSCTX_INPROC_SERVER)?; + let bitmap = + factory.CreateBitmapFromMemory(width, height, &GUID_WICPixelFormat32bppBGRA, width * 4, pixels)?; + let output = CreateStreamOnHGlobal(HGLOBAL::default(), true)?; + let encoder = factory.CreateEncoder(&GUID_ContainerFormatPng, std::ptr::null())?; + encoder.Initialize(&output, WICBitmapEncoderNoCache)?; + let mut target = None; + let mut options = None; + encoder.CreateNewFrame(&mut target, &mut options)?; + let target = target.ok_or_else(windows::core::Error::empty)?; + target.Initialize(options.as_ref())?; + target.SetSize(width, height)?; + let mut format = GUID_WICPixelFormat32bppBGRA; + target.SetPixelFormat(&mut format)?; + target.WriteSource(&bitmap, std::ptr::null())?; + target.Commit()?; + encoder.Commit()?; + output.Seek(0, STREAM_SEEK_SET, None)?; + let memory = GetHGlobalFromStream(&output)?; + let size = GlobalSize(memory); + let start = GlobalLock(memory) as *const u8; + if start.is_null() { + return Err(windows::core::Error::from_thread()); + } + let bytes = std::slice::from_raw_parts(start, size).to_vec(); + let _ = GlobalUnlock(memory); + Ok(bytes) + })(); + if apartment.is_ok() { + CoUninitialize(); + } + result + } +} diff --git a/app/src-tauri/src/audio_more/video.rs b/app/src-tauri/src/audio_more/video.rs new file mode 100644 index 00000000..1f367ea6 --- /dev/null +++ b/app/src-tauri/src/audio_more/video.rs @@ -0,0 +1,267 @@ +//! Writing the narrated video as an MP4 file with the encoders Windows has: H.264 for the picture and AAC for the +//! sound, through Media Foundation's sink writer. The media crate draws the frames and reads the audio (see +//! `opennote_media::video`), and this is the sink they go to. A PC without the encoders (the "N" editions of Windows +//! lack the media pack) says so, and no file is left behind. + +use std::path::Path; + +use opennote_media::{ + transcribe::PcmSource, + video::{self, Scene}, +}; + +use crate::ipc::{IpcError, IpcResult}; + +/// The frames a second of the video. +pub const FPS: u32 = 15; + +#[cfg(not(windows))] +pub fn write_mp4( + _path: &Path, + _scene: &Scene, + _pcm: &mut dyn PcmSource, + _duration_ns: u64, + _progress: &mut dyn FnMut(f32), +) -> IpcResult<()> { + Err(IpcError::not_implemented("Exporting a recording as a video")) +} + +#[cfg(windows)] +mod mf { + use opennote_media::audio::{AudioError, Result}; + use windows::{ + core::{Error, GUID, HSTRING}, + Win32::Media::MediaFoundation::{ + IMFMediaType, IMFSample, IMFSinkWriter, MFAudioFormat_AAC, MFAudioFormat_PCM, MFCreateMediaType, + MFCreateMemoryBuffer, MFCreateSample, MFCreateSinkWriterFromURL, MFMediaType_Audio, MFMediaType_Video, + MFVideoFormat_H264, MFVideoFormat_RGB32, MFVideoInterlace_Progressive, + MF_MT_AAC_AUDIO_PROFILE_LEVEL_INDICATION, MF_MT_AUDIO_AVG_BYTES_PER_SECOND, MF_MT_AUDIO_BITS_PER_SAMPLE, + MF_MT_AUDIO_BLOCK_ALIGNMENT, MF_MT_AUDIO_NUM_CHANNELS, MF_MT_AUDIO_SAMPLES_PER_SECOND, MF_MT_AVG_BITRATE, + MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, + MF_MT_SUBTYPE, + }, + }; + + pub struct Mp4Sink { + writer: IMFSinkWriter, + video_stream: u32, + audio_stream: u32, + width: usize, + height: usize, + flipped: Vec, + source_stride: usize, + frame_ns: u64, + } + + fn failed(error: Error) -> AudioError { + AudioError::Encoder(format!("Windows can't write this video ({}).", error.message())) + } + + fn pair(high: u32, low: u32) -> u64 { + (u64::from(high) << 32) | u64::from(low) + } + + fn video_type( + subtype: &GUID, + width: u32, + height: u32, + fps: u32, + bitrate: Option, + ) -> windows::core::Result { + // SAFETY: plain calls on a media type this function made. + unsafe { + let media = MFCreateMediaType()?; + media.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + media.SetGUID(&MF_MT_SUBTYPE, subtype)?; + if let Some(bitrate) = bitrate { + media.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; + } + media.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + media.SetUINT64(&MF_MT_FRAME_SIZE, pair(width, height))?; + media.SetUINT64(&MF_MT_FRAME_RATE, pair(fps, 1))?; + media.SetUINT64(&MF_MT_PIXEL_ASPECT_RATIO, pair(1, 1))?; + Ok(media) + } + } + + fn audio_type(subtype: &GUID, bytes_per_second: u32, bits: u32, align: u32) -> windows::core::Result { + // SAFETY: plain calls on a media type this function made. + unsafe { + let media = MFCreateMediaType()?; + media.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Audio)?; + media.SetGUID(&MF_MT_SUBTYPE, subtype)?; + media.SetUINT32(&MF_MT_AUDIO_NUM_CHANNELS, 1)?; + media.SetUINT32(&MF_MT_AUDIO_SAMPLES_PER_SECOND, 48_000)?; + media.SetUINT32(&MF_MT_AUDIO_BITS_PER_SAMPLE, bits)?; + media.SetUINT32(&MF_MT_AUDIO_BLOCK_ALIGNMENT, align)?; + media.SetUINT32(&MF_MT_AUDIO_AVG_BYTES_PER_SECOND, bytes_per_second)?; + Ok(media) + } + } + + impl Mp4Sink { + pub fn create(path: &std::path::Path, width: u32, height: u32, fps: u32) -> Result { + // H.264 needs even sides. + let source_width = width; + let (width, height) = (width & !1, height & !1); + // SAFETY: the writer and the types are COM objects owned by this value or dropped here. + unsafe { + let writer = MFCreateSinkWriterFromURL(&HSTRING::from(path.as_os_str()), None, None).map_err(failed)?; + let bitrate = (width * height * 3).clamp(800_000, 8_000_000); + let video_out = video_type(&MFVideoFormat_H264, width, height, fps, Some(bitrate)).map_err(failed)?; + let video_stream = writer.AddStream(&video_out).map_err(failed)?; + let video_in = video_type(&MFVideoFormat_RGB32, width, height, fps, None).map_err(failed)?; + writer + .SetInputMediaType(video_stream, &video_in, None) + .map_err(failed)?; + let audio_out = audio_type(&MFAudioFormat_AAC, 12_000, 16, 1).map_err(failed)?; + // AAC-LC at level 2. + audio_out + .SetUINT32(&MF_MT_AAC_AUDIO_PROFILE_LEVEL_INDICATION, 0x29) + .map_err(failed)?; + let audio_stream = writer.AddStream(&audio_out).map_err(failed)?; + let audio_in = audio_type(&MFAudioFormat_PCM, 96_000, 16, 2).map_err(failed)?; + writer + .SetInputMediaType(audio_stream, &audio_in, None) + .map_err(failed)?; + writer.BeginWriting().map_err(failed)?; + Ok(Mp4Sink { + writer, + video_stream, + audio_stream, + width: width as usize, + height: height as usize, + flipped: vec![0; width as usize * height as usize * 4], + source_stride: source_width as usize * 4, + frame_ns: 1_000_000_000 / u64::from(fps), + }) + } + } + + fn sample(&self, bytes: &[u8], at_ns: u64, duration_ns: u64) -> windows::core::Result { + // SAFETY: the buffer is locked only while the bytes are copied in, and its length is set to what was copied. + unsafe { + let buffer = MFCreateMemoryBuffer(bytes.len() as u32)?; + let (mut data, mut max) = (std::ptr::null_mut::(), 0u32); + buffer.Lock(&mut data, Some(&mut max), None)?; + std::ptr::copy_nonoverlapping(bytes.as_ptr(), data, bytes.len().min(max as usize)); + buffer.Unlock()?; + buffer.SetCurrentLength(bytes.len() as u32)?; + let sample = MFCreateSample()?; + sample.AddBuffer(&buffer)?; + sample.SetSampleTime((at_ns / 100) as i64)?; + sample.SetSampleDuration((duration_ns / 100) as i64)?; + Ok(sample) + } + } + } + + impl opennote_media::video::VideoSink for Mp4Sink { + fn frame(&mut self, bgra: &[u8], at_ns: u64) -> Result<()> { + // RGB32 is stored with the bottom row first. The scene may be a pixel wider or taller than the video. + let stride = self.width * 4; + for row in 0..self.height { + let from = row * self.source_stride; + let to = (self.height - 1 - row) * stride; + self.flipped[to..to + stride].copy_from_slice(&bgra[from..from + stride]); + } + let sample = self.sample(&self.flipped, at_ns, self.frame_ns).map_err(failed)?; + // SAFETY: the writer is live until `finish`. + unsafe { self.writer.WriteSample(self.video_stream, &sample) }.map_err(failed) + } + + fn audio(&mut self, samples: &[i16], at_ns: u64) -> Result<()> { + let mut bytes = Vec::with_capacity(samples.len() * 2); + for sample in samples { + bytes.extend_from_slice(&sample.to_le_bytes()); + } + let duration = samples.len() as u64 * 1_000_000_000 / 48_000; + let sample = self.sample(&bytes, at_ns, duration).map_err(failed)?; + // SAFETY: the writer is live until `finish`. + unsafe { self.writer.WriteSample(self.audio_stream, &sample) }.map_err(failed) + } + + fn finish(&mut self) -> Result<()> { + // SAFETY: the writer is live; this ends it. + unsafe { self.writer.Finalize() }.map_err(failed) + } + } +} + +/// Writes the video to `path`, which must not exist. A failed or partial file is removed. +#[cfg(windows)] +pub fn write_mp4( + path: &Path, + scene: &Scene, + pcm: &mut dyn PcmSource, + duration_ns: u64, + progress: &mut dyn FnMut(f32), +) -> IpcResult<()> { + use windows::Win32::{ + Media::MediaFoundation::{MFShutdown, MFStartup, MFSTARTUP_FULL, MF_VERSION}, + System::Com::{CoInitializeEx, CoUninitialize, COINIT_MULTITHREADED}, + }; + + // SAFETY: COM and Media Foundation start and stop on this thread around the work, and the sink is dropped first. + unsafe { + let apartment = CoInitializeEx(None, COINIT_MULTITHREADED); + let started = MFStartup(MF_VERSION, MFSTARTUP_FULL); + let result = (|| -> opennote_media::audio::Result<()> { + started + .clone() + .map_err(|error| opennote_media::audio::AudioError::Encoder(error.message()))?; + let mut sink = mf::Mp4Sink::create(path, scene.width, scene.height, FPS)?; + video::export(scene, pcm, duration_ns, FPS, &mut sink, progress) + })(); + if started.is_ok() { + let _ = MFShutdown(); + } + if apartment.is_ok() { + CoUninitialize(); + } + if let Err(error) = result { + let _ = std::fs::remove_file(path); + return Err(IpcError::new("videoEncoder", error.to_string())); + } + } + Ok(()) +} + +#[cfg(all(test, windows))] +mod tests { + use super::*; + use opennote_media::{ + convert::MemoryPcm, + video::{ScenePoint, SceneStroke}, + }; + + #[test] + fn a_short_recording_becomes_an_mp4_file() { + let (width, height) = (320u32, 240u32); + let rgba = vec![250u8; (width * height * 4) as usize]; + let stroke = SceneStroke { + color: [10, 30, 200, 255], + width: 5.0, + points: (0..20) + .map(|i| ScenePoint { + x: 20.0 + i as f32 * 12.0, + y: 60.0 + (i as f32 * 0.5).sin() * 20.0, + t_ns: i * 50_000_000, + }) + .collect(), + }; + let scene = Scene::from_rgba(width, height, &rgba, vec![stroke]).unwrap(); + let tone: Vec = (0..96_000).map(|i| (i as f32 * 0.05).sin() * 0.3).collect(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("out.mp4"); + let mut last = 0.0; + write_mp4(&path, &scene, &mut MemoryPcm::new(tone), 2_000_000_000, &mut |share| { + last = share + }) + .unwrap(); + let bytes = std::fs::read(&path).unwrap(); + assert!(bytes.len() > 2_000, "only {} bytes", bytes.len()); + assert_eq!(&bytes[4..8], b"ftyp", "an MP4 file starts with its type box"); + assert_eq!(last, 1.0); + } +} diff --git a/app/src-tauri/src/build_rules.rs b/app/src-tauri/src/build_rules.rs new file mode 100644 index 00000000..abae3102 --- /dev/null +++ b/app/src-tauri/src/build_rules.rs @@ -0,0 +1,61 @@ +// The build script's rule for update keys, in a file of its own so the app's unit tests can check it: build.rs +// includes this file, and lib.rs compiles it for its tests only. +// +// A build with Cargo's `release` profile is what `tauri build` makes and what the release workflow ships. Without +// an update public key in `keys/*.pub` it would ship an updater that can never install anything, so the build stops +// and says what to do. CI's own builds use the `ci` profile with test endpoints, and a dry run of the release +// workflow sets `OPENNOTE_DRY_RUN=1`, because nothing it builds is published. + +/// The Cargo profile of a build, read from its `OUT_DIR`: `target//build//out`, or +/// `target///build//out` for a cross build. +pub fn profile_from_out_dir(out_dir: &std::path::Path) -> Option { + let parts: Vec = out_dir + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect(); + let build = parts.iter().rposition(|part| part == "build")?; + parts.get(build.checked_sub(1)?).cloned() +} + +/// Why this build must stop, or `None` when it may go on. +pub fn release_key_problem(profile: &str, key_count: usize, test_endpoints: bool, dry_run: bool) -> Option { + if profile != "release" || key_count > 0 || test_endpoints || dry_run { + return None; + } + Some( + "A release build needs an update public key in app/src-tauri/keys/*.pub, or its updater could never install \ + an update. Commit the public key as docs/RELEASING.md says (Create the update signing key). A dry run of \ + the release workflow sets OPENNOTE_DRY_RUN=1 instead." + .to_owned(), + ) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + + #[test] + fn reads_the_profile_from_the_out_dir() { + let plain = Path::new(r"C:\repo\target\release\build\opennote-123\out"); + assert_eq!(profile_from_out_dir(plain).as_deref(), Some("release")); + let cross = Path::new("/repo/target/aarch64-pc-windows-msvc/ci/build/opennote-1/out"); + assert_eq!(profile_from_out_dir(cross).as_deref(), Some("ci")); + assert_eq!(profile_from_out_dir(Path::new("/nowhere")), None); + } + + #[test] + fn only_a_release_build_without_keys_stops() { + assert!(release_key_problem("release", 0, false, false).is_some_and(|why| why.contains("keys/*.pub"))); + assert_eq!(release_key_problem("release", 1, false, false), None); + assert_eq!( + release_key_problem("release", 0, true, false), + None, + "CI's test-endpoints build" + ); + assert_eq!(release_key_problem("release", 0, false, true), None, "a dry run"); + assert_eq!(release_key_problem("debug", 0, false, false), None); + assert_eq!(release_key_problem("ci", 0, false, false), None); + } +} diff --git a/app/src-tauri/src/command_list.rs b/app/src-tauri/src/command_list.rs index 109b7651..7692159b 100644 --- a/app/src-tauri/src/command_list.rs +++ b/app/src-tauri/src/command_list.rs @@ -10,17 +10,51 @@ pub const APP_COMMANDS: &[&str] = &[ "state_update", "state_flush", "window_minimize", "window_toggle_maximize", "window_close", "window_set_title", "window_set_caption_layout", "window_show_system_menu", "window_set_frame_theme", - "app_first_paint", "app_ready", "app_exit_ready", + "app_first_paint", "app_ready", "app_exit_ready", "app_close_anyway", "perf_mark", "log_write", + "audio_assets_dir", "audio_devices", "audio_clock", "audio_prepare", "audio_begin", + "audio_recording_status", "audio_pause_recording", "audio_resume_recording", + "audio_switch_microphone", "audio_switch_system_audio", "audio_stop", "audio_recover", + "audio_open_playback", "audio_play", "audio_pause_playback", "audio_seek", "audio_skip", + "audio_set_speed", "audio_set_skip_silence", "audio_playback_status", "audio_close_playback", + "audio_trim_silence", "audio_remove_part", "audio_delete_files", "audio_adopt_tracks", + "audio_split", "audio_enhance", "audio_compress", "audio_storage_scan", "audio_purge_history", "audio_export", "audio_export_video", + "audio_import_file", "audio_meeting_poll", "audio_snap", + "crash_consent_get", "crash_consent_set", "crash_example", "crash_list", "crash_prepare", "crash_send", + "crash_delete", "crash_delete_all", "diagnostics_self_check", "diagnostics_build_feedback", + "diagnostics_save_feedback", "diagnostics_startup", "diagnostics_enter_safe_mode", "diagnostics_restart", + "privacy_get", "privacy_set_offline", "install_status", "install_pick_folder", "install_check_folder", "install_move_to_user_programs", "shell_open_external", "updater_status", "updater_check", "updater_download", "updater_restart_to_update", "updater_skip", "updater_unskip", "updater_go_back", "notes_snapshot_load", "notes_snapshot_save", + "notes_load_initial", "notes_list_notebooks", "notes_list_children", "notes_get", "notes_create", "notes_rename", + "notes_set_color", "notes_move", "notes_set_page_level", "notes_trash", "notes_restore", "notes_list_trash", + "notes_restore_from_trash", "notes_purge", "notes_status", "notes_flush", "page_open", "page_apply", "page_undo", "page_redo", "page_save_now", "page_close", - "history_list", "history_open", "history_restore", "history_restore_blocks", "history_name", + "history_list", "history_open", "history_restore", "history_restore_blocks", "history_name", "history_delete", + "page_add_strokes", "page_read_strokes", "clipboard_facts", "clipboard_read", "image_import", "image_import_url", "image_import_clip", "spell_languages", "spell_check", "spell_suggest", "spell_add_word", "spell_remove_word", "speech_voices", "speech_synthesize", + "print_prepare", "print_render", "print_close", "export_pick_save", "export_write", "export_open", "export_selection_docx", + "print_printers", "print_submit", + "search_call", "shellqol_call", + "tool_window_open", "tool_windows_reset", "study_anki_read", "study_anki_write", "study_zotero_items", "study_cite_lookup", + "study_zotero_run", "study_zotero_reply", "study_zotero_cancel", + "interop_pick", "interop_detect", "interop_local_sources", "interop_preview", "interop_import", "interop_cancel", + "interop_export", "interop_reveal", "interop_more", + "intel_settings_get", "intel_settings_set", "intel_status", "intel_ocr_languages", + "intel_ocr_recognize", "intel_ink_recognize", "intel_ink_tidy", "intel_summarize", + "intel_keywords", "intel_action_items", "intel_chapters", "intel_vocabulary_offer", + "intel_speech_voices", "intel_speech_synthesize", "intel_read_aloud_start", "intel_read_aloud_next", + "intel_read_aloud_cancel", "intel_clip_audio", "intel_ext_call", + "intel_transcribe", "intel_transcribe_cancel", + "intel_stream_start", "intel_stream_stop", "intel_caption_style", + "page_extras_caret", "page_extras_link_title", "attachment_import", "attachment_open", "attachment_applied", "attachment_thumbnail", + "connectors_list", "connectors_connect", "connectors_cancel", "connectors_disconnect", "connectors_request", + "api_call", "share_take", + "accounts_call", ]; #[cfg(test)] diff --git a/app/src-tauri/src/connectors/account.rs b/app/src-tauri/src/connectors/account.rs new file mode 100644 index 00000000..19585eef --- /dev/null +++ b/app/src-tauri/src/connectors/account.rs @@ -0,0 +1,116 @@ +//! The account name for "Connected as": read from the ID token the token endpoint sent, from the token answer, or +//! from one call with the new access token, as the registry says for each service. The name is for display and for +//! naming the credential, so it is cleaned and cut short. A name that can't be found is empty, and the connection +//! still works. + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use serde_json::Value; + +use super::{ + http::{HostPolicy, Http, HttpRequest}, + oauth::{Exchange, Tokens}, + registry::AccountFrom, +}; + +/// The longest account name kept, in characters. +const MAX_NAME: usize = 100; + +/// Drops control characters and surrounding space, and cuts a long name. +pub fn clean(name: &str) -> String { + name.chars() + .filter(|c| !c.is_control()) + .collect::() + .trim() + .chars() + .take(MAX_NAME) + .collect() +} + +/// A text value of a JSON answer, by pointer. +pub fn text_at(body: &Value, pointer: &str) -> Option { + let text = clean(body.pointer(pointer)?.as_str()?); + (!text.is_empty()).then_some(text) +} + +/// The sign-in name in an ID token's payload. The token came straight from the token endpoint over HTTPS, so its +/// signature isn't checked (OpenID Connect Core section 3.1.3.7); it is read for display only. +fn from_id_token(body: &Value) -> Option { + let token = body.get("id_token")?.as_str()?; + let payload = token.split('.').nth(1)?; + let bytes = URL_SAFE_NO_PAD.decode(payload.trim_end_matches('=')).ok()?; + let claims: Value = serde_json::from_slice(&bytes).ok()?; + ["preferred_username", "email", "name"] + .iter() + .find_map(|claim| text_at(&claims, &format!("/{claim}"))) +} + +/// The account name for new tokens, or an empty string. +pub fn resolve(exchange: &Exchange, tokens: &Tokens) -> String { + match exchange.oauth.account { + AccountFrom::IdToken => from_id_token(&tokens.body), + AccountFrom::TokenResponse(pointer) => text_at(&tokens.body, pointer), + AccountFrom::Call { method, url, pointer } => { + let request = + HttpRequest::new(method, url).header("Authorization", format!("Bearer {}", tokens.access.expose())); + call(exchange.http, exchange.policy, &request, pointer) + } + } + .unwrap_or_default() +} + +fn call(http: &dyn Http, policy: &HostPolicy, request: &HttpRequest, pointer: &str) -> Option { + let response = http.send(policy, request, 64 * 1024).ok()?; + if !response.ok() { + return None; + } + text_at(&response.json()?, pointer) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn token_with(claims: Value) -> Value { + let part = |value: &Value| URL_SAFE_NO_PAD.encode(value.to_string()); + json!({ "id_token": format!("{}.{}.signature", part(&json!({ "alg": "none" })), part(&claims)) }) + } + + #[test] + fn reads_the_name_from_the_id_token_in_order_of_preference() { + let body = token_with( + json!({ "name": "Sam Student", "email": "sam@example.com", "preferred_username": "sam@school.example" }), + ); + assert_eq!(from_id_token(&body).as_deref(), Some("sam@school.example")); + assert_eq!( + from_id_token(&token_with(json!({ "name": "Sam", "email": "e@x.example" }))).as_deref(), + Some("e@x.example") + ); + assert_eq!( + from_id_token(&token_with(json!({ "name": "Sam" }))).as_deref(), + Some("Sam") + ); + assert_eq!(from_id_token(&token_with(json!({ "sub": "1234" }))), None); + } + + #[test] + fn a_damaged_or_missing_id_token_gives_no_name() { + for body in [ + json!({}), + json!({ "id_token": "one-part" }), + json!({ "id_token": "a.!!!.c" }), + json!({ "id_token": 5 }), + ] { + assert_eq!(from_id_token(&body), None, "{body}"); + } + } + + #[test] + fn names_are_cleaned_and_cut() { + assert_eq!(clean(" Sam\u{0}\n Student "), "Sam Student"); + assert_eq!(clean(&"é".repeat(300)).chars().count(), MAX_NAME); + assert_eq!(text_at(&json!({ "a": { "b": " " } }), "/a/b"), None); + assert_eq!(text_at(&json!({ "a": { "b": "x" } }), "/a/b").as_deref(), Some("x")); + } +} diff --git a/app/src-tauri/src/connectors/accounts.rs b/app/src-tauri/src/connectors/accounts.rs new file mode 100644 index 00000000..d9514dd9 --- /dev/null +++ b/app/src-tauri/src/connectors/accounts.rs @@ -0,0 +1,495 @@ +//! The extra host calls of the account features (connectors, meeting notes, sharing, sending, syncing). One command, +//! `accounts_call`, carries each by name with JSON arguments, so the command list and the capability file stay short +//! (like `shellqol_call`). +//! +//! - `state.*`: small JSON files in `accounts/state` of this device's folder, for what a sync remembers (a cursor, +//! a delta link, the ID of the copy at a service). They hold no token. +//! - `client.set`: saves a client ID typed on a card, in the connectors file. +//! - `temp.*`: files in `accounts/tmp`, where an export is written before it goes to a service and an import is +//! read after it comes back. The calls take names inside that folder, never other paths. +//! - `transfer.*`: a file down from a service or up to it, with the connection's token added by the connector. The +//! bytes stay in Rust, so a large file never crosses into the interface. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use base64::{engine::general_purpose::STANDARD, Engine as _}; +use serde::Deserialize; +use serde_json::{json, Value}; +use tauri::{async_runtime::spawn_blocking, State}; + +use super::{ + commands::{ApiRequest, ApiResponse}, + files, + http::{Body, HttpRequest}, + service::Connectors, +}; +use crate::{ + ipc::{codes, IpcError, IpcResult}, + paths::Paths, + settings::file::{write_atomic, write_json}, +}; + +/// The largest file a transfer moves in memory. +const MAX_TRANSFER: usize = 256 * 1024 * 1024; + +/// The folder of this module inside the device's folder. +pub(crate) const FOLDER: &str = "accounts"; + +fn invalid(field: &str, message: &str) -> IpcError { + IpcError::invalid(field, message) +} + +fn io_error(error: &std::io::Error) -> IpcError { + IpcError::new(codes::IO, error.to_string()) +} + +fn arg(args: &Value, name: &str) -> IpcResult { + serde_json::from_value(args.get(name).cloned().unwrap_or(Value::Null)) + .map_err(|error| invalid(name, &error.to_string())) +} + +/// A name that is one file name: letters, digits, dot, dash, underscore, and space, and not starting with a dot. +fn safe_segment(text: &str) -> bool { + !text.is_empty() + && text.len() <= 128 + && !text.starts_with('.') + && !text.ends_with(' ') + && text + .chars() + .all(|c| c.is_alphanumeric() || matches!(c, '.' | '-' | '_' | ' ' | '(' | ')')) +} + +/// A state name: lowercase letters, digits, dot, and dash. +fn safe_state_name(text: &str) -> bool { + !text.is_empty() + && text.len() <= 64 + && text + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '-')) + && !text.starts_with('.') +} + +/// Where a file named by the interface is: a relative name made of safe segments inside the temp folder, or an +/// absolute path that already lies inside it (an export reports the path it wrote). +pub(super) fn resolve_temp(root: &Path, text: &str) -> Result { + let tmp = root.join("tmp"); + let candidate = Path::new(text); + if candidate.is_absolute() { + let (Ok(inside), Ok(base)) = (candidate.canonicalize(), tmp.canonicalize()) else { + return Err(invalid("file", "That file isn't in the transfer folder.")); + }; + return if inside.starts_with(&base) { + Ok(inside) + } else { + Err(invalid("file", "That file isn't in the transfer folder.")) + }; + } + let segments: Vec<&str> = text.split('/').collect(); + if segments.is_empty() || !segments.iter().all(|segment| safe_segment(segment)) { + return Err(invalid("file", "That isn't a file name.")); + } + Ok(segments.iter().fold(tmp, |path, segment| path.join(segment))) +} + +fn state_file(root: &Path, name: &str) -> Result { + if !safe_state_name(name) { + return Err(invalid("name", "That isn't a state name.")); + } + Ok(root.join("state").join(format!("{name}.json"))) +} + +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +enum Part { + Text { + text: String, + }, + Base64 { + base64: String, + }, + File { + file: String, + #[serde(default)] + offset: Option, + #[serde(default)] + length: Option, + }, +} + +fn read_part(root: &Path, part: &Part) -> IpcResult> { + match part { + Part::Text { text } => Ok(text.clone().into_bytes()), + Part::Base64 { base64 } => STANDARD + .decode(base64) + .map_err(|_| invalid("parts", "The bytes aren't base64.")), + Part::File { file, offset, length } => { + let path = resolve_temp(root, file)?; + let bytes = fs::read(&path).map_err(|error| io_error(&error))?; + let start = usize::try_from(offset.unwrap_or(0)) + .unwrap_or(usize::MAX) + .min(bytes.len()); + let end = length + .map_or(bytes.len(), |count| { + start.saturating_add(usize::try_from(count).unwrap_or(usize::MAX)) + }) + .min(bytes.len()); + Ok(bytes[start..end].to_vec()) + } + } +} + +fn join_parts(root: &Path, parts: &[Part]) -> IpcResult> { + let mut body = Vec::new(); + for part in parts { + body.extend(read_part(root, part)?); + if body.len() > MAX_TRANSFER { + return Err(invalid("parts", "That is too much to send at once.")); + } + } + Ok(body) +} + +fn response_json(response: super::http::HttpResponse) -> Value { + json!(ApiResponse { + status: response.status, + content_type: response.content_type, + body: String::from_utf8_lossy(&response.body).into_owned(), + location: response.location, + }) +} + +/// Runs one named call. `root` is `accounts` in the device's folder. +pub(super) fn call(connectors: &Connectors, root: &Path, name: &str, args: &Value) -> IpcResult { + match name { + "state.get" => { + let path = state_file(root, &arg::(args, "name")?)?; + Ok(fs::read_to_string(path) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .unwrap_or(Value::Null)) + } + "state.set" => { + let path = state_file(root, &arg::(args, "name")?)?; + write_json(&path, args.get("value").unwrap_or(&Value::Null)).map_err(|error| io_error(&error))?; + Ok(json!({ "ok": true })) + } + "state.delete" => { + let path = state_file(root, &arg::(args, "name")?)?; + let _ = fs::remove_file(path); + Ok(json!({ "ok": true })) + } + "client.set" => { + let connector: String = arg(args, "connector")?; + let id: String = arg(args, "clientId")?; + let secret: Option = arg(args, "clientSecret")?; + // Only a connector that signs in on a page has a client. + let def = connectors.def(&connector).map_err(IpcError::from)?; + if !matches!(def.auth, super::registry::Auth::OAuth(_)) { + return Err(invalid("connector", "That connector has no client ID.")); + } + files::save_client(&connectors.0.config_file, &connector, &id, secret.as_deref()) + .map_err(|_| invalid("clientId", "That client ID can't be used."))?; + serde_json::to_value(connectors.view(&connector).map_err(IpcError::from)?) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string())) + } + "temp.dir" => { + let tmp = root.join("tmp"); + fs::create_dir_all(&tmp).map_err(|error| io_error(&error))?; + Ok(json!({ "path": tmp.to_string_lossy() })) + } + "temp.write" => { + let path = resolve_temp(root, &arg::(args, "file")?)?; + let part: Part = arg(args, "part")?; + let bytes = read_part(root, &part)?; + write_atomic(&path, &bytes).map_err(|error| io_error(&error))?; + Ok(json!({ "path": path.to_string_lossy(), "bytes": bytes.len() })) + } + "temp.read" => { + let path = resolve_temp(root, &arg::(args, "file")?)?; + let bytes = fs::read(path).map_err(|error| io_error(&error))?; + if bytes.len() > MAX_TRANSFER { + return Err(invalid("file", "That file is too large to read.")); + } + Ok(json!({ "base64": STANDARD.encode(&bytes), "bytes": bytes.len() })) + } + "temp.remove" => { + let path = resolve_temp(root, &arg::(args, "file")?)?; + let _ = if path.is_dir() { + fs::remove_dir_all(path) + } else { + fs::remove_file(path) + }; + Ok(json!({ "ok": true })) + } + "temp.clear" => { + let _ = fs::remove_dir_all(root.join("tmp")); + Ok(json!({ "ok": true })) + } + "transfer.download" => transfer_down(connectors, root, args), + "transfer.upload" => transfer_up(connectors, root, args), + "transfer.public" => transfer_public(connectors, root, args), + "fetch.preview" => fetch_preview(connectors, args), + _ => Err(invalid("name", &format!("{name} isn't an accounts call."))), + } +} + +fn access_of(args: &Value) -> IpcResult> { + arg(args, "access") +} + +fn transfer_down(connectors: &Connectors, root: &Path, args: &Value) -> IpcResult { + let id: String = arg(args, "connector")?; + let access = access_of(args)?; + let request: ApiRequest = arg(args, "request")?; + let target = resolve_temp(root, &arg::(args, "file")?)?; + let wanted: Vec<&str> = access.iter().map(String::as_str).collect(); + let request = request.into_http(&id).map_err(IpcError::from)?; + let response = connectors + .request(&id, &wanted, request, MAX_TRANSFER) + .map_err(IpcError::from)?; + let bytes = response.body.len(); + if response.ok() { + write_atomic(&target, &response.body).map_err(|error| io_error(&error))?; + } + Ok(json!({ + "status": response.status, + "contentType": response.content_type, + "bytes": bytes, + "path": target.to_string_lossy(), + "body": if response.ok() { String::new() } else { String::from_utf8_lossy(&response.body).into_owned() }, + })) +} + +fn transfer_up(connectors: &Connectors, root: &Path, args: &Value) -> IpcResult { + let id: String = arg(args, "connector")?; + let access = access_of(args)?; + let request: ApiRequest = arg(args, "request")?; + let parts: Vec = arg(args, "parts")?; + let content_type: String = arg(args, "contentType")?; + let body = join_parts(root, &parts)?; + let wanted: Vec<&str> = access.iter().map(String::as_str).collect(); + let mut http: HttpRequest = request.into_http(&id).map_err(IpcError::from)?; + http.body = Some(Body::Bytes { + content_type, + data: body, + }); + let response = connectors + .request(&id, &wanted, http, super::session::DEFAULT_MAX_BYTES) + .map_err(IpcError::from)?; + Ok(response_json(response)) +} + +/// Whether the address is a plain https address on a name, which is where a service's own answer may send an upload +/// (Canvas hands out the address of its file storage). Numbers, this computer, local names, ports, and sign-in +/// details in the address are all refused. +pub(super) fn public_upload_host(text: &str) -> Option { + let url = url::Url::parse(text).ok()?; + if url.scheme() != "https" || url.port().is_some() || !url.username().is_empty() || url.password().is_some() { + return None; + } + let host = match url.host()? { + url::Host::Domain(name) => name.to_ascii_lowercase(), + _ => return None, + }; + let local = + host == "localhost" || host.ends_with(".localhost") || host.ends_with(".local") || host.ends_with(".internal"); + (host.contains('.') && !local).then_some(host) +} + +/// The only hosts a link preview is read from: the oEmbed address and the picture of each player and live embed. +const PREVIEW_HOSTS: &[&str] = &[ + "www.youtube.com", + "i.ytimg.com", + "vimeo.com", + "i.vimeocdn.com", + "open.spotify.com", + "i.scdn.co", + "mosaic.scdn.co", + "saved-work.desmos.com", +]; + +/// The most a preview may be: a picture, or a few lines of oEmbed. +const MAX_PREVIEW: usize = 3 * 1024 * 1024; + +/// Whether the address may be fetched as a preview: https only, with neither a port nor sign-in details, on one of the listed hosts. +pub(super) fn preview_host(text: &str) -> Option { + let host = public_upload_host(text)?; + PREVIEW_HOSTS.contains(&host.as_str()).then_some(host) +} + +/// A GET with no token for the title and picture of a link on the page. Nothing is sent but the address, and the +/// answer is returned as base64 (a picture) with its type. Work offline blocks it. +fn fetch_preview(connectors: &Connectors, args: &Value) -> IpcResult { + let address: String = arg(args, "url")?; + if connectors.offline() { + return Err(invalid("url", "OpenNote is working offline.")); + } + let host = preview_host(&address).ok_or_else(|| invalid("url", "That address can't be read."))?; + let http = HttpRequest::new(super::registry::Method::Get, address); + let policy = super::http::HostPolicy::new([host]); + let response = connectors + .0 + .http + .send(&policy, &http, MAX_PREVIEW) + .map_err(|_| invalid("url", "The preview didn't load."))?; + Ok(json!({ + "status": response.status, + "contentType": response.content_type, + "base64": STANDARD.encode(&response.body), + })) +} + +/// A POST with no token to the upload address a service gave in an answer. The connector must be connected, so only +/// a feature the person has set up can send, and the bytes are whatever the feature joined. +fn transfer_public(connectors: &Connectors, root: &Path, args: &Value) -> IpcResult { + let id: String = arg(args, "connector")?; + let address: String = arg(args, "url")?; + let parts: Vec = arg(args, "parts")?; + let content_type: String = arg(args, "contentType")?; + let connected = matches!( + connectors.view(&id).map_err(IpcError::from)?.state, + super::view::StateView::Connected { .. } + ); + if !connected { + return Err(invalid("connector", "That account isn't connected.")); + } + if connectors.offline() { + return Err(invalid("url", "OpenNote is working offline.")); + } + let host = public_upload_host(&address).ok_or_else(|| invalid("url", "That upload address can't be used."))?; + let mut http = HttpRequest::new(super::registry::Method::Post, address); + // Canvas answers a finished upload with a redirect to its own host, where the app confirms with the token, so + // the redirect comes back here instead of being followed off the upload host. + http.follow_redirects = false; + http.body = Some(Body::Bytes { + content_type, + data: join_parts(root, &parts)?, + }); + let policy = super::http::HostPolicy::new([host]); + let response = connectors + .0 + .http + .send(&policy, &http, super::session::DEFAULT_MAX_BYTES) + .map_err(|_| invalid("url", "The upload didn't go through."))?; + Ok(response_json(response)) +} + +#[tauri::command] +pub async fn accounts_call( + connectors: State<'_, Connectors>, + paths: State<'_, Paths>, + name: String, + args: Value, +) -> IpcResult { + let connectors = connectors.inner().clone(); + let root = paths.local.join(FOLDER); + match spawn_blocking(move || call(&connectors, &root, &name, &args)).await { + Ok(result) => result, + Err(_) => Err(IpcError::new(codes::INTERNAL, "The account call stopped.")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_public_upload_goes_only_to_a_plain_https_name() { + assert_eq!( + public_upload_host("https://Files.School.example/upload?id=1").as_deref(), + Some("files.school.example") + ); + for bad in [ + "http://files.school.example/up", + "https://files.school.example:8443/up", + "https://127.0.0.1/up", + "https://[::1]/up", + "https://10.0.0.5/up", + "https://localhost/up", + "https://printer.local/up", + "https://intranet/up", + "https://user:pass@files.school.example/up", + "file:///C:/secret", + "not a url", + ] { + assert_eq!(public_upload_host(bad), None, "{bad}"); + } + } + + #[test] + fn a_preview_is_read_only_from_the_listed_hosts() { + for good in [ + "https://www.youtube.com/oembed?format=json&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3Dabc", + "https://i.ytimg.com/vi/abc/hqdefault.jpg", + "https://saved-work.desmos.com/calc_thumbs/production/abc.png", + ] { + assert!(preview_host(good).is_some(), "{good}"); + } + for bad in [ + "https://example.com/x.png", + "https://www.youtube.com.evil.example/x", + "https://evil.example/www.youtube.com", + "http://i.ytimg.com/x", + "https://i.ytimg.com:444/x", + "https://user@i.ytimg.com/x", + ] { + assert!(preview_host(bad).is_none(), "{bad}"); + } + } + + #[test] + fn names_are_checked_before_they_reach_the_disk() { + let root = tempfile::tempdir().expect("a folder"); + for bad in [ + "", + "..", + "a/../b", + "/etc/passwd", + "a\\b", + ".hidden", + "a/.b", + "x:y", + &"a".repeat(200), + ] { + assert!(resolve_temp(root.path(), bad).is_err(), "{bad}"); + } + let ok = resolve_temp(root.path(), "export/Page (1).docx").expect("a name"); + assert!(ok.starts_with(root.path().join("tmp"))); + assert!(state_file(root.path(), "readwise.cursor").is_ok()); + for bad in ["", "Upper", "a/b", "../x", ".x"] { + assert!(state_file(root.path(), bad).is_err(), "{bad}"); + } + } + + #[test] + fn an_absolute_path_passes_only_inside_the_temp_folder() { + let root = tempfile::tempdir().expect("a folder"); + let tmp = root.path().join("tmp"); + fs::create_dir_all(&tmp).expect("makes"); + let inside = tmp.join("a.docx"); + fs::write(&inside, b"x").expect("writes"); + assert!(resolve_temp(root.path(), &inside.to_string_lossy()).is_ok()); + let outside = root.path().join("other.txt"); + fs::write(&outside, b"x").expect("writes"); + assert!(resolve_temp(root.path(), &outside.to_string_lossy()).is_err()); + } + + #[test] + fn parts_join_text_bytes_and_a_slice_of_a_file() { + let root = tempfile::tempdir().expect("a folder"); + let tmp = root.path().join("tmp"); + fs::create_dir_all(&tmp).expect("makes"); + fs::write(tmp.join("f.bin"), b"0123456789").expect("writes"); + let parts: Vec = serde_json::from_value(json!([ + { "kind": "text", "text": "a" }, + { "kind": "base64", "base64": "Yg==" }, + { "kind": "file", "file": "f.bin", "offset": 2, "length": 3 }, + { "kind": "file", "file": "f.bin", "offset": 8, "length": 99 }, + ])) + .expect("parses"); + assert_eq!(join_parts(root.path(), &parts).expect("joins"), b"ab23489"); + } +} diff --git a/app/src-tauri/src/connectors/catalog.rs b/app/src-tauri/src/connectors/catalog.rs new file mode 100644 index 00000000..de901aa2 --- /dev/null +++ b/app/src-tauri/src/connectors/catalog.rs @@ -0,0 +1,310 @@ +//! The catalog: the entries of the registry, one per service. `registry.rs` defines what an entry holds and how to +//! look one up. To add a service, put an entry here, then its strings in `strings/en/connectors.ts`. + +use super::registry::{ + Access, AccountFrom, Auth, ConnectorDef, Group, Method, OAuthDef, Placement, Pointers, Redirect, Revoke, SecretUse, + TokenDef, STANDARD_RESPONSE, +}; + +const fn read(scope: &'static str, capability: &'static str) -> Access { + Access { + scope, + capability, + writes: false, + } +} + +const fn write(scope: &'static str, capability: &'static str) -> Access { + Access { + scope, + capability, + writes: true, + } +} + +const NO_PARAMS: &[(&str, &str)] = &[]; + +pub static CONNECTORS: &[ConnectorDef] = &[ + ConnectorDef { + id: "microsoft", + name: "Microsoft", + group: Group::Microsoft, + features: &[ + "importOneNote", + "outlookCalendar", + "microsoftToDo", + "shareToTeams", + "sendToOneDrive", + ], + hosts: &["login.microsoftonline.com", "graph.microsoft.com"], + access: &[ + read("openid", "account"), + read("email", "account"), + read("profile", "account"), + read("User.Read", "account"), + read("offline_access", "stayConnected"), + read("Notes.Read", "onenoteRead"), + read("Calendars.Read", "calendarRead"), + write("Tasks.ReadWrite", "tasksWrite"), + write("Files.ReadWrite", "onedriveFiles"), + write("ChannelMessage.Send", "teamsPost"), + read("Team.ReadBasic.All", "teamsRead"), + read("Channel.ReadBasic.All", "teamsRead"), + ], + auth: Auth::OAuth(OAuthDef { + authorize_url: "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", + token_url: "https://login.microsoftonline.com/common/oauth2/v2.0/token", + revoke: Revoke::None, + scope_param: "scope", + scope_separator: " ", + extra_params: &[("prompt", "select_account")], + pkce: true, + secret: SecretUse::Never, + redirect: Redirect::AnyPort, + account: AccountFrom::IdToken, + response: STANDARD_RESPONSE, + }), + }, + ConnectorDef { + id: "google", + name: "Google", + group: Group::Google, + features: &[ + "googleCalendar", + "googleTasks", + "googleClassroom", + "googleDrive", + "googleOffice", + "youtubeCaptions", + "youtubeUpload", + ], + hosts: &[ + "accounts.google.com", + "oauth2.googleapis.com", + "www.googleapis.com", + "tasks.googleapis.com", + "classroom.googleapis.com", + "youtube.googleapis.com", + ], + access: &[ + read("openid", "account"), + read("email", "account"), + read("https://www.googleapis.com/auth/calendar.readonly", "calendarRead"), + write("https://www.googleapis.com/auth/tasks", "tasksWrite"), + read( + "https://www.googleapis.com/auth/classroom.courses.readonly", + "classroomRead", + ), + read( + "https://www.googleapis.com/auth/classroom.coursework.me.readonly", + "classroomRead", + ), + write("https://www.googleapis.com/auth/drive.file", "driveFiles"), + read("https://www.googleapis.com/auth/youtube.readonly", "youtubeRead"), + read("https://www.googleapis.com/auth/drive.readonly", "driveRead"), + write("https://www.googleapis.com/auth/youtube.upload", "youtubeUpload"), + write("https://www.googleapis.com/auth/youtube.force-ssl", "youtubeCaptions"), + write("https://www.googleapis.com/auth/drive.appdata", "driveAppData"), + ], + auth: Auth::OAuth(OAuthDef { + authorize_url: "https://accounts.google.com/o/oauth2/v2/auth", + token_url: "https://oauth2.googleapis.com/token", + revoke: Revoke::Form { + url: "https://oauth2.googleapis.com/revoke", + with_client: false, + }, + scope_param: "scope", + scope_separator: " ", + extra_params: &[("access_type", "offline"), ("prompt", "consent")], + pkce: true, + secret: SecretUse::Optional, + redirect: Redirect::AnyPort, + account: AccountFrom::IdToken, + response: STANDARD_RESPONSE, + }), + }, + ConnectorDef { + id: "dropbox", + name: "Dropbox", + group: Group::Storage, + features: &["syncDropbox", "sendToDropbox"], + hosts: &["www.dropbox.com", "api.dropboxapi.com", "content.dropboxapi.com"], + access: &[ + read("account_info.read", "dropboxAccount"), + read("files.metadata.read", "dropboxRead"), + read("files.content.read", "dropboxRead"), + write("files.content.write", "dropboxWrite"), + ], + auth: Auth::OAuth(OAuthDef { + authorize_url: "https://www.dropbox.com/oauth2/authorize", + token_url: "https://api.dropboxapi.com/oauth2/token", + revoke: Revoke::BearerPost { + url: "https://api.dropboxapi.com/2/auth/token/revoke", + }, + scope_param: "scope", + scope_separator: " ", + extra_params: &[("token_access_type", "offline")], + pkce: true, + secret: SecretUse::Optional, + redirect: Redirect::FixedPort(53682), + account: AccountFrom::Call { + method: Method::Post, + url: "https://api.dropboxapi.com/2/users/get_current_account", + pointer: "/email", + }, + response: STANDARD_RESPONSE, + }), + }, + ConnectorDef { + id: "box", + name: "Box", + group: Group::Storage, + features: &["syncBox", "sendToBox"], + hosts: &["account.box.com", "api.box.com", "upload.box.com"], + // Box sets its scopes on the app registration, so none goes on the sign-in address. + access: &[write("", "boxFiles")], + auth: Auth::OAuth(OAuthDef { + authorize_url: "https://account.box.com/api/oauth2/authorize", + token_url: "https://api.box.com/oauth2/token", + revoke: Revoke::Form { + url: "https://api.box.com/oauth2/revoke", + with_client: true, + }, + scope_param: "", + scope_separator: " ", + extra_params: NO_PARAMS, + pkce: true, + secret: SecretUse::Required, + redirect: Redirect::FixedPort(53683), + account: AccountFrom::Call { + method: Method::Get, + url: "https://api.box.com/2.0/users/me", + pointer: "/login", + }, + response: STANDARD_RESPONSE, + }), + }, + ConnectorDef { + id: "slack", + name: "Slack", + group: Group::Other, + features: &["shareToSlack"], + hosts: &["slack.com", "files.slack.com"], + access: &[ + write("chat:write", "slackPost"), + write("files:write", "slackFiles"), + read("channels:read", "slackChannels"), + ], + auth: Auth::OAuth(OAuthDef { + authorize_url: "https://slack.com/oauth/v2/authorize", + token_url: "https://slack.com/api/oauth.v2.access", + revoke: Revoke::BearerPost { + url: "https://slack.com/api/auth.revoke", + }, + scope_param: "user_scope", + scope_separator: ",", + extra_params: NO_PARAMS, + pkce: false, + secret: SecretUse::Required, + redirect: Redirect::FixedPort(53681), + account: AccountFrom::TokenResponse("/team/name"), + response: Pointers { + access: "/authed_user/access_token", + refresh: "/authed_user/refresh_token", + expires_in: "/authed_user/expires_in", + scope: "/authed_user/scope", + }, + }), + }, + ConnectorDef { + id: "vimeo", + name: "Vimeo", + group: Group::Other, + features: &["vimeoCaptions"], + hosts: &["api.vimeo.com", "captions.cloud.vimeo.com"], + access: &[read("public", "vimeoRead"), read("private", "vimeoRead")], + auth: Auth::OAuth(OAuthDef { + authorize_url: "https://api.vimeo.com/oauth/authorize", + token_url: "https://api.vimeo.com/oauth/access_token", + revoke: Revoke::BearerDelete { + url: "https://api.vimeo.com/tokens", + }, + scope_param: "scope", + scope_separator: " ", + extra_params: NO_PARAMS, + pkce: false, + secret: SecretUse::Required, + redirect: Redirect::FixedPort(53684), + account: AccountFrom::TokenResponse("/user/name"), + response: STANDARD_RESPONSE, + }), + }, + ConnectorDef { + id: "readwise", + name: "Readwise", + group: Group::Other, + features: &["readwiseHighlights"], + hosts: &["readwise.io"], + access: &[read("", "readwiseRead")], + auth: Auth::Token(TokenDef { + needs_base_url: false, + placement: Placement::Header("Token"), + check_method: Method::Get, + check_url: "https://readwise.io/api/v2/auth/", + check_form: NO_PARAMS, + account_pointer: "", + }), + }, + ConnectorDef { + id: "canvas", + name: "Canvas", + group: Group::Learning, + features: &["canvasAssignments"], + hosts: &[], + access: &[read("", "canvasRead"), write("", "canvasSubmit")], + auth: Auth::Token(TokenDef { + needs_base_url: true, + placement: Placement::Header("Bearer"), + check_method: Method::Get, + check_url: "/api/v1/users/self", + check_form: NO_PARAMS, + account_pointer: "/name", + }), + }, + ConnectorDef { + id: "moodle", + name: "Moodle", + group: Group::Learning, + features: &["moodleAssignments"], + hosts: &[], + access: &[read("", "moodleRead"), write("", "moodleSubmit")], + auth: Auth::Token(TokenDef { + needs_base_url: true, + placement: Placement::FormField("wstoken"), + check_method: Method::Post, + check_url: "/webservice/rest/server.php", + check_form: &[ + ("wsfunction", "core_webservice_get_site_info"), + ("moodlewsrestformat", "json"), + ], + account_pointer: "/fullname", + }), + }, + ConnectorDef { + id: "webdav", + name: "WebDAV", + group: Group::Storage, + features: &["sendToWebdav"], + hosts: &[], + access: &[write("", "webdavFiles")], + auth: Auth::Token(TokenDef { + needs_base_url: true, + // The token is "user:password", and the connector sends it as Basic. + placement: Placement::Basic, + check_method: Method::Propfind, + check_url: "/", + check_form: NO_PARAMS, + account_pointer: "", + }), + }, +]; diff --git a/app/src-tauri/src/connectors/commands.rs b/app/src-tauri/src/connectors/commands.rs new file mode 100644 index 00000000..a4db251e --- /dev/null +++ b/app/src-tauri/src/connectors/commands.rs @@ -0,0 +1,172 @@ +//! The commands of the Settings page. None of them returns a token, a code, or a client ID: they return the +//! state of each connector, and `connectors_request` returns what the service answered. A pasted token goes in +//! through `connectors_connect` and nowhere else. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use tauri::{async_runtime::spawn_blocking, State}; + +use super::{ + error::{ConnectorError, Failure}, + http::{Body, HttpRequest}, + registry::Method, + service::Connectors, + session::DEFAULT_MAX_BYTES, + view::{ConnectInput, ConnectorView, Disconnected}, +}; +use crate::ipc::{codes, IpcError, IpcResult}; + +/// A request from the interface to a connector's service. The connector adds the token, so there is no +/// `Authorization` header here, and the address must be on the connector's own hosts. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ApiRequest { + pub method: String, + pub url: String, + #[serde(default)] + pub headers: BTreeMap, + /// A text body, such as JSON. + #[serde(default)] + pub body: Option, + #[serde(default)] + pub content_type: Option, + /// Form fields, for a service that takes its token in the form, such as Moodle. + #[serde(default)] + pub form: Option>, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ApiResponse { + pub status: u16, + pub content_type: Option, + pub body: String, + /// The `Location` header of an answer, for a resumable upload. + pub location: Option, +} + +impl ApiRequest { + pub(super) fn into_http(self, connector: &str) -> Result { + let bad = || ConnectorError::new(Failure::BadInput, connector); + let method = Method::parse(&self.method).ok_or_else(bad)?; + let mut request = HttpRequest::new(method, self.url); + request.headers = self.headers.into_iter().collect(); + request.body = match (self.form, self.body) { + (Some(_), Some(_)) => return Err(bad()), + (Some(form), None) => Some(Body::Form(form.into_iter().collect())), + (None, Some(text)) => Some(Body::Bytes { + content_type: self.content_type.unwrap_or_else(|| "application/json".to_owned()), + data: text.into_bytes(), + }), + (None, None) => None, + }; + Ok(request) + } +} + +fn joined(result: Result, tauri::Error>) -> IpcResult { + match result { + Ok(inner) => inner.map_err(IpcError::from), + Err(_) => Err(IpcError::new(codes::INTERNAL, "The connector call stopped.")), + } +} + +#[tauri::command] +pub fn connectors_list(connectors: State<'_, Connectors>) -> Vec { + connectors.list() +} + +/// Signs in. For an OAuth connector this opens the person's browser and waits, up to five minutes, so the +/// interface shows a waiting card and offers `connectors_cancel`. +#[tauri::command] +pub async fn connectors_connect( + connectors: State<'_, Connectors>, + id: String, + input: Option, +) -> IpcResult { + let connectors = connectors.inner().clone(); + joined(spawn_blocking(move || connectors.connect(&id, input.unwrap_or_default())).await) +} + +#[tauri::command] +pub fn connectors_cancel(connectors: State<'_, Connectors>, id: String) { + connectors.cancel(&id); +} + +#[tauri::command] +pub async fn connectors_disconnect(connectors: State<'_, Connectors>, id: String) -> IpcResult { + let connectors = connectors.inner().clone(); + joined(spawn_blocking(move || connectors.disconnect(&id)).await) +} + +#[tauri::command] +pub async fn connectors_request( + connectors: State<'_, Connectors>, + id: String, + scopes: Vec, + request: ApiRequest, +) -> IpcResult { + let connectors = connectors.inner().clone(); + let work = move || { + let request = request.into_http(&id)?; + let wanted: Vec<&str> = scopes.iter().map(String::as_str).collect(); + let response = connectors.request(&id, &wanted, request, DEFAULT_MAX_BYTES)?; + Ok(ApiResponse { + status: response.status, + content_type: response.content_type, + body: String::from_utf8_lossy(&response.body).into_owned(), + location: response.location, + }) + }; + joined(spawn_blocking(work).await) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(json: serde_json::Value) -> ApiRequest { + serde_json::from_value(json).expect("parses") + } + + #[test] + fn a_request_becomes_an_http_request_without_a_token() { + let request = parse(serde_json::json!({ + "method": "post", "url": "https://graph.microsoft.com/v1.0/me/todo/lists", + "headers": { "Accept": "application/json" }, "body": "{\"a\":1}" + })) + .into_http("microsoft") + .expect("converts"); + assert_eq!(request.method, Method::Post); + assert_eq!(request.headers, [("Accept".to_owned(), "application/json".to_owned())]); + assert!( + matches!(request.body, Some(Body::Bytes { ref content_type, .. }) if content_type == "application/json") + ); + } + + #[test] + fn odd_requests_are_refused_before_they_go_anywhere() { + for json in [ + serde_json::json!({ "method": "TRACE", "url": "https://x.example/" }), + serde_json::json!({ "method": "GET", "url": "https://x.example/", "body": "a", "form": { "b": "c" } }), + ] { + let error = parse(json).into_http("canvas").err().expect("refused"); + assert_eq!(error.failure, Failure::BadInput); + } + } + + #[test] + fn the_response_has_no_field_that_could_hold_a_token_of_ours() { + let json = serde_json::to_value(ApiResponse { + status: 200, + content_type: None, + body: "x".into(), + location: None, + }) + .expect("serializes"); + let mut keys: Vec<_> = json.as_object().expect("an object").keys().cloned().collect(); + keys.sort(); + assert_eq!(keys, ["body", "contentType", "location", "status"]); + } +} diff --git a/app/src-tauri/src/connectors/connect.rs b/app/src-tauri/src/connectors/connect.rs new file mode 100644 index 00000000..536f1499 --- /dev/null +++ b/app/src-tauri/src/connectors/connect.rs @@ -0,0 +1,395 @@ +//! Connecting: an OAuth connector signs in through the person's own browser (RFC 8252), and the others check a +//! pasted token with one request. Nothing is stored until the service has accepted the code or the token. + +use std::{ + sync::{atomic::AtomicBool, Arc}, + time::Duration, +}; + +use url::{Host, Url}; + +use super::{ + account, + error::{ConnectorError, Failure, Result}, + files::{self, Holds}, + http::{HostPolicy, HttpError, HttpRequest}, + loopback::{Callback, Ended, Listener}, + oauth::{Exchange, Tokens}, + pkce, + registry::{Auth, ConnectorDef, Method, OAuthDef, Redirect, SecretUse, TokenDef}, + secret::Secret, + service::{Cached, Connectors}, + session::{with_token, DEFAULT_LIFETIME}, + store::target_name, + view::{ConnectInput, ConnectorView}, +}; + +/// The answer of a token check is read up to this size. +const MAX_CHECK: usize = 64 * 1024; + +/// A sign-in that is waiting. It leaves the list of waiting sign-ins when it ends, however it ends. +pub(super) struct Pending { + owner: Connectors, + id: String, + pub flag: Arc, +} + +impl Drop for Pending { + fn drop(&mut self) { + self.owner.lock().pending.remove(&self.id); + } +} + +/// What a successful sign-in or token check hands to [`Connectors::finish`]. +struct NewConnection { + account: String, + /// The refresh token, or an access token that doesn't expire. This is what the credential store keeps. + secret: Secret, + holds: Holds, + scopes: Vec, + base_url: Option, + /// A current access token to keep in memory, and when it ends. + access: Option, +} + +impl Connectors { + /// Connects the person's account. For an OAuth connector this opens the browser and waits, up to five + /// minutes, until the person finishes or [`Connectors::cancel`] is called. + pub fn connect(&self, id: &str, input: ConnectInput) -> Result { + let def = self.def(id)?; + if self.offline() { + return Err(ConnectorError::new(Failure::Offline, id)); + } + match def.auth { + Auth::OAuth(oauth) => self.connect_oauth(def, &oauth)?, + Auth::Token(token) => self.connect_token(def, &token, input)?, + } + self.view(id) + } + + fn begin(&self, id: &str) -> Result { + let mut state = self.lock(); + if state.pending.contains_key(id) { + return Err(ConnectorError::new(Failure::Busy, id)); + } + let flag = Arc::new(AtomicBool::new(false)); + state.pending.insert(id.to_owned(), Arc::clone(&flag)); + Ok(Pending { + owner: self.clone(), + id: id.to_owned(), + flag, + }) + } + + fn connect_oauth(&self, def: &'static ConnectorDef, oauth: &OAuthDef) -> Result<()> { + let fail = |failure| ConnectorError::new(failure, def.id); + let client = files::load_client(&self.0.config_file, def.id) + .filter(|client| oauth.secret != SecretUse::Required || client.secret.is_some()) + .ok_or_else(|| fail(Failure::NotConfigured))?; + let pending = self.begin(def.id)?; + let port = match oauth.redirect { + Redirect::AnyPort => 0, + Redirect::FixedPort(default) => client.redirect_port.unwrap_or(default), + }; + let listener = Listener::bind(port).map_err(|_| fail(Failure::PortInUse))?; + let redirect_uri = listener.redirect_uri(); + let state = pkce::new_state().ok_or_else(|| fail(Failure::Unknown))?; + let verifier = if oauth.pkce { + Some(pkce::new_verifier().ok_or_else(|| fail(Failure::Unknown))?) + } else { + None + }; + + let policy = self.policy(def, None); + let exchange = Exchange { + http: &*self.0.http, + policy: &policy, + def, + oauth, + client: &client, + }; + let challenge = verifier.as_deref().map(pkce::challenge); + let url = exchange + .authorize_url(&redirect_uri, &state, challenge.as_deref()) + .ok_or_else(|| fail(Failure::Unknown))?; + // The sign-in page must be on one of the service's own hosts before the browser is sent there. + if !policy.allows(&url) { + return Err(fail(Failure::ForeignHost)); + } + if !self.0.opener.open(&url) { + return Err(fail(Failure::BrowserFailed)); + } + let code = wait_for_code(listener, &state, self.0.wait, &pending.flag).map_err(fail)?; + // Work offline may have been turned on while the browser was open. + if self.offline() { + return Err(fail(Failure::Offline)); + } + let tokens = exchange + .exchange_code(&code, verifier.as_deref(), &redirect_uri) + .map_err(fail)?; + let account = account::resolve(&exchange, &tokens); + self.finish(def, self.new_from_tokens(def, tokens, account)) + } + + fn new_from_tokens(&self, def: &ConnectorDef, tokens: Tokens, account: String) -> NewConnection { + let scopes = tokens + .scopes + .unwrap_or_else(|| def.scopes().into_iter().map(str::to_owned).collect()); + // A token that comes with a refresh token is renewed by it, so it always gets an end. + let lifetime = tokens + .expires_in + .or_else(|| tokens.refresh.is_some().then_some(DEFAULT_LIFETIME)); + let expires_unix = lifetime.map(|seconds| self.now() + seconds); + let (secret, holds) = match tokens.refresh { + Some(refresh) => (refresh, Holds::Refresh), + None => (tokens.access.clone(), Holds::Access), + }; + NewConnection { + account, + secret, + holds, + scopes, + base_url: None, + access: Some(Cached { + token: tokens.access, + expires_unix, + }), + } + } + + fn connect_token(&self, def: &'static ConnectorDef, token_def: &TokenDef, input: ConnectInput) -> Result<()> { + let fail = |failure| ConnectorError::new(failure, def.id); + let token = clean_token(input.token.as_deref().unwrap_or_default()).ok_or_else(|| fail(Failure::BadInput))?; + let base = if token_def.needs_base_url { + Some( + normalize_base_url(input.base_url.as_deref().unwrap_or_default()) + .ok_or_else(|| fail(Failure::BadInput))?, + ) + } else { + None + }; + let policy = self.policy(def, base.as_deref()); + let url = match &base { + Some(base) => format!("{base}{}", token_def.check_url), + None => token_def.check_url.to_owned(), + }; + let account = self.check_token(def, token_def, &policy, &url, &token)?; + self.finish( + def, + NewConnection { + account, + secret: token, + holds: Holds::Access, + scopes: Vec::new(), + base_url: base, + access: None, + }, + ) + } + + /// Sends the token to the service once and returns the account name from the answer. + pub(super) fn check_token( + &self, + def: &ConnectorDef, + token_def: &TokenDef, + policy: &HostPolicy, + url: &str, + token: &Secret, + ) -> Result { + let fail = |failure| ConnectorError::new(failure, def.id); + let mut request = HttpRequest::new(token_def.check_method, url); + if !token_def.check_form.is_empty() { + request = request.form( + token_def + .check_form + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(), + ); + } + // A folder listing asks for the folder alone, which servers answer and a listing of everything they refuse. + if token_def.check_method == Method::Propfind { + request = request.header("Depth", "0"); + } + let request = with_token(request, token_def.placement, token).ok_or_else(|| fail(Failure::BadInput))?; + let response = self.0.http.send(policy, &request, MAX_CHECK).map_err(|error| { + fail(match error { + HttpError::ForeignHost => Failure::ForeignHost, + HttpError::Network | HttpError::TooLarge => Failure::Network, + }) + })?; + if response.status >= 500 { + return Err(fail(Failure::Network)); + } + if !response.ok() { + return Err(fail(Failure::Rejected)); + } + if token_def.account_pointer.is_empty() { + return Ok(String::new()); + } + // A service that answers 200 for a bad token, as Moodle does, has no name in the answer. + let body = response.json().ok_or_else(|| fail(Failure::Rejected))?; + account::text_at(&body, token_def.account_pointer).ok_or_else(|| fail(Failure::Rejected)) + } + + /// Keeps a new connection: the secret in the credential store, the metadata in the connections file. + fn finish(&self, def: &ConnectorDef, new: NewConnection) -> Result<()> { + let storage = || ConnectorError::new(Failure::Storage, def.id); + let target = target_name(def.id, &new.account); + self.0.store.put(&target, &new.secret).map_err(|_| storage())?; + let mut state = self.lock(); + let account = new.account.clone(); + let previous = state.connections.insert( + def.id.to_owned(), + files::Connection { + account: new.account, + scopes: new.scopes, + connected_unix: self.now(), + holds: new.holds, + base_url: new.base_url, + expired: false, + last_used_unix: None, + }, + ); + let previous_access = state.access.remove(def.id); + if let Some(cached) = new.access { + state.access.insert(def.id.to_owned(), cached); + } + if let Err(error) = self.save(&state, def.id) { + // Without a record of the connection nothing could ever remove the credential, so it goes too, + // unless it replaced the same account's older one, which the kept record still points at. + match previous { + Some(old) => { + if old.account != account { + let _ = self.0.store.delete(&target); + } + state.connections.insert(def.id.to_owned(), old); + } + None => { + let _ = self.0.store.delete(&target); + state.connections.remove(def.id); + } + } + state.access.remove(def.id); + if let Some(cached) = previous_access { + state.access.insert(def.id.to_owned(), cached); + } + return Err(error); + } + // Connecting as another account replaces the old one, and the old credential goes with it. + if let Some(old) = previous.filter(|old| old.account != account) { + let _ = self.0.store.delete(&target_name(def.id, &old.account)); + } + Ok(()) + } +} + +/// Waits for the browser's return and turns how it ended into a failure or the code. +fn wait_for_code( + listener: Listener, + state: &str, + wait: Duration, + cancel: &AtomicBool, +) -> std::result::Result { + match listener.wait(state, wait, cancel) { + Ok(Callback::Code(code)) => Ok(code), + Ok(Callback::Refused(reason)) if reason == "access_denied" => Err(Failure::Denied), + Ok(Callback::Refused(_)) => Err(Failure::Rejected), + Err(Ended::Canceled) => Err(Failure::Canceled), + Err(Ended::TimedOut) => Err(Failure::TimedOut), + Err(Ended::Mismatch) => Err(Failure::Mismatch), + Err(Ended::Io) => Err(Failure::Network), + } +} + +/// A pasted token: printable ASCII with no spaces, of a plausible length. +fn clean_token(text: &str) -> Option { + let text = text.trim(); + let plausible = (8..=4096).contains(&text.len()) && text.bytes().all(|byte| byte.is_ascii_graphic()); + plausible.then(|| Secret::new(text)) +} + +/// A school's address as `https://host` or `https://host/path`: HTTPS only, a real domain name (never an IP +/// address, `localhost`, or a name with a user name, port, query, or fragment). Anything else is refused, so +/// a pasted token can only go to the school's own server. +pub fn normalize_base_url(text: &str) -> Option { + let text = text.trim(); + if text.is_empty() || text.len() > 200 || text.contains('\\') { + return None; + } + let with_scheme = if text.contains("://") { + text.to_owned() + } else { + format!("https://{text}") + }; + let url = Url::parse(&with_scheme).ok()?; + let Some(Host::Domain(host)) = url.host() else { + return None; + }; + let plain = url.scheme() == "https" + && url.username().is_empty() + && url.password().is_none() + && url.port().is_none() + && url.query().is_none() + && url.fragment().is_none() + && host.contains('.') + && !host.ends_with(".localhost") + && host.chars().all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-'); + if !plain { + return None; + } + let path = url.path().trim_end_matches('/'); + Some(format!("https://{}{path}", host.to_ascii_lowercase())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn school_addresses_are_normalized() { + for (input, expected) in [ + ("https://school.instructure.com", "https://school.instructure.com"), + ("school.instructure.com/", "https://school.instructure.com"), + ( + " HTTPS://Moodle.School.EDU/moodle/ ", + "https://moodle.school.edu/moodle", + ), + ] { + assert_eq!(normalize_base_url(input).as_deref(), Some(expected), "{input}"); + } + } + + #[test] + fn school_addresses_that_could_send_a_token_elsewhere_are_refused() { + for input in [ + "", + "http://school.edu", + "https://localhost", + "https://127.0.0.1", + "https://[::1]/", + "https://192.168.1.5", + "https://school.edu:8443", + "https://user:pass@school.edu", + "https://school.edu@evil.example", + "https://school.edu?x=1", + "https://school.edu/#frag", + "https://school", + "ftp://school.edu", + "https://school.edu\\@evil.example", + "https://sch ool.edu", + "https://xn--", + ] { + assert_eq!(normalize_base_url(input), None, "{input}"); + } + } + + #[test] + fn pasted_tokens_are_trimmed_and_must_be_plain() { + assert_eq!(clean_token(" abc12345 "), Some(Secret::new("abc12345"))); + for bad in ["", "short", "has space inside", "line\nbreak1234", "unicodé-token-1234"] { + assert_eq!(clean_token(bad), None, "{bad:?}"); + } + assert_eq!(clean_token(&"a".repeat(5000)), None); + } +} diff --git a/app/src-tauri/src/connectors/error.rs b/app/src-tauri/src/connectors/error.rs new file mode 100644 index 00000000..02e9b7d8 --- /dev/null +++ b/app/src-tauri/src/connectors/error.rs @@ -0,0 +1,175 @@ +//! Why a connector call failed. Each reason has a fixed code that the interface turns into a sentence from its +//! strings. The messages are for the log, and none of them can hold a token, a code, a client ID, or a URL with a +//! query: they name the connector and the kind of failure only. + +use std::fmt; + +use crate::ipc::IpcError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Failure { + /// Work offline is on, so nothing signs in, refreshes, or sends. + Offline, + /// No connector has this ID. + Unknown, + /// The owner has not put a client ID in the connectors file or the build. + NotConfigured, + /// The person has not connected it. + NotConnected, + /// The sign-in no longer works, so the person has to connect again. + Expired, + /// The connection was made without the access a feature asked for. + MissingAccess, + /// A sign-in for this connector is already waiting for the browser. + Busy, + /// The person canceled in OpenNote. + Canceled, + /// Nobody finished signing in within five minutes. + TimedOut, + /// The person said no in the service's own page. + Denied, + /// The browser came back with the wrong state value or a damaged address. + Mismatch, + /// The loopback port could not be opened. + PortInUse, + /// The browser could not be opened. + BrowserFailed, + /// The service could not be reached. + Network, + /// The service answered, but refused the code, the token, or the request. + Rejected, + /// A school address or a token that is not acceptable. + BadInput, + /// An address that is not one of the connector's allowed hosts. + ForeignHost, + /// Windows Credential Manager, or the connections file, could not be used. + Storage, +} + +impl Failure { + /// The code the interface receives. + pub const fn code(self) -> &'static str { + match self { + Self::Offline => "offline", + Self::Unknown => "unknown", + Self::NotConfigured => "notConfigured", + Self::NotConnected => "notConnected", + Self::Expired => "expired", + Self::MissingAccess => "missingAccess", + Self::Busy => "busy", + Self::Canceled => "canceled", + Self::TimedOut => "timedOut", + Self::Denied => "denied", + Self::Mismatch => "mismatch", + Self::PortInUse => "portInUse", + Self::BrowserFailed => "browserFailed", + Self::Network => "network", + Self::Rejected => "rejected", + Self::BadInput => "badInput", + Self::ForeignHost => "foreignHost", + Self::Storage => "storage", + } + } + + const fn text(self) -> &'static str { + match self { + Self::Offline => "Work offline is on", + Self::Unknown => "no such connector", + Self::NotConfigured => "no client ID is set up", + Self::NotConnected => "not connected", + Self::Expired => "the sign-in expired", + Self::MissingAccess => "the connection lacks the access asked for", + Self::Busy => "a sign-in is already waiting", + Self::Canceled => "canceled", + Self::TimedOut => "the sign-in timed out", + Self::Denied => "the person denied access", + Self::Mismatch => "the browser's answer didn't match the sign-in", + Self::PortInUse => "couldn't open the loopback port", + Self::BrowserFailed => "couldn't open the browser", + Self::Network => "couldn't reach the service", + Self::Rejected => "the service refused it", + Self::BadInput => "the address or token isn't acceptable", + Self::ForeignHost => "the address isn't an allowed host", + Self::Storage => "couldn't store the connection", + } + } +} + +/// A failure and the connector it happened to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConnectorError { + pub failure: Failure, + pub connector: String, +} + +impl ConnectorError { + pub fn new(failure: Failure, connector: &str) -> Self { + Self { + failure, + connector: connector.to_owned(), + } + } + + pub const fn code(&self) -> &'static str { + self.failure.code() + } +} + +impl fmt::Display for ConnectorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}: {}", self.connector, self.failure.text()) + } +} + +impl std::error::Error for ConnectorError {} + +impl From for IpcError { + fn from(error: ConnectorError) -> Self { + IpcError { + code: error.code().to_owned(), + message: error.to_string(), + field: Some(error.connector), + } + } +} + +pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_failure_has_its_own_code() { + let all = [ + Failure::Offline, + Failure::Unknown, + Failure::NotConfigured, + Failure::NotConnected, + Failure::Expired, + Failure::MissingAccess, + Failure::Busy, + Failure::Canceled, + Failure::TimedOut, + Failure::Denied, + Failure::Mismatch, + Failure::PortInUse, + Failure::BrowserFailed, + Failure::Network, + Failure::Rejected, + Failure::BadInput, + Failure::ForeignHost, + Failure::Storage, + ]; + let codes: std::collections::BTreeSet<_> = all.iter().map(|failure| failure.code()).collect(); + assert_eq!(codes.len(), all.len()); + } + + #[test] + fn the_ipc_error_names_the_connector_and_nothing_else() { + let error: IpcError = ConnectorError::new(Failure::Rejected, "dropbox").into(); + assert_eq!(error.code, "rejected"); + assert_eq!(error.field.as_deref(), Some("dropbox")); + assert_eq!(error.message, "dropbox: the service refused it"); + } +} diff --git a/app/src-tauri/src/connectors/files.rs b/app/src-tauri/src/connectors/files.rs new file mode 100644 index 00000000..654cc6d2 --- /dev/null +++ b/app/src-tauri/src/connectors/files.rs @@ -0,0 +1,318 @@ +//! The two small files of the connectors, both in this device's folder and both free of secrets. +//! +//! - `connectors.json` is written by the owner, not by OpenNote. It holds the client ID of each service. +//! Where a service wants one, it also holds the client secret of the app registration, which belongs to the +//! app and not to the person. A build with no client IDs works once this file has them. docs/CONNECTORS.md +//! says how to fill it in. +//! - `connections.json` is written by OpenNote. It holds only what Settings shows about each connection: the +//! account name, the access that was granted, and when it was connected. A token never goes into it. + +use std::{collections::BTreeMap, fs, io, path::Path}; + +use serde::{Deserialize, Serialize}; + +use super::secret::Secret; +use crate::settings::file::write_json; + +pub const CONFIG_FILE: &str = "connectors.json"; +pub const CONNECTIONS_FILE: &str = "connections.json"; + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", default)] +struct ConfigFile { + clients: BTreeMap, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", default)] +struct ClientEntry { + client_id: String, + client_secret: String, + redirect_port: Option, +} + +/// An app registration: what the service knows OpenNote by. +#[derive(Clone)] +pub struct Client { + pub id: String, + pub secret: Option, + /// A port to use instead of the default of a service that needs a fixed one. + pub redirect_port: Option, +} + +/// The longest client ID or secret accepted. Real ones are far shorter. +const MAX_CLIENT_TEXT: usize = 512; + +fn clean(text: &str) -> Option { + let text = text.trim(); + let usable = + !text.is_empty() && text.len() <= MAX_CLIENT_TEXT && !text.chars().any(|c| c.is_whitespace() || c.is_control()); + usable.then(|| text.to_owned()) +} + +/// The client IDs compiled into this build, from `OPENNOTE__CLIENT_ID` and `..._CLIENT_SECRET` at build +/// time. The repository holds none, so a build without them has none. +fn built_in(connector: &str) -> (Option<&'static str>, Option<&'static str>) { + macro_rules! pair { + ($name:literal) => { + ( + option_env!(concat!("OPENNOTE_", $name, "_CLIENT_ID")), + option_env!(concat!("OPENNOTE_", $name, "_CLIENT_SECRET")), + ) + }; + } + match connector { + "microsoft" => pair!("MICROSOFT"), + "google" => pair!("GOOGLE"), + "slack" => pair!("SLACK"), + "dropbox" => pair!("DROPBOX"), + "box" => pair!("BOX"), + "vimeo" => pair!("VIMEO"), + _ => (None, None), + } +} + +/// The app registration for a connector: the connectors file first, then the build. A file that is missing or +/// can't be read counts as empty, so a damaged file never blocks the Settings page. +pub fn load_client(config: &Path, connector: &str) -> Option { + let from_file = fs::read_to_string(config) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .and_then(|file| { + file.clients.get(connector).map(|entry| { + ( + clean(&entry.client_id), + clean(&entry.client_secret), + entry.redirect_port, + ) + }) + }); + if let Some((Some(id), secret, redirect_port)) = from_file { + return Some(Client { + id, + secret: secret.map(Secret::new), + redirect_port, + }); + } + let (id, secret) = built_in(connector); + Some(Client { + id: clean(id?)?, + secret: secret.and_then(clean).map(Secret::new), + redirect_port: None, + }) +} + +/// Saves the app registration of a connector in the connectors file, from the Settings card. The other entries and +/// the keys this function doesn't know (such as `redirectPort`) stay as they were. A client ID is not a secret, but a +/// client secret belongs to the app, so it goes in this device's file and nowhere else. +pub fn save_client(config: &Path, connector: &str, id: &str, secret: Option<&str>) -> io::Result<()> { + let bad = |what: &str| io::Error::new(io::ErrorKind::InvalidInput, what.to_owned()); + let id = clean(id).ok_or_else(|| bad("client id"))?; + let secret = match secret.map(str::trim).filter(|text| !text.is_empty()) { + Some(text) => Some(clean(text).ok_or_else(|| bad("client secret"))?), + None => None, + }; + let mut root = fs::read_to_string(config) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .filter(serde_json::Value::is_object) + .unwrap_or_else(|| serde_json::json!({})); + let entry = object_at(object_at(&mut root, "clients"), connector); + let fields = entry.as_object_mut().expect("an object"); + fields.insert("clientId".to_owned(), serde_json::Value::String(id)); + match secret { + Some(text) => fields.insert("clientSecret".to_owned(), serde_json::Value::String(text)), + None => fields.remove("clientSecret"), + }; + write_json(config, &root) +} + +/// The object under `key` of an object, made when it is missing or isn't one. +fn object_at<'a>(parent: &'a mut serde_json::Value, key: &str) -> &'a mut serde_json::Value { + let slot = parent + .as_object_mut() + .expect("an object") + .entry(key.to_owned()) + .or_insert_with(|| serde_json::json!({})); + if !slot.is_object() { + *slot = serde_json::json!({}); + } + slot +} + +/// What a stored credential is: a refresh token to trade for access tokens, or an access token that doesn't expire, +/// such as a pasted personal token. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Holds { + Refresh, + Access, +} + +/// What OpenNote keeps about one connection, outside the credential store. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Connection { + pub account: String, + /// The access the service granted. + pub scopes: Vec, + /// When the person connected, in seconds since 1970. + pub connected_unix: u64, + pub holds: Holds, + /// The school's address, for a connector that needs one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_url: Option, + /// True once the service refused to renew the sign-in. The person connects again to clear it. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub expired: bool, + /// When a feature last used the connection, for the Privacy section. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_used_unix: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +struct ConnectionsFile { + #[serde(default)] + connections: BTreeMap, +} + +/// Reads the connections. A missing or damaged file reads as no connections. +pub fn read_connections(path: &Path) -> BTreeMap { + fs::read_to_string(path) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .map(|file| file.connections) + .unwrap_or_default() +} + +pub fn write_connections(path: &Path, connections: &BTreeMap) -> io::Result<()> { + let file = ConnectionsFile { + connections: connections.clone(), + }; + write_json(path, &serde_json::to_value(file).map_err(io::Error::other)?) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write(dir: &Path, name: &str, text: &str) -> std::path::PathBuf { + let path = dir.join(name); + fs::write(&path, text).expect("writes"); + path + } + + #[test] + fn a_client_comes_from_the_connectors_file() { + let dir = tempfile::tempdir().expect("a temporary folder"); + let path = write( + dir.path(), + CONFIG_FILE, + r#"{"clients":{"google":{"clientId":" example-id ","clientSecret":"example-secret"}, + "dropbox":{"clientId":"d-id","redirectPort":53999}}}"#, + ); + let google = load_client(&path, "google").expect("has a client"); + assert_eq!(google.id, "example-id"); + assert_eq!(google.secret, Some(Secret::new("example-secret"))); + let dropbox = load_client(&path, "dropbox").expect("has a client"); + assert_eq!((dropbox.secret, dropbox.redirect_port), (None, Some(53999))); + assert!(load_client(&path, "slack").is_none()); + } + + #[test] + fn saving_a_client_keeps_the_other_entries_and_the_unknown_keys() { + let dir = tempfile::tempdir().expect("a temporary folder"); + let path = write( + dir.path(), + CONFIG_FILE, + r#"{"clients":{"dropbox":{"clientId":"d-id","redirectPort":53999}},"note":"mine"}"#, + ); + save_client(&path, "google", " g-id ", Some("g-secret")).expect("saves"); + save_client(&path, "dropbox", "d-id-two", None).expect("saves"); + let google = load_client(&path, "google").expect("has a client"); + assert_eq!(google.id, "g-id"); + assert_eq!(google.secret, Some(Secret::new("g-secret"))); + let dropbox = load_client(&path, "dropbox").expect("has a client"); + assert_eq!((dropbox.id.as_str(), dropbox.redirect_port), ("d-id-two", Some(53999))); + let text = fs::read_to_string(&path).expect("reads"); + assert!(text.contains("\"note\""), "{text}"); + } + + #[test] + fn saving_refuses_ids_with_spaces_and_starts_a_missing_file() { + let dir = tempfile::tempdir().expect("a temporary folder"); + let path = dir.path().join("nested").join(CONFIG_FILE); + assert!(save_client(&path, "google", "a b", None).is_err()); + assert!(save_client(&path, "google", "", None).is_err()); + assert!(!path.exists()); + save_client(&path, "google", "g-id", None).expect("saves"); + assert_eq!(load_client(&path, "google").expect("a client").id, "g-id"); + } + + #[test] + fn a_missing_empty_or_damaged_file_means_no_client() { + let dir = tempfile::tempdir().expect("a temporary folder"); + assert!(load_client(&dir.path().join("none.json"), "google").is_none()); + for text in [ + "", + "not json", + "[]", + r#"{"clients":{"google":{"clientId":""}}}"#, + r#"{"clients":{"google":{"clientId":"a b"}}}"#, + ] { + let path = write(dir.path(), "c.json", text); + assert!(load_client(&path, "google").is_none(), "{text}"); + } + } + + #[test] + fn this_build_ships_with_no_client_ids() { + // CI builds the app without them, and the repository holds none. A build for release sets them itself. + if std::env::var("OPENNOTE_GOOGLE_CLIENT_ID").is_err() { + assert!(load_client(Path::new("missing.json"), "google").is_none()); + } + } + + #[test] + fn connections_round_trip_and_hold_only_metadata() { + let dir = tempfile::tempdir().expect("a temporary folder"); + let path = dir.path().join(CONNECTIONS_FILE); + assert!(read_connections(&path).is_empty()); + let mut all = BTreeMap::new(); + all.insert( + "canvas".to_owned(), + Connection { + account: "Sam Student".into(), + scopes: vec![], + connected_unix: 1_790_000_000, + holds: Holds::Access, + base_url: Some("https://school.example".into()), + expired: false, + last_used_unix: None, + }, + ); + write_connections(&path, &all).expect("writes"); + assert_eq!(read_connections(&path), all); + let text = fs::read_to_string(&path).expect("reads"); + let value: serde_json::Value = serde_json::from_str(&text).expect("json"); + let keys: Vec<&str> = value["connections"]["canvas"] + .as_object() + .expect("an object") + .keys() + .map(String::as_str) + .collect(); + for key in &keys { + assert!( + ["account", "scopes", "connectedUnix", "holds", "baseUrl"].contains(key), + "unexpected key {key}" + ); + } + } + + #[test] + fn a_damaged_connections_file_reads_as_none() { + let dir = tempfile::tempdir().expect("a temporary folder"); + let path = write(dir.path(), CONNECTIONS_FILE, "{ broken"); + assert!(read_connections(&path).is_empty()); + } +} diff --git a/app/src-tauri/src/connectors/http.rs b/app/src-tauri/src/connectors/http.rs new file mode 100644 index 00000000..d3868004 --- /dev/null +++ b/app/src-tauri/src/connectors/http.rs @@ -0,0 +1,357 @@ +//! The HTTP client of the connectors, built the way the updater's is (`crates/updater/src/fetch.rs`): `ureq` with +//! Windows' TLS stack and certificate store, HTTPS only, and a pinned list of hosts. Every address, and every +//! redirect, is checked against the connector's hosts before a byte is sent, so a token can only go to the +//! service it belongs to. The messages of an error never hold an address, because an address can hold a code. + +use std::{io::Read, time::Duration}; + +use url::Url; + +use super::registry::Method; + +/// The hosts a connector may talk to, as the client reads them. A user name before the host +/// (`graph.microsoft.com@evil.example`), a backslash, and any port but the default don't pass for a pinned host. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostPolicy { + hosts: Vec, +} + +impl HostPolicy { + pub fn new(hosts: impl IntoIterator>) -> HostPolicy { + HostPolicy { + hosts: hosts.into_iter().map(|host| host.into().to_ascii_lowercase()).collect(), + } + } + + pub fn hosts(&self) -> &[String] { + &self.hosts + } + + /// Whether `text` is an HTTPS address on one of the hosts. Unit tests also accept `http://127.0.0.1:` + /// when `127.0.0.1` is listed, so they can serve answers from a local mock. That branch is only in tests and + /// in builds with the `test-endpoints` feature (see `test_hook.rs`), never in a release. + pub fn allows(&self, text: &str) -> bool { + if text.contains('\\') { + return false; + } + let Ok(url) = Url::parse(text) else { return false }; + if !url.username().is_empty() || url.password().is_some() { + return false; + } + let Some(host) = url.host_str().map(str::to_ascii_lowercase) else { + return false; + }; + let local_test = + (cfg!(test) || cfg!(feature = "test-endpoints")) && url.scheme() == "http" && host == "127.0.0.1"; + if !local_test && (url.scheme() != "https" || url.port().is_some()) { + return false; + } + self.hosts.contains(&host) + } +} + +#[derive(Clone)] +pub enum Body { + /// Sent as `application/x-www-form-urlencoded`. + Form(Vec<(String, String)>), + Bytes { + content_type: String, + data: Vec, + }, +} + +pub struct HttpRequest { + pub method: Method, + pub url: String, + pub headers: Vec<(String, String)>, + pub body: Option, + /// Whether a redirect is followed, on a pinned host only. When false the 3xx comes back with its `Location`, for + /// a step whose next address is on another service, such as a Canvas upload that answers with the confirm + /// address on the Canvas host. + pub follow_redirects: bool, +} + +impl HttpRequest { + pub fn new(method: Method, url: impl Into) -> HttpRequest { + HttpRequest { + method, + url: url.into(), + headers: Vec::new(), + body: None, + follow_redirects: true, + } + } + + pub fn header(mut self, name: &str, value: impl Into) -> HttpRequest { + self.headers.push((name.to_owned(), value.into())); + self + } + + pub fn form(mut self, fields: Vec<(String, String)>) -> HttpRequest { + self.body = Some(Body::Form(fields)); + self + } +} + +pub struct HttpResponse { + pub status: u16, + pub content_type: Option, + pub body: Vec, + /// The `Location` header of an answer that is not a redirect: where a resumable upload goes on. + pub location: Option, +} + +impl HttpResponse { + pub fn ok(&self) -> bool { + (200..300).contains(&self.status) + } + + /// The body as JSON, or None when it isn't. + pub fn json(&self) -> Option { + serde_json::from_slice(&self.body).ok() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpError { + /// The address, or a redirect, is not on a pinned host. + ForeignHost, + /// The service could not be reached, or the connection broke. + Network, + /// The answer is longer than the limit. + TooLarge, +} + +/// Sends one request. Tests serve answers from a mock instead of the network. +pub trait Http: Send + Sync { + fn send(&self, policy: &HostPolicy, request: &HttpRequest, max_bytes: usize) -> Result; +} + +/// The most redirects one request follows. +const MAX_REDIRECTS: usize = 3; + +pub struct UreqHttp { + agent: ureq::Agent, +} + +impl Default for UreqHttp { + fn default() -> Self { + use ureq::tls::{RootCerts, TlsConfig, TlsProvider}; + let tls = TlsConfig::builder() + .provider(TlsProvider::NativeTls) + .root_certs(RootCerts::PlatformVerifier) + .build(); + let config = ureq::Agent::config_builder() + .tls_config(tls) + .https_only(!(cfg!(test) || cfg!(feature = "test-endpoints"))) + .max_redirects(0) + .max_redirects_will_error(false) + .http_status_as_error(false) + .timeout_connect(Some(Duration::from_secs(10))) + .timeout_global(Some(Duration::from_secs(30))) + .user_agent(concat!("OpenNote/", env!("CARGO_PKG_VERSION"))) + .build(); + Self { agent: config.into() } + } +} + +/// Form fields as a request body. +pub fn encode_form(fields: &[(String, String)]) -> String { + url::form_urlencoded::Serializer::new(String::new()) + .extend_pairs(fields) + .finish() +} + +impl UreqHttp { + fn once( + &self, + request: &HttpRequest, + url: &str, + drop_body: bool, + ) -> Result, HttpError> { + let mut builder = ureq::http::Request::builder().method(request.method.as_str()).uri(url); + for (name, value) in &request.headers { + builder = builder.header(name.as_str(), value.as_str()); + } + let body = match (&request.body, drop_body) { + (Some(Body::Form(fields)), false) => { + builder = builder.header("Content-Type", "application/x-www-form-urlencoded"); + Some(encode_form(fields).into_bytes()) + } + (Some(Body::Bytes { content_type, data }), false) => { + builder = builder.header("Content-Type", content_type.as_str()); + Some(data.clone()) + } + _ => None, + }; + let result = match body { + Some(data) => self.agent.run(builder.body(data).map_err(|_| HttpError::Network)?), + None => self.agent.run(builder.body(()).map_err(|_| HttpError::Network)?), + }; + result.map_err(|_| HttpError::Network) + } +} + +impl Http for UreqHttp { + fn send(&self, policy: &HostPolicy, request: &HttpRequest, max_bytes: usize) -> Result { + let mut current = request.url.clone(); + let mut drop_body = false; + for _ in 0..=MAX_REDIRECTS { + if !policy.allows(¤t) { + return Err(HttpError::ForeignHost); + } + let response = self.once(request, ¤t, drop_body)?; + let status = response.status().as_u16(); + // A 308 with no Location is "Resume Incomplete" of a chunked upload (Google), an answer and not a redirect. + let redirect = matches!(status, 301 | 302 | 303 | 307 | 308) + && !(status == 308 && response.headers().get("location").is_none()); + if redirect && request.follow_redirects { + let location = response + .headers() + .get("location") + .and_then(|value| value.to_str().ok()) + .ok_or(HttpError::Network)?; + let next = Url::parse(¤t) + .and_then(|base| base.join(location)) + .map_err(|_| HttpError::Network)?; + // A 303 turns the call into a plain GET. The others keep the method and the body. + drop_body |= status == 303; + current = next.into(); + continue; + } + let content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + // A redirect handed back unfollowed carries its target as a full address. + let location = response + .headers() + .get("location") + .and_then(|value| value.to_str().ok()) + .map( + |location| match Url::parse(¤t).and_then(|base| base.join(location)) { + Ok(full) if redirect => full.into(), + _ => location.to_owned(), + }, + ); + let mut body = Vec::new(); + let limit = u64::try_from(max_bytes).unwrap_or(u64::MAX).saturating_add(1); + response + .into_body() + .into_reader() + .take(limit) + .read_to_end(&mut body) + .map_err(|_| HttpError::Network)?; + if body.len() > max_bytes { + return Err(HttpError::TooLarge); + } + return Ok(HttpResponse { + status, + content_type, + body, + location, + }); + } + Err(HttpError::Network) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy() -> HostPolicy { + HostPolicy::new(["graph.microsoft.com", "login.microsoftonline.com"]) + } + + #[test] + fn allows_https_on_a_pinned_host_in_any_case() { + let policy = policy(); + assert!(policy.allows("https://graph.microsoft.com/v1.0/me")); + assert!(policy.allows("https://GRAPH.microsoft.com/v1.0/me?x=1")); + assert!(policy.allows("https://login.microsoftonline.com:443/common/oauth2/v2.0/token")); + } + + #[test] + fn refuses_every_other_way_of_naming_a_server() { + let policy = policy(); + for url in [ + "http://graph.microsoft.com/v1.0/me", + "https://graph.microsoft.com.evil.example/v1.0/me", + "https://evil.example/graph.microsoft.com", + "https://graph.microsoft.com@evil.example/", + "https://evil.example@graph.microsoft.com/", + "https://graph.microsoft.com:8443/", + "https://graph.microsoft.com\\@evil.example/", + "https://127.0.0.1/", + "ftp://graph.microsoft.com/", + "//graph.microsoft.com/", + "graph.microsoft.com", + "", + ] { + assert!(!policy.allows(url), "{url}"); + } + } + + #[test] + fn a_plain_http_address_passes_only_for_the_local_mock_in_tests() { + let policy = HostPolicy::new(["127.0.0.1"]); + assert!(policy.allows("http://127.0.0.1:4555/token")); + assert!(!HostPolicy::new(["graph.microsoft.com"]).allows("http://127.0.0.1:4555/token")); + assert!(!policy.allows("http://localhost:4555/token")); + assert!(!policy.allows("http://127.0.0.1.evil.example:4555/token")); + } + + /// Answers one request on a local port with `answer`, and gives the address. + fn answer_once(answer: &'static str) -> String { + use std::{io::Write, net::TcpListener}; + let listener = TcpListener::bind("127.0.0.1:0").expect("binds"); + let address = format!( + "http://127.0.0.1:{}/upload", + listener.local_addr().expect("port").port() + ); + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let mut buffer = [0u8; 4096]; + let _ = stream.read(&mut buffer); + let _ = stream.write_all(answer.as_bytes()); + } + }); + address + } + + const MOVED: &str = concat!( + "HTTP/1.1 301 Moved\r\nLocation: https://canvas.example/api/v1/files/1/create_success\r\n", + "Content-Length: 0\r\nConnection: close\r\n\r\n" + ); + + #[test] + fn an_unfollowed_redirect_to_another_host_comes_back_with_its_location() { + let address = answer_once(MOVED); + let mut request = HttpRequest::new(Method::Post, address); + request.follow_redirects = false; + let response = UreqHttp::default() + .send(&HostPolicy::new(["127.0.0.1"]), &request, 1024) + .unwrap_or_else(|_| panic!("the 301 comes back")); + assert_eq!(response.status, 301); + assert_eq!( + response.location.as_deref(), + Some("https://canvas.example/api/v1/files/1/create_success") + ); + } + + #[test] + fn a_followed_redirect_to_another_host_is_refused() { + let address = answer_once(MOVED); + let request = HttpRequest::new(Method::Post, address); + let result = UreqHttp::default().send(&HostPolicy::new(["127.0.0.1"]), &request, 1024); + assert_eq!(result.err(), Some(HttpError::ForeignHost)); + } + + #[test] + fn form_fields_are_encoded() { + let fields = vec![("a".to_owned(), "b c&d".to_owned()), ("e".to_owned(), "f/g".to_owned())]; + assert_eq!(encode_form(&fields), "a=b+c%26d&e=f%2Fg"); + } +} diff --git a/app/src-tauri/src/connectors/loopback.rs b/app/src-tauri/src/connectors/loopback.rs new file mode 100644 index 00000000..cb623e33 --- /dev/null +++ b/app/src-tauri/src/connectors/loopback.rs @@ -0,0 +1,247 @@ +//! The loopback listener that receives the browser's return from a sign-in (RFC 8252 section 7.3). It binds +//! `127.0.0.1` only, on a random port unless the service needs a fixed one, and it answers on one exact path. +//! +//! It ends at the first request that is the sign-in coming back: the right path and a `state` value to check. A +//! request for anything else (the wrong path, the wrong method, the wrong `Host`, such as a page that probes the +//! port) gets a plain error and is ignored. A return with the wrong or missing `state` ends the attempt, because it +//! means the answer isn't for this sign-in. After the end the socket is closed, so a second request finds nothing. +//! Five minutes without a return ends it too. The page it shows says only that the tab can be closed, and it +//! never repeats anything from the address. + +use std::{ + io::{self, Read, Write}, + net::{Ipv4Addr, Shutdown, TcpListener, TcpStream}, + sync::atomic::{AtomicBool, Ordering}, + thread, + time::{Duration, Instant}, +}; + +use super::{pkce, secret::Secret}; + +/// The only path that counts as the sign-in coming back. +pub const CALLBACK_PATH: &str = "/callback"; + +/// How long a sign-in waits for the person to finish in the browser. +pub const WAIT: Duration = Duration::from_secs(5 * 60); + +/// The longest request head read, in bytes. +const MAX_HEAD: usize = 8 * 1024; +const POLL: Duration = Duration::from_millis(25); +const READ_LIMIT: Duration = Duration::from_secs(2); + +/// What came back in the address. +#[derive(Debug, PartialEq, Eq)] +pub enum Callback { + /// The authorization code. It works once, and only with this sign-in's verifier. + Code(Secret), + /// The service refused, such as `access_denied`. Only a short code of letters, digits, and `_` is kept. + Refused(String), +} + +/// Why the wait ended without a code. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Ended { + Canceled, + TimedOut, + /// The return had the right state value but no usable code. + Mismatch, + Io, +} + +pub struct Listener { + socket: TcpListener, + port: u16, +} + +impl Listener { + /// Listens on `127.0.0.1`. Port 0 picks a free port. + pub fn bind(port: u16) -> io::Result { + let socket = TcpListener::bind((Ipv4Addr::LOCALHOST, port))?; + socket.set_nonblocking(true)?; + let port = socket.local_addr()?.port(); + Ok(Listener { socket, port }) + } + + #[cfg(test)] + pub fn port(&self) -> u16 { + self.port + } + + /// The address to give the service as the redirect URI. + pub fn redirect_uri(&self) -> String { + format!("http://127.0.0.1:{}{CALLBACK_PATH}", self.port) + } + + /// Waits for the return. The listener is consumed, so the port closes as soon as this ends. + pub fn wait(self, state: &str, timeout: Duration, cancel: &AtomicBool) -> Result { + let deadline = Instant::now() + timeout; + loop { + if cancel.load(Ordering::Relaxed) { + return Err(Ended::Canceled); + } + if Instant::now() >= deadline { + return Err(Ended::TimedOut); + } + match self.socket.accept() { + Ok((stream, _)) => { + if let Some(end) = self.serve(stream, state) { + return end; + } + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => thread::sleep(POLL), + Err(_) => return Err(Ended::Io), + } + } + } + + /// Answers one connection. Some(..) ends the wait, and None keeps waiting. + fn serve(&self, mut stream: TcpStream, state: &str) -> Option> { + // A socket accepted from a non-blocking listener may be non-blocking too. + stream.set_nonblocking(false).ok()?; + stream.set_read_timeout(Some(READ_LIMIT)).ok()?; + stream.set_write_timeout(Some(READ_LIMIT)).ok()?; + let head = read_head(&mut stream)?; + let verdict = judge(&head, self.port, state); + let _ = write_page(&mut stream, verdict.status, verdict.page); + verdict.end + } +} + +/// Reads up to the blank line that ends the request head. +fn read_head(stream: &mut TcpStream) -> Option { + let mut head = Vec::new(); + let mut buffer = [0u8; 1024]; + while !head.windows(4).any(|window| window == b"\r\n\r\n") { + if head.len() >= MAX_HEAD { + return None; + } + match stream.read(&mut buffer) { + Ok(0) | Err(_) => return None, + Ok(count) => head.extend_from_slice(&buffer[..count]), + } + } + Some(String::from_utf8_lossy(&head).into_owned()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Page { + Done, + Failed, + Nothing, +} + +#[derive(Debug, PartialEq, Eq)] +struct Verdict { + status: u16, + page: Page, + end: Option>, +} + +impl Verdict { + fn ignore(status: u16) -> Verdict { + Verdict { + status, + page: Page::Nothing, + end: None, + } + } + + fn finish(end: Result) -> Verdict { + match end { + Ok(Callback::Code(_)) => Verdict { + status: 200, + page: Page::Done, + end: Some(end), + }, + _ => Verdict { + status: 400, + page: Page::Failed, + end: Some(end), + }, + } + } +} + +/// Decides what a request head means. A pure function, so every case is tested without a socket. +fn judge(head: &str, port: u16, state: &str) -> Verdict { + let mut lines = head.split("\r\n"); + let mut request = lines.next().unwrap_or_default().split(' '); + let (method, target) = (request.next().unwrap_or_default(), request.next().unwrap_or_default()); + if method != "GET" { + return Verdict::ignore(405); + } + let host = lines + .filter_map(|line| line.split_once(':')) + .find(|(name, _)| name.trim().eq_ignore_ascii_case("host")) + .map(|(_, value)| value.trim()); + // A page that makes the browser reach this port under another name is not the sign-in coming back. + if host != Some(format!("127.0.0.1:{port}").as_str()) { + return Verdict::ignore(400); + } + let (path, query) = target.split_once('?').unwrap_or((target, "")); + if path != CALLBACK_PATH { + return Verdict::ignore(404); + } + let pairs: Vec<(String, String)> = url::form_urlencoded::parse(query.as_bytes()) + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + let get = |name: &str| { + pairs + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + }; + // Any page in the browser can send a request here, and the fixed ports are public, so a wrong + // or missing state is answered and ignored: it never ends the wait for the real return. + if !get("state").is_some_and(|returned| pkce::same(returned, state)) { + return Verdict { + status: 400, + page: Page::Failed, + end: None, + }; + } + if let Some(error) = get("error") { + let kept: String = error + .chars() + .filter(|c| c.is_ascii_alphanumeric() || *c == '_') + .take(64) + .collect(); + return Verdict::finish(Ok(Callback::Refused(kept))); + } + match get("code") { + Some(code) if !code.is_empty() && code.len() <= 4096 => Verdict::finish(Ok(Callback::Code(Secret::new(code)))), + _ => Verdict::finish(Err(Ended::Mismatch)), + } +} + +const DONE_PAGE: &str = "OpenNote\ +\ +\ +

You can close this tab

OpenNote finished signing in. Go back to the app.

"; + +const FAILED_PAGE: &str = "OpenNote\ +\ +\ +

You can close this tab

The sign-in didn't finish. Go back to OpenNote to try again.

"; + +fn write_page(stream: &mut TcpStream, status: u16, page: Page) -> io::Result<()> { + let (reason, body) = match (status, page) { + (200, _) => ("OK", DONE_PAGE), + (_, Page::Failed) => ("Bad Request", FAILED_PAGE), + (404, _) => ("Not Found", ""), + (405, _) => ("Method Not Allowed", ""), + _ => ("Bad Request", ""), + }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\n\ +Cache-Control: no-store\r\nReferrer-Policy: no-referrer\r\nX-Content-Type-Options: nosniff\r\n\ +Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes())?; + stream.flush()?; + stream.shutdown(Shutdown::Write) +} + +#[cfg(test)] +mod tests; diff --git a/app/src-tauri/src/connectors/loopback/tests.rs b/app/src-tauri/src/connectors/loopback/tests.rs new file mode 100644 index 00000000..4cc86741 --- /dev/null +++ b/app/src-tauri/src/connectors/loopback/tests.rs @@ -0,0 +1,198 @@ +use std::{net::TcpStream, sync::Arc}; + +use super::*; + +const STATE: &str = "state-value-0123456789abcdef-xyz"; + +fn head(target: &str, port: u16) -> String { + format!("GET {target} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nUser-Agent: test\r\n\r\n") +} + +fn judge_target(target: &str) -> Verdict { + judge(&head(target, 5555), 5555, STATE) +} + +#[test] +fn a_return_with_the_right_state_and_a_code_ends_the_wait() { + let verdict = judge_target(&format!("/callback?code=abc%2F123&state={STATE}")); + assert_eq!(verdict.status, 200); + assert_eq!(verdict.page, Page::Done); + assert_eq!(verdict.end, Some(Ok(Callback::Code(Secret::new("abc/123"))))); +} + +#[test] +fn the_wrong_state_is_refused_and_the_wait_goes_on() { + let verdict = judge_target("/callback?code=abc&state=somebody-elses-state-value-123456"); + assert_eq!(verdict.end, None); + assert_eq!(verdict.status, 400); + assert_eq!(verdict.page, Page::Failed); +} + +#[test] +fn a_missing_state_is_refused_and_ignored_too() { + assert_eq!(judge_target("/callback?code=abc").end, None); + assert_eq!(judge_target("/callback").end, None); +} + +#[test] +fn a_return_with_the_state_but_no_code_is_not_accepted() { + assert_eq!( + judge_target(&format!("/callback?state={STATE}")).end, + Some(Err(Ended::Mismatch)) + ); + assert_eq!( + judge_target(&format!("/callback?code=&state={STATE}")).end, + Some(Err(Ended::Mismatch)) + ); +} + +#[test] +fn the_wrong_path_is_ignored_and_the_wait_goes_on() { + for target in [ + "/", + "/favicon.ico", + "/callback/", + "/callbackx", + "/x/callback", + "/CALLBACK", + ] { + let verdict = judge_target(&format!("{target}?code=abc&state={STATE}")); + assert_eq!(verdict.status, 404, "{target}"); + assert_eq!(verdict.end, None, "{target}"); + } +} + +#[test] +fn other_methods_and_other_host_names_are_ignored() { + let post = format!("POST /callback?code=a&state={STATE} HTTP/1.1\r\nHost: 127.0.0.1:5555\r\n\r\n"); + assert_eq!(judge(&post, 5555, STATE).end, None); + let rebound = format!("GET /callback?code=a&state={STATE} HTTP/1.1\r\nHost: evil.example:5555\r\n\r\n"); + assert_eq!(judge(&rebound, 5555, STATE).end, None); + let other_port = format!("GET /callback?code=a&state={STATE} HTTP/1.1\r\nHost: 127.0.0.1:6666\r\n\r\n"); + assert_eq!(judge(&other_port, 5555, STATE).end, None); + let no_host = format!("GET /callback?code=a&state={STATE} HTTP/1.1\r\n\r\n"); + assert_eq!(judge(&no_host, 5555, STATE).end, None); +} + +#[test] +fn a_refusal_keeps_only_a_short_plain_code() { + let verdict = judge_target(&format!( + "/callback?error=access_denied&state={STATE}&error_description=%3Cscript%3E" + )); + assert_eq!(verdict.end, Some(Ok(Callback::Refused("access_denied".into())))); + let odd = judge_target(&format!("/callback?error=%3Cb%3Ex%3C%2Fb%3E&state={STATE}")); + assert_eq!(odd.end, Some(Ok(Callback::Refused("bxb".into())))); +} + +/// Sends one request and reads the whole answer. +fn request(port: u16, target: &str) -> std::io::Result { + let mut stream = TcpStream::connect((Ipv4Addr::LOCALHOST, port))?; + stream.set_read_timeout(Some(Duration::from_secs(3)))?; + stream.write_all(head(target, port).as_bytes())?; + let mut answer = String::new(); + stream.read_to_string(&mut answer)?; + Ok(answer) +} + +fn never_canceled() -> AtomicBool { + AtomicBool::new(false) +} + +#[test] +fn accepts_one_return_then_closes_the_port() { + let listener = Listener::bind(0).expect("binds"); + let port = listener.port(); + assert_eq!(listener.redirect_uri(), format!("http://127.0.0.1:{port}/callback")); + let client = thread::spawn(move || { + // A stray request first, as a browser makes for the icon, then the real return. + let stray = request(port, "/favicon.ico").expect("answered"); + let page = request(port, &format!("/callback?code=the-code&state={STATE}")).expect("answered"); + (stray, page) + }); + let outcome = listener.wait(STATE, Duration::from_secs(10), &never_canceled()); + assert_eq!(outcome, Ok(Callback::Code(Secret::new("the-code")))); + let (stray, page) = client.join().expect("the client finished"); + assert!(stray.starts_with("HTTP/1.1 404"), "{stray}"); + assert!(page.starts_with("HTTP/1.1 200"), "{page}"); + assert!(page.contains("You can close this tab")); + // The listener is gone, so a second request, with the same good code or not, reaches nothing. + assert!(TcpStream::connect((Ipv4Addr::LOCALHOST, port)).is_err()); +} + +#[test] +fn the_page_never_repeats_anything_from_the_address() { + let listener = Listener::bind(0).expect("binds"); + let port = listener.port(); + let client = + thread::spawn(move || request(port, "/callback?error=%3Cscript%3Ealert(1)&state=wrong&code=top-secret")); + let outcome = listener.wait(STATE, Duration::from_millis(500), &never_canceled()); + assert_eq!(outcome, Err(Ended::TimedOut)); + let page = client.join().expect("finished").expect("answered"); + for echoed in ["script", "alert", "top-secret", "wrong"] { + assert!(!page.contains(echoed), "the page repeated {echoed}"); + } + assert!(page.contains("Referrer-Policy: no-referrer")); + assert!(page.contains("Cache-Control: no-store")); +} + +#[test] +fn a_wrong_state_from_another_page_does_not_end_the_real_sign_in() { + let listener = Listener::bind(0).expect("binds"); + let port = listener.port(); + let client = thread::spawn(move || { + let probe = request(port, &format!("/callback?code=c&state=wrong-{STATE}")).expect("answered"); + let page = request(port, &format!("/callback?code=real&state={STATE}")).expect("answered"); + (probe, page) + }); + assert_eq!( + listener.wait(STATE, Duration::from_secs(10), &never_canceled()), + Ok(Callback::Code(Secret::new("real"))) + ); + let (probe, page) = client.join().expect("finished"); + assert!(probe.starts_with("HTTP/1.1 400"), "{probe}"); + assert!(page.starts_with("HTTP/1.1 200"), "{page}"); +} + +#[test] +fn gives_up_after_the_time_limit() { + let listener = Listener::bind(0).expect("binds"); + let started = Instant::now(); + let outcome = listener.wait(STATE, Duration::from_millis(150), &never_canceled()); + assert_eq!(outcome, Err(Ended::TimedOut)); + assert!(started.elapsed() < Duration::from_secs(5)); +} + +#[test] +fn the_real_limit_is_five_minutes() { + assert_eq!(WAIT, Duration::from_secs(300)); +} + +#[test] +fn stops_when_canceled() { + let listener = Listener::bind(0).expect("binds"); + let cancel = Arc::new(AtomicBool::new(false)); + let flag = Arc::clone(&cancel); + let canceler = thread::spawn(move || { + thread::sleep(Duration::from_millis(100)); + flag.store(true, Ordering::Relaxed); + }); + assert_eq!( + listener.wait(STATE, Duration::from_secs(30), &cancel), + Err(Ended::Canceled) + ); + canceler.join().expect("finished"); +} + +#[test] +fn listens_on_the_loopback_address_only() { + let listener = Listener::bind(0).expect("binds"); + let address = listener.socket.local_addr().expect("has an address"); + assert!(address.ip().is_loopback()); + assert_eq!(address.ip().to_string(), "127.0.0.1"); +} + +#[test] +fn a_fixed_port_that_is_taken_is_an_error_not_a_second_listener() { + let first = Listener::bind(0).expect("binds"); + assert!(Listener::bind(first.port()).is_err()); +} diff --git a/app/src-tauri/src/connectors/mock.rs b/app/src-tauri/src/connectors/mock.rs new file mode 100644 index 00000000..d06bd1fb --- /dev/null +++ b/app/src-tauri/src/connectors/mock.rs @@ -0,0 +1,278 @@ +//! A small HTTP server on 127.0.0.1 for the tests, standing in for a service's token and API endpoints. It +//! answers each path from a list of canned replies, in order, repeating the last one, and records every request so +//! a test can check what OpenNote sent. It also has the doubles for the browser and the clock. + +use std::{ + io::{Read, Write}, + net::{Ipv4Addr, TcpListener, TcpStream}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, PoisonError, + }, + thread::{self, JoinHandle}, + time::Duration, +}; + +#[derive(Debug, Clone)] +pub struct Reply { + pub status: u16, + pub body: String, + pub headers: Vec<(String, String)>, +} + +impl Reply { + pub fn json(status: u16, body: serde_json::Value) -> Reply { + Reply { + status, + body: body.to_string(), + headers: vec![("Content-Type".into(), "application/json".into())], + } + } + + pub fn empty(status: u16) -> Reply { + Reply { + status, + body: String::new(), + headers: Vec::new(), + } + } + + pub fn redirect(location: &str) -> Reply { + Reply { + status: 302, + body: String::new(), + headers: vec![("Location".into(), location.into())], + } + } +} + +#[derive(Debug, Clone)] +pub struct Recorded { + pub method: String, + /// The path and the query. + pub target: String, + pub headers: Vec<(String, String)>, + pub body: String, +} + +impl Recorded { + pub fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + /// The value of a field of a form body. + pub fn field(&self, name: &str) -> Option { + url::form_urlencoded::parse(self.body.as_bytes()) + .find(|(key, _)| key == name) + .map(|(_, value)| value.into_owned()) + } +} + +type Routes = Arc)>>>; + +pub struct MockServer { + port: u16, + routes: Routes, + seen: Arc>>, + stop: Arc, + worker: Option>, +} + +impl MockServer { + pub fn start() -> MockServer { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("binds a port"); + listener.set_nonblocking(true).expect("non-blocking"); + let port = listener.local_addr().expect("has an address").port(); + let routes: Routes = Arc::default(); + let seen: Arc>> = Arc::default(); + let stop = Arc::new(AtomicBool::new(false)); + let worker = { + let (routes, seen, stop) = (Arc::clone(&routes), Arc::clone(&seen), Arc::clone(&stop)); + thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + match listener.accept() { + Ok((stream, _)) => serve(stream, &routes, &seen), + Err(_) => thread::sleep(Duration::from_millis(5)), + } + } + }) + }; + MockServer { + port, + routes, + seen, + stop, + worker: Some(worker), + } + } + + /// Answers `path` with these replies in turn. + pub fn route(&self, path: &str, replies: Vec) -> &MockServer { + self.routes + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push((path.to_owned(), replies)); + self + } + + pub fn clear_routes(&self) { + self.routes.lock().unwrap_or_else(PoisonError::into_inner).clear(); + } + + pub fn url(&self, path: &str) -> String { + format!("http://127.0.0.1:{}{path}", self.port) + } + + pub fn requests(&self) -> Vec { + self.seen.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + /// The requests to one path. + pub fn requests_to(&self, path: &str) -> Vec { + let wanted = |request: &&Recorded| request.target.split('?').next() == Some(path); + self.requests().iter().filter(wanted).cloned().collect() + } +} + +impl Drop for MockServer { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +fn serve(mut stream: TcpStream, routes: &Routes, seen: &Arc>>) { + let _ = stream.set_nonblocking(false); + let _ = stream.set_read_timeout(Some(Duration::from_secs(3))); + let Some(request) = read_request(&mut stream) else { + return; + }; + let path = request.target.split('?').next().unwrap_or_default().to_owned(); + seen.lock().unwrap_or_else(PoisonError::into_inner).push(request); + let reply = { + let mut routes = routes.lock().unwrap_or_else(PoisonError::into_inner); + match routes.iter_mut().find(|(route, _)| *route == path) { + Some((_, replies)) if replies.len() > 1 => replies.remove(0), + Some((_, replies)) => replies.first().cloned().unwrap_or_else(|| Reply::empty(500)), + None => Reply::empty(404), + } + }; + let mut head = format!( + "HTTP/1.1 {} Reply\r\nContent-Length: {}\r\nConnection: close\r\n", + reply.status, + reply.body.len() + ); + for (name, value) in &reply.headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + let _ = stream.write_all(format!("{head}\r\n{}", reply.body).as_bytes()); + let _ = stream.flush(); +} + +fn read_request(stream: &mut TcpStream) -> Option { + let mut data = Vec::new(); + let mut buffer = [0u8; 2048]; + let end = loop { + if let Some(at) = data.windows(4).position(|window| window == b"\r\n\r\n") { + break at + 4; + } + let count = stream.read(&mut buffer).ok().filter(|count| *count > 0)?; + data.extend_from_slice(&buffer[..count]); + }; + let head = String::from_utf8_lossy(&data[..end]).into_owned(); + let mut lines = head.split("\r\n"); + let mut first = lines.next()?.split(' '); + let (method, target) = (first.next()?.to_owned(), first.next()?.to_owned()); + let headers: Vec<(String, String)> = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.trim().to_owned(), value.trim().to_owned())) + .collect(); + let length = headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .and_then(|(_, value)| value.parse::().ok()) + .unwrap_or(0); + while data.len() < end + length { + let count = stream.read(&mut buffer).ok().filter(|count| *count > 0)?; + data.extend_from_slice(&buffer[..count]); + } + Some(Recorded { + method, + target, + headers, + body: String::from_utf8_lossy(&data[end..end + length]).into_owned(), + }) +} + +/// Helpers that make a connector entry whose endpoints are on the mock server. +pub mod fixtures { + use super::MockServer; + use crate::connectors::registry::{ + Access, AccountFrom, Auth, ConnectorDef, Group, Method, OAuthDef, Redirect, Revoke, SecretUse, + STANDARD_RESPONSE, + }; + + fn leak(value: T) -> &'static T { + Box::leak(Box::new(value)) + } + + pub fn leak_str(text: String) -> &'static str { + Box::leak(text.into_boxed_str()) + } + + /// OAuth settings that point at `/authorize`, `/token`, `/revoke`, and `/me` of the mock. + pub fn oauth(server: &MockServer) -> OAuthDef { + OAuthDef { + authorize_url: leak_str(server.url("/authorize")), + token_url: leak_str(server.url("/token")), + revoke: Revoke::Form { + url: leak_str(server.url("/revoke")), + with_client: false, + }, + scope_param: "scope", + scope_separator: " ", + extra_params: &[("access_type", "offline")], + pkce: true, + secret: SecretUse::Optional, + redirect: Redirect::AnyPort, + account: AccountFrom::Call { + method: Method::Get, + url: leak_str(server.url("/me")), + pointer: "/email", + }, + response: STANDARD_RESPONSE, + } + } + + pub fn connector(id: &'static str, auth: Auth) -> &'static ConnectorDef { + leak(ConnectorDef { + id, + name: "Example", + group: Group::Other, + features: &["importOneNote"], + hosts: &["127.0.0.1"], + access: &[ + Access { + scope: "scope-a", + capability: "calendarRead", + writes: false, + }, + Access { + scope: "scope-b", + capability: "tasksWrite", + writes: true, + }, + ], + auth, + }) + } + + pub fn registry(defs: Vec<&'static ConnectorDef>) -> &'static [ConnectorDef] { + leak(defs.into_iter().copied().collect::>()).as_slice() + } +} diff --git a/app/src-tauri/src/connectors/mod.rs b/app/src-tauri/src/connectors/mod.rs new file mode 100644 index 00000000..92723053 --- /dev/null +++ b/app/src-tauri/src/connectors/mod.rs @@ -0,0 +1,60 @@ +//! Connectors: the accounts OpenNote can sign in to, such as Microsoft, Google, Dropbox, and a school's Canvas. +//! Settings, then Connectors, lists them. Features that need an account use the small API below. Everything is +//! off until the person connects. +//! +//! - **Registry** (`registry.rs`, `catalog.rs`): every service as data, with its endpoints, its hosts, and the +//! least access each feature needs. +//! - **Sign-in** (`connect.rs`, `loopback.rs`, `oauth.rs`, `pkce.rs`): RFC 8252 for native apps. The person's own +//! browser opens the service's page. The code comes back on `127.0.0.1` with a random port and an exact path, +//! and it is traded over HTTPS for tokens with PKCE. A pasted personal token is checked with one request. +//! - **Secrets** (`store.rs`): tokens live only in Windows Credential Manager, as `OpenNote//`. +//! Settings learns the account name, the granted access, and the connection time, never a token. +//! - **Client IDs** (`files.rs`): OpenNote ships with none. An OAuth connector reads its client ID from +//! `connectors.json` in this device's folder or from the build. Without one it shows "Needs setup", and +//! docs/CONNECTORS.md says how to add it. +//! - **Work offline**: no sign-in, renewal, or request happens while it is on. +//! +//! The API for features is on [`Connectors`], a managed Tauri state that is cheap to clone: +//! +//! - `is_connected(id)` says whether the person connected it and the sign-in still works. +//! - `access_token(id, access)` gives a fresh access token for a capability such as `calendarRead`, or a scope. +//! - `request(id, access, request, max_bytes)` sends a request with the token added, to an allowed host only. +//! +//! The interface can call `connectors_request` for the last one. It never gets the token. + +mod account; +mod accounts; +mod catalog; +mod commands; +mod connect; +mod error; +mod files; +mod http; +mod loopback; +#[cfg(test)] +mod mock; +mod oauth; +mod pkce; +mod registry; +mod secret; +mod service; +mod session; +mod store; +#[cfg(feature = "test-endpoints")] +mod test_hook; +#[cfg(test)] +mod tests; +mod view; + +// The glob also carries the items Tauri generates for each command, which lib.rs needs by path. +pub use accounts::*; +pub use commands::*; +pub use connect::normalize_base_url; +pub use error::{ConnectorError, Failure}; +pub use http::{Body, HostPolicy, Http, HttpError, HttpRequest, HttpResponse, UreqHttp}; +pub use registry::{find, Method, CONNECTORS}; +pub use secret::Secret; +pub use service::{Connectors, Opener, Parts}; +pub use session::DEFAULT_MAX_BYTES; +pub use store::{platform_store, MemoryStore, SecretStore}; +pub use view::{ConnectInput, ConnectorView, Disconnected, RevokeOutcome, StateView}; diff --git a/app/src-tauri/src/connectors/oauth.rs b/app/src-tauri/src/connectors/oauth.rs new file mode 100644 index 00000000..83517755 --- /dev/null +++ b/app/src-tauri/src/connectors/oauth.rs @@ -0,0 +1,193 @@ +//! The OAuth 2.0 requests of a sign-in: the address the browser opens, the exchange of the returned code for tokens, +//! the refresh, and the revoke (RFC 6749, with PKCE from RFC 7636). Every request goes to the connector's own +//! endpoints through the host policy, and no error here says anything but the kind of failure. + +use serde_json::Value; +use url::Url; + +use super::{ + error::Failure, + files::Client, + http::{HostPolicy, Http, HttpRequest, HttpResponse}, + registry::{ConnectorDef, Method, OAuthDef, Revoke, SecretUse}, + secret::Secret, +}; + +/// The largest answer a token endpoint is read for. +const MAX_ANSWER: usize = 256 * 1024; + +/// What the token endpoint gave. Not `Debug`, because it holds tokens. +pub struct Tokens { + pub access: Secret, + pub refresh: Option, + pub expires_in: Option, + pub scopes: Option>, + /// The whole answer, for the ID token and the account name. + pub body: Value, +} + +/// A connector's OAuth settings, its client, and the way to reach it. +pub struct Exchange<'a> { + pub http: &'a dyn Http, + pub policy: &'a HostPolicy, + pub def: &'a ConnectorDef, + pub oauth: &'a OAuthDef, + pub client: &'a Client, +} + +impl Exchange<'_> { + /// The address to open in the browser (RFC 6749 section 4.1.1, with the PKCE challenge of RFC 7636). + pub fn authorize_url(&self, redirect_uri: &str, state: &str, challenge: Option<&str>) -> Option { + let mut url = Url::parse(self.oauth.authorize_url).ok()?; + { + let mut query = url.query_pairs_mut(); + query.append_pair("response_type", "code"); + query.append_pair("client_id", &self.client.id); + query.append_pair("redirect_uri", redirect_uri); + query.append_pair("state", state); + if !self.oauth.scope_param.is_empty() { + query.append_pair( + self.oauth.scope_param, + &self.def.scopes().join(self.oauth.scope_separator), + ); + } + if let (true, Some(challenge)) = (self.oauth.pkce, challenge) { + query.append_pair("code_challenge", challenge); + query.append_pair("code_challenge_method", "S256"); + } + for (name, value) in self.oauth.extra_params { + query.append_pair(name, value); + } + } + Some(url.into()) + } + + /// Trades the returned code for tokens (RFC 6749 section 4.1.3). + pub fn exchange_code(&self, code: &Secret, verifier: Option<&str>, redirect_uri: &str) -> Result { + let mut fields = vec![ + ("grant_type".to_owned(), "authorization_code".to_owned()), + ("code".to_owned(), code.expose().to_owned()), + ("redirect_uri".to_owned(), redirect_uri.to_owned()), + ]; + if let (true, Some(verifier)) = (self.oauth.pkce, verifier) { + fields.push(("code_verifier".to_owned(), verifier.to_owned())); + } + self.parse(self.token_call(fields)?, false) + } + + /// Trades a refresh token for a new access token (RFC 6749 section 6). + pub fn refresh(&self, refresh_token: &Secret) -> Result { + let fields = vec![ + ("grant_type".to_owned(), "refresh_token".to_owned()), + ("refresh_token".to_owned(), refresh_token.expose().to_owned()), + ]; + self.parse(self.token_call(fields)?, true) + } + + /// Tells the service to forget a sign-in. `token` is the refresh token for the form kind and the access + /// token for the bearer kinds. Answers `Ok(false)` when the service has no revoke endpoint. + pub fn revoke(&self, token: &Secret) -> Result { + let request = match self.oauth.revoke { + Revoke::None => return Ok(false), + Revoke::Form { url, with_client } => { + let mut fields = vec![("token".to_owned(), token.expose().to_owned())]; + if with_client { + fields.extend(self.client_fields()); + } + HttpRequest::new(Method::Post, url).form(fields) + } + Revoke::BearerPost { url } => { + HttpRequest::new(Method::Post, url).header("Authorization", format!("Bearer {}", token.expose())) + } + Revoke::BearerDelete { url } => { + HttpRequest::new(Method::Delete, url).header("Authorization", format!("Bearer {}", token.expose())) + } + }; + let response = self.send(&request)?; + // A token the service no longer knows (400 or 401, or Slack's 200 with "ok": false) is revoked enough. + match response.status { + 200..=299 | 400 | 401 => Ok(true), + 500.. => Err(Failure::Network), + _ => Err(Failure::Rejected), + } + } + + /// The client ID, and the secret when the service takes one and the registration has one. + fn client_fields(&self) -> Vec<(String, String)> { + let mut fields = vec![("client_id".to_owned(), self.client.id.clone())]; + if let (Some(secret), false) = (&self.client.secret, self.oauth.secret == SecretUse::Never) { + fields.push(("client_secret".to_owned(), secret.expose().to_owned())); + } + fields + } + + fn token_call(&self, mut fields: Vec<(String, String)>) -> Result { + fields.extend(self.client_fields()); + let request = HttpRequest::new(Method::Post, self.oauth.token_url) + .header("Accept", "application/json") + .form(fields); + self.send(&request) + } + + fn send(&self, request: &HttpRequest) -> Result { + use super::http::HttpError; + self.http + .send(self.policy, request, MAX_ANSWER) + .map_err(|error| match error { + HttpError::ForeignHost => Failure::ForeignHost, + HttpError::Network | HttpError::TooLarge => Failure::Network, + }) + } + + fn parse(&self, response: HttpResponse, refreshing: bool) -> Result { + let json = response.json(); + let error = json.as_ref().and_then(|body| body.get("error")).and_then(Value::as_str); + let refused = !response.ok() + || json + .as_ref() + .is_some_and(|body| body.get("ok") == Some(&Value::Bool(false))); + if refused { + if response.status >= 500 { + return Err(Failure::Network); + } + let gone = matches!(error, Some("invalid_grant" | "invalid_refresh_token" | "invalid_token")); + return Err(if refreshing && gone { + Failure::Expired + } else { + Failure::Rejected + }); + } + let body = json.ok_or(Failure::Rejected)?; + let pointers = self.oauth.response; + let access = text_at(&body, pointers.access).ok_or(Failure::Rejected)?; + Ok(Tokens { + access: Secret::new(access), + refresh: text_at(&body, pointers.refresh).map(Secret::new), + expires_in: body.pointer(pointers.expires_in).and_then(number), + scopes: text_at(&body, pointers.scope).map(|text| { + text.split(|c: char| c.is_whitespace() || c == ',') + .filter(|scope| !scope.is_empty()) + .map(str::to_owned) + .collect() + }), + body, + }) + } +} + +fn text_at(body: &Value, pointer: &str) -> Option { + body.pointer(pointer) + .and_then(Value::as_str) + .filter(|text| !text.is_empty()) + .map(str::to_owned) +} + +/// A number the service may send as a JSON number or as a string. +fn number(value: &Value) -> Option { + value + .as_u64() + .or_else(|| value.as_str().and_then(|text| text.parse().ok())) +} + +#[cfg(test)] +mod tests; diff --git a/app/src-tauri/src/connectors/oauth/tests.rs b/app/src-tauri/src/connectors/oauth/tests.rs new file mode 100644 index 00000000..0ead597c --- /dev/null +++ b/app/src-tauri/src/connectors/oauth/tests.rs @@ -0,0 +1,352 @@ +//! The sign-in requests against a local mock server: the address, the code exchange, the refresh, and the revoke. + +use serde_json::json; + +use super::*; +use crate::connectors::{ + http::UreqHttp, + mock::{ + fixtures::{connector, oauth}, + MockServer, Reply, + }, + registry::Auth, +}; + +const REDIRECT: &str = "http://127.0.0.1:4711/callback"; + +fn client(secret: Option<&str>) -> Client { + Client { + id: "example-client-id".into(), + secret: secret.map(Secret::new), + redirect_port: None, + } +} + +/// Runs `check` with an exchange against the mock, with the given OAuth settings changed by `adjust`. +fn with_exchange( + server: &MockServer, + secret: Option<&str>, + adjust: impl FnOnce(&mut OAuthDef), + check: impl FnOnce(&Exchange) -> R, +) -> R { + let mut settings = oauth(server); + adjust(&mut settings); + let def = connector("example", Auth::OAuth(settings)); + let http = UreqHttp::default(); + let policy = HostPolicy::new(["127.0.0.1"]); + let client = client(secret); + check(&Exchange { + http: &http, + policy: &policy, + def, + oauth: &settings, + client: &client, + }) +} + +fn query_of(url: &str) -> std::collections::HashMap { + Url::parse(url) + .expect("a URL") + .query_pairs() + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect() +} + +#[test] +fn the_sign_in_address_carries_state_pkce_scopes_and_the_loopback_redirect() { + let server = MockServer::start(); + let url = with_exchange( + &server, + Some("never-in-the-address"), + |_| {}, + |exchange| { + exchange + .authorize_url(REDIRECT, "the-state", Some("the-challenge")) + .expect("builds") + }, + ); + assert!(url.starts_with(&server.url("/authorize"))); + let query = query_of(&url); + assert_eq!(query["response_type"], "code"); + assert_eq!(query["client_id"], "example-client-id"); + assert_eq!(query["redirect_uri"], REDIRECT); + assert_eq!(query["state"], "the-state"); + assert_eq!(query["code_challenge"], "the-challenge"); + assert_eq!(query["code_challenge_method"], "S256"); + assert_eq!(query["scope"], "scope-a scope-b"); + assert_eq!(query["access_type"], "offline"); + assert!(!url.contains("never-in-the-address") && !query.contains_key("client_secret")); +} + +#[test] +fn a_service_without_pkce_gets_no_challenge_and_slack_style_scopes_go_in_their_own_parameter() { + let server = MockServer::start(); + let url = with_exchange( + &server, + None, + |settings| { + settings.pkce = false; + settings.scope_param = "user_scope"; + settings.scope_separator = ","; + }, + |exchange| exchange.authorize_url(REDIRECT, "s", Some("c")).expect("builds"), + ); + let query = query_of(&url); + assert!(!query.contains_key("code_challenge") && !query.contains_key("scope")); + assert_eq!(query["user_scope"], "scope-a,scope-b"); +} + +#[test] +fn trades_the_code_for_tokens_with_the_verifier_and_the_client() { + let server = MockServer::start(); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "access_token": "access-1", "refresh_token": "refresh-1", "expires_in": 3599, "scope": "scope-a scope-b" }), + )], + ); + let tokens = with_exchange( + &server, + Some("example-secret"), + |_| {}, + |exchange| { + exchange + .exchange_code(&Secret::new("the-code"), Some("the-verifier"), REDIRECT) + .expect("exchanges") + }, + ); + assert_eq!(tokens.access.expose(), "access-1"); + assert_eq!(tokens.refresh.as_ref().map(Secret::expose), Some("refresh-1")); + assert_eq!(tokens.expires_in, Some(3599)); + assert_eq!(tokens.scopes, Some(vec!["scope-a".to_owned(), "scope-b".to_owned()])); + + let sent = &server.requests_to("/token")[0]; + assert_eq!(sent.method, "POST"); + assert_eq!(sent.header("content-type"), Some("application/x-www-form-urlencoded")); + for (name, value) in [ + ("grant_type", "authorization_code"), + ("code", "the-code"), + ("code_verifier", "the-verifier"), + ("redirect_uri", REDIRECT), + ("client_id", "example-client-id"), + ("client_secret", "example-secret"), + ] { + assert_eq!(sent.field(name).as_deref(), Some(value), "{name}"); + } +} + +#[test] +fn the_client_secret_is_left_out_when_the_service_takes_pkce_alone() { + let server = MockServer::start(); + server.route("/token", vec![Reply::json(200, json!({ "access_token": "a" }))]); + with_exchange( + &server, + Some("configured-anyway"), + |settings| settings.secret = SecretUse::Never, + |exchange| { + exchange + .exchange_code(&Secret::new("c"), Some("v"), REDIRECT) + .expect("exchanges"); + }, + ); + assert_eq!(server.requests_to("/token")[0].field("client_secret"), None); +} + +#[test] +fn reads_nested_answers_and_numbers_sent_as_text() { + let server = MockServer::start(); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "ok": true, "authed_user": { "access_token": "user-token", "scope": "chat:write,files:write", "expires_in": "43200" }, + "team": { "name": "Study group" } }), + )], + ); + let tokens = with_exchange( + &server, + None, + |settings| { + settings.response = crate::connectors::registry::Pointers { + access: "/authed_user/access_token", + refresh: "/authed_user/refresh_token", + expires_in: "/authed_user/expires_in", + scope: "/authed_user/scope", + }; + }, + |exchange| { + exchange + .exchange_code(&Secret::new("c"), None, REDIRECT) + .expect("exchanges") + }, + ); + assert_eq!(tokens.access.expose(), "user-token"); + assert!(tokens.refresh.is_none()); + assert_eq!(tokens.expires_in, Some(43200)); + assert_eq!( + tokens.scopes, + Some(vec!["chat:write".to_owned(), "files:write".to_owned()]) + ); +} + +#[test] +fn a_refused_exchange_is_a_rejection_and_a_server_error_is_a_network_failure() { + let server = MockServer::start(); + let cases = [ + (Reply::json(400, json!({ "error": "invalid_grant" })), Failure::Rejected), + ( + Reply::json(200, json!({ "ok": false, "error": "invalid_code" })), + Failure::Rejected, + ), + (Reply::json(200, json!({ "token_type": "bearer" })), Failure::Rejected), + (Reply::empty(200), Failure::Rejected), + (Reply::empty(503), Failure::Network), + ]; + for (reply, expected) in cases { + server.route("/token", vec![reply]); + let outcome = with_exchange( + &server, + None, + |_| {}, + |exchange| exchange.exchange_code(&Secret::new("c"), None, REDIRECT).map(|_| ()), + ); + assert_eq!(outcome, Err(expected)); + server.clear_routes(); + } +} + +#[test] +fn refreshes_with_the_refresh_token_and_tells_a_dead_one_from_a_busy_server() { + let server = MockServer::start(); + server.route( + "/token", + vec![ + Reply::json( + 200, + json!({ "access_token": "access-2", "refresh_token": "refresh-2", "expires_in": 3600 }), + ), + Reply::json(400, json!({ "error": "invalid_grant" })), + Reply::empty(502), + ], + ); + with_exchange( + &server, + None, + |_| {}, + |exchange| { + let tokens = exchange.refresh(&Secret::new("refresh-1")).expect("refreshes"); + assert_eq!(tokens.access.expose(), "access-2"); + assert_eq!(tokens.refresh.as_ref().map(Secret::expose), Some("refresh-2")); + assert_eq!( + exchange.refresh(&Secret::new("refresh-2")).map(|_| ()), + Err(Failure::Expired) + ); + assert_eq!( + exchange.refresh(&Secret::new("refresh-2")).map(|_| ()), + Err(Failure::Network) + ); + }, + ); + let first = &server.requests_to("/token")[0]; + assert_eq!(first.field("grant_type").as_deref(), Some("refresh_token")); + assert_eq!(first.field("refresh_token").as_deref(), Some("refresh-1")); + assert_eq!(first.field("code"), None); +} + +#[test] +fn never_sends_to_a_token_endpoint_outside_the_pinned_hosts() { + let server = MockServer::start(); + let outcome = with_exchange( + &server, + None, + |settings| settings.token_url = "https://evil.example/token", + |exchange| exchange.exchange_code(&Secret::new("c"), None, REDIRECT).map(|_| ()), + ); + assert_eq!(outcome, Err(Failure::ForeignHost)); + assert!(server.requests().is_empty()); +} + +#[test] +fn a_redirect_off_the_pinned_hosts_is_not_followed() { + let server = MockServer::start(); + server.route("/token", vec![Reply::redirect("https://evil.example/steal")]); + let outcome = with_exchange( + &server, + None, + |_| {}, + |exchange| exchange.exchange_code(&Secret::new("c"), None, REDIRECT).map(|_| ()), + ); + assert_eq!(outcome, Err(Failure::ForeignHost)); +} + +#[test] +fn revokes_in_the_way_each_service_asks() { + let server = MockServer::start(); + server.route("/revoke", vec![Reply::empty(200)]); + let token = Secret::new("the-token"); + let form = with_exchange(&server, None, |_| {}, |exchange| exchange.revoke(&token)); + assert_eq!(form, Ok(true)); + assert_eq!( + server.requests_to("/revoke")[0].field("token").as_deref(), + Some("the-token") + ); + + let with_client = with_exchange( + &server, + Some("example-secret"), + |settings| { + settings.revoke = Revoke::Form { + url: crate::connectors::mock::fixtures::leak_str(server.url("/revoke")), + with_client: true, + } + }, + |exchange| exchange.revoke(&token), + ); + assert_eq!(with_client, Ok(true)); + let sent = &server.requests_to("/revoke")[1]; + assert_eq!(sent.field("client_id").as_deref(), Some("example-client-id")); + assert_eq!(sent.field("client_secret").as_deref(), Some("example-secret")); + + let bearer = with_exchange( + &server, + None, + |settings| { + settings.revoke = Revoke::BearerDelete { + url: crate::connectors::mock::fixtures::leak_str(server.url("/revoke")), + } + }, + |exchange| exchange.revoke(&token), + ); + assert_eq!(bearer, Ok(true)); + let sent = &server.requests_to("/revoke")[2]; + assert_eq!( + (sent.method.as_str(), sent.header("authorization")), + ("DELETE", Some("Bearer the-token")) + ); + + let none = with_exchange( + &server, + None, + |settings| settings.revoke = Revoke::None, + |exchange| exchange.revoke(&token), + ); + assert_eq!(none, Ok(false)); + assert_eq!(server.requests_to("/revoke").len(), 3); +} + +#[test] +fn a_revoke_the_service_cannot_do_right_now_is_reported() { + let server = MockServer::start(); + server.route("/revoke", vec![Reply::empty(503), Reply::empty(403)]); + let token = Secret::new("t"); + with_exchange( + &server, + None, + |_| {}, + |exchange| { + assert_eq!(exchange.revoke(&token), Err(Failure::Network)); + assert_eq!(exchange.revoke(&token), Err(Failure::Rejected)); + }, + ); +} diff --git a/app/src-tauri/src/connectors/pkce.rs b/app/src-tauri/src/connectors/pkce.rs new file mode 100644 index 00000000..46c0ca38 --- /dev/null +++ b/app/src-tauri/src/connectors/pkce.rs @@ -0,0 +1,79 @@ +//! The random values of a sign-in: the PKCE verifier and its S256 challenge (RFC 7636), and the state value that +//! the browser must bring back (RFC 6749 section 10.12). All of them come from the operating system's generator. + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use sha2::{Digest, Sha256}; + +/// 32 random bytes as 43 URL-safe characters, the shortest verifier RFC 7636 allows with full strength. +pub fn new_verifier() -> Option { + random_text::<32>() +} + +/// 24 random bytes as 32 URL-safe characters. +pub fn new_state() -> Option { + random_text::<24>() +} + +fn random_text() -> Option { + let mut bytes = [0u8; N]; + getrandom::fill(&mut bytes).ok()?; + Some(URL_SAFE_NO_PAD.encode(bytes)) +} + +/// The S256 challenge of a verifier: the unpadded URL-safe base64 of its SHA-256. +pub fn challenge(verifier: &str) -> String { + URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes())) +} + +/// Compares two values without stopping at the first difference, so the time taken tells nothing about how much +/// of a guessed state value was right. +pub fn same(left: &str, right: &str) -> bool { + let (left, right) = (left.as_bytes(), right.as_bytes()); + let mut difference = left.len() ^ right.len(); + for (index, byte) in left.iter().enumerate() { + difference |= usize::from(byte ^ right.get(index).copied().unwrap_or(0)); + } + difference == 0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_challenge_matches_the_example_in_rfc_7636() { + assert_eq!( + challenge("dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk"), + "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM" + ); + } + + #[test] + fn verifiers_follow_the_rfc_and_never_repeat() { + let first = new_verifier().expect("the generator works"); + let second = new_verifier().expect("the generator works"); + assert_ne!(first, second); + // RFC 7636 section 4.1: 43 to 128 characters from the unreserved set. + assert_eq!(first.len(), 43); + assert!(first.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')); + } + + #[test] + fn state_values_are_long_and_fresh() { + let first = new_state().expect("the generator works"); + let second = new_state().expect("the generator works"); + assert_ne!(first, second); + assert_eq!(first.len(), 32); + assert!(first.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')); + } + + #[test] + fn comparison_sees_every_kind_of_difference() { + assert!(same("abc", "abc")); + assert!(!same("abc", "abd")); + assert!(!same("abc", "ab")); + assert!(!same("ab", "abc")); + assert!(same("", "")); + assert!(!same("", "a")); + } +} diff --git a/app/src-tauri/src/connectors/registry.rs b/app/src-tauri/src/connectors/registry.rs new file mode 100644 index 00000000..bd5873a5 --- /dev/null +++ b/app/src-tauri/src/connectors/registry.rs @@ -0,0 +1,229 @@ +//! The connector registry: every service OpenNote can sign in to, as data. One entry says what the service is +//! called, where its sign-in, token, and revoke endpoints are, which hosts OpenNote may talk to for it, the least +//! access each feature needs, and which OpenNote features use it. Nothing else in OpenNote names a service, and no +//! entry holds a client ID, a client secret, or a token (see `files.rs` for where a client ID comes from). + +use serde::Serialize; + +pub use super::catalog::CONNECTORS; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Group { + Microsoft, + Google, + Storage, + Learning, + Other, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Method { + Get, + Post, + Put, + Patch, + Delete, + /// WebDAV: list a folder (`Depth` says how deep). + Propfind, + /// WebDAV: make a folder. + Mkcol, +} + +impl Method { + pub const fn as_str(self) -> &'static str { + match self { + Self::Get => "GET", + Self::Post => "POST", + Self::Put => "PUT", + Self::Patch => "PATCH", + Self::Delete => "DELETE", + Self::Propfind => "PROPFIND", + Self::Mkcol => "MKCOL", + } + } + + pub fn parse(text: &str) -> Option { + [ + Self::Get, + Self::Post, + Self::Put, + Self::Patch, + Self::Delete, + Self::Propfind, + Self::Mkcol, + ] + .into_iter() + .find(|method| method.as_str().eq_ignore_ascii_case(text)) + } +} + +/// One permission OpenNote asks for. `capability` names a sentence in the interface's strings that says in plain +/// words what it lets OpenNote read or change, and `writes` is true when it lets OpenNote change something. +#[derive(Debug, Clone, Copy)] +pub struct Access { + pub scope: &'static str, + pub capability: &'static str, + pub writes: bool, +} + +/// How Disconnect tells the service to forget the sign-in. +#[derive(Debug, Clone, Copy)] +pub enum Revoke { + /// The service has no revoke endpoint. The sign-in is removed from this computer only. + None, + /// A form post with the token, and with the client ID and secret when `with_client` is set. + Form { url: &'static str, with_client: bool }, + /// A post with the access token as a bearer token, and no body. + BearerPost { url: &'static str }, + /// A delete with the access token as a bearer token. + BearerDelete { url: &'static str }, +} + +/// Whether the token endpoint wants a client secret. Desktop apps cannot keep one private, so a service that +/// takes PKCE alone is `Never`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SecretUse { + Never, + Optional, + Required, +} + +/// Where the sign-in comes back to. A service that lets the loopback port vary takes `AnyPort`, as RFC 8252 +/// asks. One that matches the address exactly takes a fixed port, which the owner registers (docs/CONNECTORS.md). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Redirect { + AnyPort, + FixedPort(u16), +} + +/// Where the account name for "Connected as" comes from. +#[derive(Debug, Clone, Copy)] +pub enum AccountFrom { + /// The `preferred_username`, `email`, or `name` claim of the ID token the token endpoint sent. + IdToken, + /// A JSON pointer into the token response. + TokenResponse(&'static str), + /// A call made with the new access token, then a JSON pointer into its answer. + Call { + method: Method, + url: &'static str, + pointer: &'static str, + }, +} + +/// JSON pointers to the parts of a token response. +#[derive(Debug, Clone, Copy)] +pub struct Pointers { + pub access: &'static str, + pub refresh: &'static str, + pub expires_in: &'static str, + pub scope: &'static str, +} + +pub const STANDARD_RESPONSE: Pointers = Pointers { + access: "/access_token", + refresh: "/refresh_token", + expires_in: "/expires_in", + scope: "/scope", +}; + +/// OAuth 2.0 authorization code, with PKCE where the service takes it (RFC 6749, RFC 7636, RFC 8252). +#[derive(Debug, Clone, Copy)] +pub struct OAuthDef { + pub authorize_url: &'static str, + pub token_url: &'static str, + pub revoke: Revoke, + /// The query parameter that carries the scopes, or empty when the service sets them on the app instead. + pub scope_param: &'static str, + pub scope_separator: &'static str, + pub extra_params: &'static [(&'static str, &'static str)], + pub pkce: bool, + pub secret: SecretUse, + pub redirect: Redirect, + pub account: AccountFrom, + pub response: Pointers, +} + +/// Where a pasted token goes on a request. +#[derive(Debug, Clone, Copy)] +pub enum Placement { + /// `Authorization: `. + Header(&'static str), + /// A field of the form body. + FormField(&'static str), + /// `Authorization: Basic `, where the token is `user:password` (WebDAV). + Basic, +} + +/// A personal token the person pastes. With `needs_base_url`, the person also gives the school's address, and +/// that address's host is the only host OpenNote talks to for it. +#[derive(Debug, Clone, Copy)] +pub struct TokenDef { + pub needs_base_url: bool, + pub placement: Placement, + pub check_method: Method, + /// A full address, or with `needs_base_url` a path that follows the school's address. + pub check_url: &'static str, + pub check_form: &'static [(&'static str, &'static str)], + /// Where the account name is in the answer, or empty when the service doesn't say. + pub account_pointer: &'static str, +} + +#[derive(Debug, Clone, Copy)] +pub enum Auth { + OAuth(OAuthDef), + Token(TokenDef), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum AuthKind { + Oauth, + Token, + TokenAndUrl, +} + +#[derive(Debug, Clone, Copy)] +pub struct ConnectorDef { + pub id: &'static str, + pub name: &'static str, + pub group: Group, + /// The OpenNote features that use this connector, by feature ID. The interface names them in plain words. + pub features: &'static [&'static str], + /// The only hosts OpenNote talks to for this connector. A school address adds its own host (`TokenDef`). + pub hosts: &'static [&'static str], + pub access: &'static [Access], + pub auth: Auth, +} + +impl ConnectorDef { + pub const fn kind(&self) -> AuthKind { + match self.auth { + Auth::OAuth(_) => AuthKind::Oauth, + Auth::Token(TokenDef { + needs_base_url: true, .. + }) => AuthKind::TokenAndUrl, + Auth::Token(_) => AuthKind::Token, + } + } + + /// The scopes to ask for: every non-empty scope of `access`, once. + pub fn scopes(&self) -> Vec<&'static str> { + let mut scopes: Vec<&'static str> = Vec::new(); + for access in self.access { + if !access.scope.is_empty() && !scopes.contains(&access.scope) { + scopes.push(access.scope); + } + } + scopes + } +} + +/// Looks a connector up by ID. +pub fn find(id: &str) -> Option<&'static ConnectorDef> { + CONNECTORS.iter().find(|def| def.id == id) +} + +#[cfg(test)] +mod tests; diff --git a/app/src-tauri/src/connectors/registry/tests.rs b/app/src-tauri/src/connectors/registry/tests.rs new file mode 100644 index 00000000..418931c5 --- /dev/null +++ b/app/src-tauri/src/connectors/registry/tests.rs @@ -0,0 +1,140 @@ +//! Checks on the catalog: complete entries, pinned hosts, and a sentence in the strings for every name. + +use std::collections::BTreeSet; + +use super::*; + +fn host_of(url: &str) -> &str { + url.strip_prefix("https://") + .expect("an https address") + .split(['/', '?']) + .next() + .unwrap_or_default() +} + +#[test] +fn ids_are_unique_and_every_entry_is_complete() { + let ids: BTreeSet<_> = CONNECTORS.iter().map(|def| def.id).collect(); + assert_eq!(ids.len(), CONNECTORS.len()); + for def in CONNECTORS { + assert!( + !def.name.is_empty() && !def.features.is_empty() && !def.access.is_empty(), + "{}", + def.id + ); + assert!(def.id.chars().all(|c| c.is_ascii_lowercase()), "{}", def.id); + } +} + +#[test] +fn every_oauth_endpoint_is_https_on_a_pinned_host() { + for def in CONNECTORS { + let Auth::OAuth(oauth) = def.auth else { continue }; + let mut urls = vec![oauth.authorize_url, oauth.token_url]; + match oauth.revoke { + Revoke::Form { url, .. } | Revoke::BearerPost { url } | Revoke::BearerDelete { url } => urls.push(url), + Revoke::None => {} + } + if let AccountFrom::Call { url, .. } = oauth.account { + urls.push(url); + } + for url in urls { + assert!( + def.hosts.contains(&host_of(url)), + "{}: {url} is not on a pinned host", + def.id + ); + } + } +} + +#[test] +fn hosts_are_plain_names() { + for def in CONNECTORS { + for host in def.hosts { + assert!(host + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '.' || c == '-')); + assert!(host.contains('.'), "{host}"); + } + } +} + +#[test] +fn school_connectors_have_no_fixed_host_and_the_others_have_some() { + for def in CONNECTORS { + let school = def.kind() == AuthKind::TokenAndUrl; + assert_eq!(def.hosts.is_empty(), school, "{}", def.id); + } +} + +#[test] +fn fixed_ports_are_distinct_and_unprivileged() { + let ports: Vec = CONNECTORS + .iter() + .filter_map(|def| match def.auth { + Auth::OAuth(OAuthDef { + redirect: Redirect::FixedPort(port), + .. + }) => Some(port), + _ => None, + }) + .collect(); + assert_eq!(ports.iter().collect::>().len(), ports.len()); + assert!(ports.iter().all(|port| *port >= 1024)); +} + +#[test] +fn the_big_two_let_the_port_vary_as_rfc_8252_asks() { + for id in ["microsoft", "google"] { + let Some(ConnectorDef { + auth: Auth::OAuth(oauth), + .. + }) = find(id) + else { + panic!("{id} is OAuth"); + }; + assert_eq!(oauth.redirect, Redirect::AnyPort); + assert!(oauth.pkce); + } +} + +#[test] +fn the_listed_services_are_all_there() { + let ids: Vec<_> = CONNECTORS.iter().map(|def| def.id).collect(); + for id in [ + "microsoft", + "google", + "slack", + "dropbox", + "box", + "vimeo", + "readwise", + "canvas", + "moodle", + "webdav", + ] { + assert!(ids.contains(&id), "{id}"); + } +} + +#[test] +fn every_capability_has_a_sentence_in_the_interface_strings() { + let strings = include_str!("../../../../src/strings/en/connectors.ts"); + for def in CONNECTORS { + for access in def.access { + assert!( + strings.contains(&format!("{}:", access.capability)), + "strings/en/connectors.ts has no sentence for {}", + access.capability + ); + } + assert!(strings.contains(&format!("{}:", def.id)), "no strings for {}", def.id); + for feature in def.features { + assert!( + strings.contains(&format!("{feature}:")), + "no feature name for {feature}" + ); + } + } +} diff --git a/app/src-tauri/src/connectors/secret.rs b/app/src-tauri/src/connectors/secret.rs new file mode 100644 index 00000000..82fe929b --- /dev/null +++ b/app/src-tauri/src/connectors/secret.rs @@ -0,0 +1,42 @@ +//! A value that must never reach a log, a file, a crash report, or the interface: an access token, a refresh token, +//! a pasted personal token, or a client secret. It has no `Display`, no `Serialize`, and a `Debug` that hides it, so +//! the only way to read it is to call [`Secret::expose`] on purpose, at the one place that sends it to its service. + +use std::fmt; + +#[derive(Clone, PartialEq, Eq)] +pub struct Secret(String); + +impl Secret { + pub fn new(value: impl Into) -> Secret { + Secret(value.into()) + } + + /// The text, for the request that needs it and for the credential store. Nothing else should call this. + pub fn expose(&self) -> &str { + &self.0 + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Debug for Secret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Secret()") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn debug_output_never_holds_the_value() { + let secret = Secret::new("not-a-real-token"); + assert_eq!(format!("{secret:?}"), "Secret()"); + assert_eq!(format!("{:?}", Some(secret.clone())), "Some(Secret())"); + assert_eq!(secret.expose(), "not-a-real-token"); + } +} diff --git a/app/src-tauri/src/connectors/service.rs b/app/src-tauri/src/connectors/service.rs new file mode 100644 index 00000000..5307a7b7 --- /dev/null +++ b/app/src-tauri/src/connectors/service.rs @@ -0,0 +1,221 @@ +//! The connectors service: the registry, the connections file, the credential store, and the sign-ins in progress. +//! The pieces that touch the outside (the network, the browser, the clock, Work offline) are passed in, so the tests +//! run the whole sign-in against a mock server and a fake browser. +//! +//! `connect.rs` signs in, and `session.rs` renews, uses, and ends a connection. Features of OpenNote call +//! [`Connectors::is_connected`], [`Connectors::access_token`], and [`Connectors::request`]. + +use std::{ + collections::{BTreeMap, HashMap}, + path::PathBuf, + sync::{atomic::AtomicBool, Arc, Mutex, MutexGuard, PoisonError}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use super::{ + error::{ConnectorError, Failure, Result}, + files::{self, Connection}, + http::{HostPolicy, Http, UreqHttp}, + loopback, + registry::{self, Auth, ConnectorDef}, + secret::Secret, + store::{platform_store, SecretStore}, + view::{view_of, ConnectorView}, +}; +use crate::paths::Paths; + +/// Opens the sign-in address in the person's default browser. Never an embedded web view (RFC 8252 section 8.12). +pub trait Opener: Send + Sync { + fn open(&self, url: &str) -> bool; +} + +struct DefaultBrowser; + +impl Opener for DefaultBrowser { + fn open(&self, url: &str) -> bool { + crate::shell::open(url).is_ok() + } +} + +/// An access token kept in memory until it nears its end. It is never written anywhere. +pub(super) struct Cached { + pub token: Secret, + pub expires_unix: Option, +} + +#[derive(Default)] +pub(super) struct State { + pub connections: BTreeMap, + pub access: HashMap, + /// The cancel flag of each sign-in that is waiting for the browser. + pub pending: HashMap>, +} + +pub(super) struct Inner { + pub config_file: PathBuf, + pub connections_file: PathBuf, + pub registry: &'static [ConnectorDef], + pub store: Arc, + pub http: Arc, + pub opener: Arc, + pub offline: Arc bool + Send + Sync>, + pub now: Arc u64 + Send + Sync>, + pub wait: Duration, + pub state: Mutex, + /// One refresh at a time, so two features asking together don't both spend a rotating refresh token. + pub refreshing: Mutex<()>, +} + +/// Everything a [`Connectors`] needs from outside. `Parts::real` is what the app uses. +pub struct Parts { + pub config_file: PathBuf, + pub connections_file: PathBuf, + pub registry: &'static [ConnectorDef], + pub store: Arc, + pub http: Arc, + pub opener: Arc, + pub offline: Arc bool + Send + Sync>, + pub now: Arc u64 + Send + Sync>, + pub wait: Duration, +} + +impl Parts { + pub fn real(paths: &Paths) -> Parts { + Parts { + config_file: paths.local.join(files::CONFIG_FILE), + connections_file: paths.local.join(files::CONNECTIONS_FILE), + registry: registry::CONNECTORS, + store: platform_store(), + http: Arc::new(UreqHttp::default()), + opener: Arc::new(DefaultBrowser), + offline: Arc::new(crate::hardening::offline), + now: Arc::new(unix_now), + wait: loopback::WAIT, + } + } +} + +pub fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +/// The app's connectors. Cheap to clone: the copies share one state. +#[derive(Clone)] +pub struct Connectors(pub(super) Arc); + +impl Connectors { + pub fn new(paths: &Paths) -> Connectors { + #[cfg(feature = "test-endpoints")] + if let Some(connectors) = super::test_hook::connectors(paths) { + return connectors; + } + Connectors::with(Parts::real(paths)) + } + + pub fn with(parts: Parts) -> Connectors { + let connections = files::read_connections(&parts.connections_file); + Connectors(Arc::new(Inner { + config_file: parts.config_file, + connections_file: parts.connections_file, + registry: parts.registry, + store: parts.store, + http: parts.http, + opener: parts.opener, + offline: parts.offline, + now: parts.now, + wait: parts.wait, + state: Mutex::new(State { + connections, + ..State::default() + }), + refreshing: Mutex::default(), + })) + } + + pub(super) fn lock(&self) -> MutexGuard<'_, State> { + self.0.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub(super) fn now(&self) -> u64 { + (self.0.now)() + } + + pub(super) fn offline(&self) -> bool { + (self.0.offline)() + } + + pub(super) fn def(&self, id: &str) -> Result<&'static ConnectorDef> { + self.0 + .registry + .iter() + .find(|def| def.id == id) + .ok_or_else(|| ConnectorError::new(Failure::Unknown, id)) + } + + /// Writes the connections file from the state that is passed in. + pub(super) fn save(&self, state: &State, id: &str) -> Result<()> { + files::write_connections(&self.0.connections_file, &state.connections).map_err(|error| { + ::log::warn!("Couldn't save the connections file: {error}"); + ConnectorError::new(Failure::Storage, id) + }) + } + + /// The hosts the connector may use: its own, and the host of the school address it was connected with. + pub(super) fn policy(&self, def: &ConnectorDef, base_url: Option<&str>) -> HostPolicy { + let school = base_url + .and_then(|base| url::Url::parse(base).ok()) + .and_then(|base| base.host_str().map(str::to_owned)); + HostPolicy::new(def.hosts.iter().map(|host| (*host).to_owned()).chain(school)) + } + + pub fn list(&self) -> Vec { + let state = self.lock(); + self.0 + .registry + .iter() + .map(|def| { + let connection = state.connections.get(def.id); + let pending = state.pending.contains_key(def.id); + view_of(def, connection, self.needs_setup(def), pending) + }) + .collect() + } + + pub fn view(&self, id: &str) -> Result { + let def = self.def(id)?; + let state = self.lock(); + Ok(view_of( + def, + state.connections.get(id), + self.needs_setup(def), + state.pending.contains_key(id), + )) + } + + /// An OAuth connector with no usable app registration. + pub(super) fn needs_setup(&self, def: &ConnectorDef) -> bool { + let Auth::OAuth(oauth) = def.auth else { return false }; + match files::load_client(&self.0.config_file, def.id) { + None => true, + Some(client) => oauth.secret == registry::SecretUse::Required && client.secret.is_none(), + } + } + + /// Whether the person has connected it and the sign-in still works. A feature checks this before it offers + /// anything that needs the account. + pub fn is_connected(&self, id: &str) -> bool { + self.lock() + .connections + .get(id) + .is_some_and(|connection| !connection.expired) + } + + /// Stops a sign-in that is waiting for the browser. + pub fn cancel(&self, id: &str) { + if let Some(flag) = self.lock().pending.get(id) { + flag.store(true, std::sync::atomic::Ordering::Relaxed); + } + } +} diff --git a/app/src-tauri/src/connectors/session.rs b/app/src-tauri/src/connectors/session.rs new file mode 100644 index 00000000..072c9189 --- /dev/null +++ b/app/src-tauri/src/connectors/session.rs @@ -0,0 +1,377 @@ +//! Using a connection: a fresh access token for the access a feature needs, an authorized request to an allowed +//! host, and Disconnect. These are the calls the features of OpenNote make later. The token a feature gets +//! stays in Rust, and the interface can only ask for [`Connectors::request`], which adds the token itself. + +use super::{ + error::{ConnectorError, Failure, Result}, + files::{Connection, Holds}, + http::{Body, HttpError, HttpRequest, HttpResponse}, + oauth::Exchange, + registry::{Auth, ConnectorDef, Placement, Revoke, TokenDef}, + secret::Secret, + service::{Cached, Connectors}, + store::target_name, + view::{Disconnected, RevokeOutcome}, +}; + +/// An access token is renewed this many seconds before it ends. +const RENEW_BEFORE: u64 = 120; +/// With no stated lifetime, an access token that has a refresh token behind it is renewed after this many seconds. +pub(super) const DEFAULT_LIFETIME: u64 = 3300; +/// The largest answer `request` reads, unless the caller asks for more. +pub const DEFAULT_MAX_BYTES: usize = 8 * 1024 * 1024; + +/// Headers the caller can never set: the token is the connector's to add, and the rest name the server. +const FORBIDDEN_HEADERS: [&str; 5] = [ + "authorization", + "proxy-authorization", + "cookie", + "host", + "content-length", +]; + +/// Puts the token where the service wants it. +pub(super) fn with_token(mut request: HttpRequest, placement: Placement, token: &Secret) -> Option { + match placement { + Placement::Header(scheme) => Some(request.header("Authorization", format!("{scheme} {}", token.expose()))), + Placement::Basic => { + use base64::{engine::general_purpose::STANDARD, Engine as _}; + let encoded = STANDARD.encode(token.expose()); + Some(request.header("Authorization", format!("Basic {encoded}"))) + } + Placement::FormField(name) => { + let mut fields = match request.body.take() { + None => Vec::new(), + Some(Body::Form(fields)) => fields, + Some(Body::Bytes { .. }) => return None, + }; + fields.push((name.to_owned(), token.expose().to_owned())); + request.body = Some(Body::Form(fields)); + Some(request) + } + } +} + +fn placement_of(def: &ConnectorDef) -> Placement { + match def.auth { + Auth::OAuth(_) => Placement::Header("Bearer"), + Auth::Token(token) => token.placement, + } +} + +/// Whether the connection was granted `wanted`. Services spell a granted scope in different ways, such as +/// Microsoft's `https://graph.microsoft.com/Notes.Read` for `Notes.Read`, and in any case. +/// +/// The sign-in scopes (`openid`, `email`, `profile`, `offline_access`) count as granted on any connection: the +/// sign-in itself proves them, and services leave them out of the token answer (Microsoft) or spell them as +/// URLs such as `https://www.googleapis.com/auth/userinfo.email` (Google). +fn granted(connection: &Connection, wanted: &str) -> bool { + let short = |scope: &str| scope.rsplit('/').next().unwrap_or(scope).to_lowercase(); + if matches!( + short(wanted).as_str(), + "openid" | "email" | "profile" | "offline_access" + ) { + return true; + } + connection + .scopes + .iter() + .any(|have| have == wanted || short(have) == short(wanted)) +} + +/// The scopes a feature means: a capability name (such as `calendarRead`) stands for the scopes the registry lists +/// for it, and anything else is a scope as the service spells it. +fn expand(def: &ConnectorDef, wanted: &[&str]) -> Vec { + let mut scopes: Vec = Vec::new(); + for want in wanted { + let matching: Vec<&str> = def + .access + .iter() + .filter(|a| a.capability == *want && !a.scope.is_empty()) + .map(|a| a.scope) + .collect(); + if matching.is_empty() { + scopes.push((*want).to_owned()); + } else { + scopes.extend(matching.into_iter().map(str::to_owned)); + } + } + scopes +} + +impl Connectors { + /// A fresh access token for the given access. It is renewed first when it is about to end. This fails with + /// `Offline` while Work offline is on, `NotConnected` or `Expired` when the person needs to connect, and + /// `MissingAccess` when the connection was made without what is asked for. + pub fn access_token(&self, id: &str, wanted: &[&str]) -> Result { + let def = self.def(id)?; + if self.offline() { + return Err(ConnectorError::new(Failure::Offline, id)); + } + self.token_for(def, wanted) + } + + fn token_for(&self, def: &'static ConnectorDef, wanted: &[&str]) -> Result { + let fail = |failure| ConnectorError::new(failure, def.id); + let connection = self + .lock() + .connections + .get(def.id) + .cloned() + .ok_or_else(|| fail(Failure::NotConnected))?; + if connection.expired { + return Err(fail(Failure::Expired)); + } + if expand(def, wanted).iter().any(|scope| !granted(&connection, scope)) { + return Err(fail(Failure::MissingAccess)); + } + let token = match connection.holds { + Holds::Access => self.stored(def, &connection)?, + Holds::Refresh => self.renewed(def, &connection)?, + }; + if let Some(entry) = self.lock().connections.get_mut(def.id) { + entry.last_used_unix = Some(self.now()); + } + Ok(token) + } + + fn stored(&self, def: &ConnectorDef, connection: &Connection) -> Result { + match self.0.store.get(&target_name(def.id, &connection.account)) { + Ok(Some(secret)) => Ok(secret), + // The credential is gone, so the person has to connect again. + Ok(None) => Err(self.expire(def, Failure::Expired)), + Err(_) => Err(ConnectorError::new(Failure::Storage, def.id)), + } + } + + /// The cached access token while it is good, or a new one from the refresh token. + fn renewed(&self, def: &'static ConnectorDef, connection: &Connection) -> Result { + let fresh = |cached: &Cached| cached.expires_unix.is_none_or(|end| end > self.now() + RENEW_BEFORE); + if let Some(cached) = self.lock().access.get(def.id).filter(|cached| fresh(cached)) { + return Ok(cached.token.clone()); + } + let _one_at_a_time = self + .0 + .refreshing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // Another call may have renewed it while this one waited. + if let Some(cached) = self.lock().access.get(def.id).filter(|cached| fresh(cached)) { + return Ok(cached.token.clone()); + } + self.refresh_now(def, connection) + } + + /// Trades the stored refresh token for a new access token and keeps what comes back. + fn refresh_now(&self, def: &'static ConnectorDef, connection: &Connection) -> Result { + let fail = |failure| ConnectorError::new(failure, def.id); + let Auth::OAuth(oauth) = def.auth else { + return Err(fail(Failure::Unknown)); + }; + let refresh = self.stored(def, connection)?; + let client = + super::files::load_client(&self.0.config_file, def.id).ok_or_else(|| fail(Failure::NotConfigured))?; + let policy = self.policy(def, None); + let exchange = Exchange { + http: &*self.0.http, + policy: &policy, + def, + oauth: &oauth, + client: &client, + }; + let tokens = match exchange.refresh(&refresh) { + Ok(tokens) => tokens, + Err(Failure::Expired) => return Err(self.expire(def, Failure::Expired)), + Err(failure) => return Err(fail(failure)), + }; + // A service that rotates refresh tokens sends a new one, and the old one stops working. + if let Some(rotated) = &tokens.refresh { + self.0 + .store + .put(&target_name(def.id, &connection.account), rotated) + .map_err(|_| fail(Failure::Storage))?; + } + let expires_unix = Some(self.now() + tokens.expires_in.unwrap_or(DEFAULT_LIFETIME)); + let token = tokens.access.clone(); + self.lock().access.insert( + def.id.to_owned(), + Cached { + token: tokens.access, + expires_unix, + }, + ); + Ok(token) + } + + /// Marks the connection as needing a new sign-in, and forgets its access token. + fn expire(&self, def: &ConnectorDef, failure: Failure) -> ConnectorError { + let mut state = self.lock(); + state.access.remove(def.id); + if let Some(connection) = state.connections.get_mut(def.id) { + connection.expired = true; + } + // Failing to save the mark is not worth hiding the real reason for. + let _ = self.save(&state, def.id); + ConnectorError::new(failure, def.id) + } + + /// Runs a request with the connection's token added. The address must be HTTPS on one of the connector's + /// hosts, and the caller can't set the `Authorization` header. A 401 renews the token and tries once more. + pub fn request(&self, id: &str, wanted: &[&str], request: HttpRequest, max_bytes: usize) -> Result { + let def = self.def(id)?; + if self.offline() { + return Err(ConnectorError::new(Failure::Offline, id)); + } + let fail = |failure| ConnectorError::new(failure, id); + if request.headers.iter().any(|(name, value)| !header_ok(name, value)) { + return Err(fail(Failure::BadInput)); + } + let base = self + .lock() + .connections + .get(id) + .and_then(|connection| connection.base_url.clone()); + let policy = self.policy(def, base.as_deref()); + if !policy.allows(&request.url) { + return Err(fail(Failure::ForeignHost)); + } + let mut retried = false; + loop { + let token = self.token_for(def, wanted)?; + let signed = with_token(clone_request(&request), placement_of(def), &token) + .ok_or_else(|| fail(Failure::BadInput))?; + let response = self.0.http.send(&policy, &signed, max_bytes).map_err(|error| { + fail(match error { + HttpError::ForeignHost => Failure::ForeignHost, + HttpError::Network | HttpError::TooLarge => Failure::Network, + }) + })?; + if response.status != 401 { + return Ok(response); + } + if let Auth::Token(token_def) = &def.auth { + return Err(self.token_refused(def, token_def, base.as_deref(), &token)); + } + if retried || self.holds_access_only(id) { + return Err(self.expire(def, Failure::Expired)); + } + retried = true; + self.lock().access.remove(id); + } + } + + /// A pasted token got a 401. Services such as Canvas answer 401 for a valid token that lacks one endpoint's + /// scope, so the token is checked again: only when that check is refused too is the connection expired. + fn token_refused( + &self, + def: &'static ConnectorDef, + token_def: &TokenDef, + base: Option<&str>, + token: &Secret, + ) -> ConnectorError { + let url = match base { + Some(base) => format!("{base}{}", token_def.check_url), + None => token_def.check_url.to_owned(), + }; + let policy = self.policy(def, base); + match self.check_token(def, token_def, &policy, &url, token) { + Ok(_) => ConnectorError::new(Failure::Rejected, def.id), + Err(error) if error.failure == Failure::Rejected => self.expire(def, Failure::Expired), + Err(error) => error, + } + } + + fn holds_access_only(&self, id: &str) -> bool { + self.lock() + .connections + .get(id) + .is_some_and(|connection| connection.holds == Holds::Access) + } + + /// Removes the connection from this computer, and tells the service to forget it where the service allows. + /// The person is disconnected here even when the service can't be reached. + pub fn disconnect(&self, id: &str) -> Result { + let def = self.def(id)?; + let Some(connection) = self.lock().connections.get(id).cloned() else { + return Ok(Disconnected { + view: self.view(id)?, + revoke: RevokeOutcome::Nothing, + }); + }; + let revoke = self.revoke(def, &connection); + self.0 + .store + .delete(&target_name(id, &connection.account)) + .map_err(|_| ConnectorError::new(Failure::Storage, id))?; + { + let mut state = self.lock(); + state.connections.remove(id); + state.access.remove(id); + self.save(&state, id)?; + } + Ok(Disconnected { + view: self.view(id)?, + revoke, + }) + } + + fn revoke(&self, def: &'static ConnectorDef, connection: &Connection) -> RevokeOutcome { + let Auth::OAuth(oauth) = def.auth else { + return RevokeOutcome::NotSupported; + }; + if matches!(oauth.revoke, Revoke::None) { + return RevokeOutcome::NotSupported; + } + if self.offline() { + return RevokeOutcome::SkippedOffline; + } + let Some(client) = super::files::load_client(&self.0.config_file, def.id) else { + return RevokeOutcome::Failed; + }; + // The bearer kinds need a working access token, and the form kind takes the refresh token itself. + let token = match (oauth.revoke, connection.holds) { + (Revoke::Form { .. }, _) => self.stored(def, connection), + _ if connection.expired => return RevokeOutcome::Failed, + _ => self.token_for(def, &[]), + }; + let Ok(token) = token else { return RevokeOutcome::Failed }; + let policy = self.policy(def, None); + let exchange = Exchange { + http: &*self.0.http, + policy: &policy, + def, + oauth: &oauth, + client: &client, + }; + match exchange.revoke(&token) { + Ok(true) => RevokeOutcome::Revoked, + Ok(false) => RevokeOutcome::NotSupported, + Err(_) => RevokeOutcome::Failed, + } + } +} + +fn header_ok(name: &str, value: &str) -> bool { + let name_ok = !name.is_empty() && name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'); + let value_ok = value.bytes().all(|b| b == b'\t' || (0x20..0x7f).contains(&b)); + name_ok && value_ok && !FORBIDDEN_HEADERS.contains(&name.to_ascii_lowercase().as_str()) +} + +fn clone_request(request: &HttpRequest) -> HttpRequest { + HttpRequest { + follow_redirects: true, + method: request.method, + url: request.url.clone(), + headers: request.headers.clone(), + body: request.body.as_ref().map(|body| match body { + Body::Form(fields) => Body::Form(fields.clone()), + Body::Bytes { content_type, data } => Body::Bytes { + content_type: content_type.clone(), + data: data.clone(), + }, + }), + } +} + +#[cfg(test)] +mod tests; diff --git a/app/src-tauri/src/connectors/session/tests.rs b/app/src-tauri/src/connectors/session/tests.rs new file mode 100644 index 00000000..2a4902e7 --- /dev/null +++ b/app/src-tauri/src/connectors/session/tests.rs @@ -0,0 +1,98 @@ +//! Checks on scopes, headers, and where a token goes on a request. + +use super::*; +use crate::connectors::registry; + +fn connection(scopes: &[&str]) -> Connection { + Connection { + account: "a".into(), + scopes: scopes.iter().map(|s| (*s).to_owned()).collect(), + connected_unix: 0, + holds: Holds::Refresh, + base_url: None, + expired: false, + last_used_unix: None, + } +} + +#[test] +fn granted_scopes_match_in_the_spellings_services_use() { + let have = connection(&["https://graph.microsoft.com/Notes.Read", "openid"]); + assert!(granted(&have, "Notes.Read")); + assert!(granted(&have, "OPENID")); + assert!(!granted(&have, "Calendars.Read")); + assert!(!granted(&connection(&[]), "Notes.Read")); +} + +#[test] +fn sign_in_scopes_count_as_granted_however_the_service_answers() { + // Microsoft's token answer lists only the resource scopes; Google's spells email as a URL. + let microsoft = connection(&["User.Read", "Calendars.Read"]); + let google = connection(&[ + "openid", + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/calendar.readonly", + ]); + for scope in ["openid", "email", "profile", "offline_access"] { + assert!(granted(µsoft, scope), "{scope}"); + assert!(granted(&google, scope), "{scope}"); + } + assert!(!granted(µsoft, "Notes.Read")); +} + +#[test] +fn a_capability_stands_for_its_scopes_and_other_names_are_scopes() { + let google = registry::find("google").expect("google"); + assert_eq!( + expand(google, &["classroomRead"]), + [ + "https://www.googleapis.com/auth/classroom.courses.readonly", + "https://www.googleapis.com/auth/classroom.coursework.me.readonly" + ] + ); + assert_eq!(expand(google, &["some-scope"]), ["some-scope"]); + assert!(expand(google, &[]).is_empty()); +} + +#[test] +fn callers_cannot_set_the_token_or_name_the_server() { + for name in [ + "Authorization", + "authorization", + "Cookie", + "Host", + "Proxy-Authorization", + "Content-Length", + ] { + assert!(!header_ok(name, "x"), "{name}"); + } + assert!(header_ok("Accept", "application/json")); + assert!(!header_ok("X-Test", "a\r\nInjected: 1")); + assert!(!header_ok("Bad Name", "x")); + assert!(!header_ok("", "x")); +} + +#[test] +fn the_token_goes_in_the_header_or_the_form_as_the_service_wants() { + let token = Secret::new("tok-123"); + let request = || HttpRequest::new(registry::Method::Get, "https://example.test/"); + let header = with_token(request(), Placement::Header("Bearer"), &token).expect("signs"); + assert_eq!( + header.headers, + [("Authorization".to_owned(), "Bearer tok-123".to_owned())] + ); + let form = with_token(request(), Placement::FormField("wstoken"), &token).expect("signs"); + let Some(Body::Form(fields)) = form.body else { + panic!("a form body") + }; + assert_eq!(fields, [("wstoken".to_owned(), "tok-123".to_owned())]); + let bytes = HttpRequest { + follow_redirects: true, + body: Some(Body::Bytes { + content_type: "x".into(), + data: vec![], + }), + ..request() + }; + assert!(with_token(bytes, Placement::FormField("wstoken"), &token).is_none()); +} diff --git a/app/src-tauri/src/connectors/store.rs b/app/src-tauri/src/connectors/store.rs new file mode 100644 index 00000000..754f7bd0 --- /dev/null +++ b/app/src-tauri/src/connectors/store.rs @@ -0,0 +1,304 @@ +//! Where the secrets live: Windows Credential Manager, as generic credentials named +//! `OpenNote//`. This is the only place a token, a refresh token, or a pasted personal token +//! is kept. They never go into the notes folder, the settings files, the connections file, a log, a crash report, or +//! the feedback file. The same trait has a memory store for tests and for systems that have no credential manager +//! yet, where a secret lasts until the app exits and the connection then reads as "Sign-in expired". + +use std::{ + collections::HashMap, + sync::{Arc, Mutex, PoisonError}, +}; + +use super::secret::Secret; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StoreError; + +pub trait SecretStore: Send + Sync { + fn put(&self, target: &str, secret: &Secret) -> Result<(), StoreError>; + fn get(&self, target: &str) -> Result, StoreError>; + /// Removes the secret. A secret that isn't there is not an error. + fn delete(&self, target: &str) -> Result<(), StoreError>; +} + +/// The credential name for a connector's account. Control characters are dropped and a long name is cut, so the +/// name is always one a credential can have. +pub fn target_name(connector: &str, account: &str) -> String { + let account: String = account.chars().filter(|c| !c.is_control()).take(100).collect(); + let account = account.trim(); + let account = if account.is_empty() { "default" } else { account }; + format!("OpenNote/{connector}/{account}") +} + +/// The store this system has. +pub fn platform_store() -> Arc { + #[cfg(windows)] + { + Arc::new(windows_store::WindowsStore) + } + #[cfg(not(windows))] + { + Arc::new(MemoryStore::default()) + } +} + +#[derive(Default)] +pub struct MemoryStore { + secrets: Mutex>, +} + +impl SecretStore for MemoryStore { + fn put(&self, target: &str, secret: &Secret) -> Result<(), StoreError> { + let mut secrets = self.secrets.lock().unwrap_or_else(PoisonError::into_inner); + secrets.insert(target.to_owned(), secret.expose().to_owned()); + Ok(()) + } + + fn get(&self, target: &str) -> Result, StoreError> { + let secrets = self.secrets.lock().unwrap_or_else(PoisonError::into_inner); + Ok(secrets.get(target).map(Secret::new)) + } + + fn delete(&self, target: &str) -> Result<(), StoreError> { + self.secrets + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(target); + Ok(()) + } +} + +#[cfg(windows)] +mod windows_store { + //! A credential holds at most 2,560 bytes, and a Microsoft refresh token can come close. A longer secret is + //! split over ``, `#1`, and so on. The first part starts with the number of parts and a bar. + + use windows::{ + core::{HRESULT, HSTRING, PWSTR}, + Win32::{ + Foundation::ERROR_NOT_FOUND, + Security::Credentials::{ + CredDeleteW, CredFree, CredReadW, CredWriteW, CREDENTIALW, CRED_PERSIST_LOCAL_MACHINE, + CRED_TYPE_GENERIC, + }, + }, + }; + + use super::{Secret, SecretStore, StoreError}; + + /// Bytes of secret per credential, well under the limit. + const PART_BYTES: usize = 1800; + /// The most parts a secret can have. A count above this in a stored value means the value is not ours. + const MAX_PARTS: usize = 16; + + pub struct WindowsStore; + + fn part_target(target: &str, index: usize) -> String { + if index == 0 { + target.to_owned() + } else { + format!("{target}#{index}") + } + } + + fn split(text: &str) -> Vec<&str> { + let mut parts = Vec::new(); + let mut rest = text; + while rest.len() > PART_BYTES { + let cut = (0..=PART_BYTES) + .rev() + .find(|&at| rest.is_char_boundary(at)) + .unwrap_or(0); + let (head, tail) = rest.split_at(cut); + parts.push(head); + rest = tail; + } + parts.push(rest); + parts + } + + fn write_one(target: &str, text: &str) -> Result<(), StoreError> { + let mut name: Vec = target.encode_utf16().chain(Some(0)).collect(); + let mut user: Vec = "OpenNote".encode_utf16().chain(Some(0)).collect(); + let mut blob = text.as_bytes().to_vec(); + let credential = CREDENTIALW { + Type: CRED_TYPE_GENERIC, + TargetName: PWSTR(name.as_mut_ptr()), + UserName: PWSTR(user.as_mut_ptr()), + CredentialBlobSize: u32::try_from(blob.len()).map_err(|_| StoreError)?, + CredentialBlob: blob.as_mut_ptr(), + Persist: CRED_PERSIST_LOCAL_MACHINE, + ..Default::default() + }; + // SAFETY: the credential points at buffers that live until the call returns, and the name and the user + // name end with a null character. Windows copies everything before CredWriteW returns. + unsafe { CredWriteW(&credential, 0) }.map_err(|_| StoreError) + } + + fn read_one(target: &str) -> Result, StoreError> { + let name = HSTRING::from(target); + let mut found: *mut CREDENTIALW = std::ptr::null_mut(); + // SAFETY: `found` is a valid place for Windows to put a pointer, and the name is NUL-terminated. + match unsafe { CredReadW(&name, CRED_TYPE_GENERIC, None, &mut found) } { + Ok(()) => { + // SAFETY: CredReadW succeeded, so `found` points at a credential that stays valid until CredFree. + let text = unsafe { + let credential = &*found; + let bytes = if credential.CredentialBlob.is_null() { + &[][..] + } else { + std::slice::from_raw_parts(credential.CredentialBlob, credential.CredentialBlobSize as usize) + }; + let text = String::from_utf8(bytes.to_vec()); + CredFree(found.cast()); + text + }; + text.map(Some).map_err(|_| StoreError) + } + Err(error) if error.code() == HRESULT::from_win32(ERROR_NOT_FOUND.0) => Ok(None), + Err(_) => Err(StoreError), + } + } + + fn delete_one(target: &str) -> Result<(), StoreError> { + let name = HSTRING::from(target); + // SAFETY: the name is NUL-terminated. + match unsafe { CredDeleteW(&name, CRED_TYPE_GENERIC, None) } { + Ok(()) => Ok(()), + Err(error) if error.code() == HRESULT::from_win32(ERROR_NOT_FOUND.0) => Ok(()), + Err(_) => Err(StoreError), + } + } + + /// How many parts the stored value has, from the first part. None when there is no first part. + fn stored_parts(target: &str) -> Result, StoreError> { + let Some(first) = read_one(target)? else { + return Ok(None); + }; + let (count, rest) = first.split_once('|').ok_or(StoreError)?; + let count: usize = count.parse().map_err(|_| StoreError)?; + if count == 0 || count > MAX_PARTS { + return Err(StoreError); + } + Ok(Some((count, rest.to_owned()))) + } + + impl SecretStore for WindowsStore { + fn put(&self, target: &str, secret: &Secret) -> Result<(), StoreError> { + // When the old first part can't be read, its count is unknown, so every possible part goes. + let old = match stored_parts(target) { + Ok(Some((count, _))) => count, + Ok(None) => 0, + Err(_) => MAX_PARTS, + }; + let parts = split(secret.expose()); + if parts.len() > MAX_PARTS { + return Err(StoreError); + } + for (index, part) in parts.iter().enumerate().skip(1) { + write_one(&part_target(target, index), part)?; + } + write_one(target, &format!("{}|{}", parts.len(), parts[0]))?; + for index in parts.len()..old { + delete_one(&part_target(target, index))?; + } + Ok(()) + } + + fn get(&self, target: &str) -> Result, StoreError> { + let Some((count, mut text)) = stored_parts(target)? else { + return Ok(None); + }; + for index in 1..count { + text.push_str(&read_one(&part_target(target, index))?.ok_or(StoreError)?); + } + Ok(Some(Secret::new(text))) + } + + fn delete(&self, target: &str) -> Result<(), StoreError> { + let count = match stored_parts(target) { + Ok(Some((count, _))) => count, + Ok(None) => 1, + Err(_) => MAX_PARTS, + }; + for index in (0..count).rev() { + delete_one(&part_target(target, index))?; + } + Ok(()) + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn splits_on_character_boundaries_and_rejoins() { + let text = "é".repeat(2000); + let parts = split(&text); + assert!(parts.iter().all(|part| part.len() <= PART_BYTES)); + assert_eq!(parts.concat(), text); + assert_eq!(split("short"), vec!["short"]); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The round trip every store must pass: put, get, replace, delete, and a missing secret. + pub(super) fn round_trip(store: &dyn SecretStore, target: &str) { + assert_eq!(store.get(target), Ok(None)); + let first = Secret::new("first-secret-value"); + store.put(target, &first).expect("stores"); + assert_eq!(store.get(target), Ok(Some(first))); + let longer = Secret::new(format!("{}é", "x".repeat(4500))); + store.put(target, &longer).expect("replaces with a longer secret"); + assert_eq!(store.get(target), Ok(Some(longer))); + let shorter = Secret::new("short"); + store.put(target, &shorter).expect("replaces with a shorter secret"); + assert_eq!(store.get(target), Ok(Some(shorter))); + store.delete(target).expect("deletes"); + assert_eq!(store.get(target), Ok(None)); + store.delete(target).expect("deleting what isn't there is fine"); + } + + #[test] + fn names_follow_opennote_connector_account() { + assert_eq!( + target_name("google", "sam@example.com"), + "OpenNote/google/sam@example.com" + ); + assert_eq!(target_name("readwise", ""), "OpenNote/readwise/default"); + assert_eq!(target_name("slack", "Team\nName"), "OpenNote/slack/TeamName"); + assert_eq!( + target_name("slack", &"a".repeat(300)).len(), + "OpenNote/slack/".len() + 100 + ); + } + + #[test] + fn the_memory_store_round_trips() { + round_trip(&MemoryStore::default(), "OpenNote/test/memory"); + } + + #[test] + fn two_connectors_keep_separate_secrets() { + let store = MemoryStore::default(); + store.put("OpenNote/a/x", &Secret::new("one")).expect("stores"); + store.put("OpenNote/b/x", &Secret::new("two")).expect("stores"); + store.delete("OpenNote/a/x").expect("deletes"); + assert_eq!(store.get("OpenNote/b/x"), Ok(Some(Secret::new("two")))); + } + + /// The real Credential Manager, under a name no connector uses, removed at the end. + #[cfg(windows)] + #[test] + fn windows_credential_manager_round_trips() { + let target = format!("OpenNote-test/{}/round-trip", std::process::id()); + let store = windows_store::WindowsStore; + round_trip(&store, &target); + store.delete(&target).expect("cleans up"); + } +} diff --git a/app/src-tauri/src/connectors/test_hook.rs b/app/src-tauri/src/connectors/test_hook.rs new file mode 100644 index 00000000..cf161dfb --- /dev/null +++ b/app/src-tauri/src/connectors/test_hook.rs @@ -0,0 +1,143 @@ +//! For the end-to-end builds only (the `test-endpoints` feature, which release builds never have: build.rs compiles +//! a marker into them and `check-release-exe.ts` refuses an exe that holds it). It lets a test drive the real app +//! against a mock service without a real account: +//! +//! - `OPENNOTE_TEST_CONNECTOR_BASE=http://127.0.0.1:` sends every connector request to that mock instead, as +//! `//`. The pinned-host check still runs on the real address first, +//! so the test also proves a token would only have gone to the service's own host. +//! +//! - `OPENNOTE_TEST_CONNECTED=microsoft:alice@example.com,google:bob@example.com` starts with those connectors +//! already connected, with every access they can be granted and a fixed test token in memory. Nothing is written +//! to the Credential Manager. + +use std::sync::Arc; + +use super::{ + files::{Connection, Holds}, + http::{HostPolicy, Http, HttpError, HttpRequest, HttpResponse}, + registry, + secret::Secret, + service::{unix_now, Connectors, Parts}, + store::{target_name, MemoryStore, SecretStore}, +}; +use crate::paths::Paths; + +const BASE: &str = "OPENNOTE_TEST_CONNECTOR_BASE"; +const CONNECTED: &str = "OPENNOTE_TEST_CONNECTED"; +const TOKEN: &str = "test-token-not-a-secret"; + +struct Redirect { + inner: Arc, + base: String, +} + +impl Http for Redirect { + fn send(&self, policy: &HostPolicy, request: &HttpRequest, max_bytes: usize) -> Result { + if !policy.allows(&request.url) { + return Err(HttpError::ForeignHost); + } + let url = url::Url::parse(&request.url).map_err(|_| HttpError::ForeignHost)?; + let host = url.host_str().ok_or(HttpError::ForeignHost)?; + let query = url.query().map(|query| format!("?{query}")).unwrap_or_default(); + let moved = HttpRequest { + follow_redirects: request.follow_redirects, + method: request.method, + url: format!("{}/{host}{}{query}", self.base, url.path()), + headers: request.headers.clone(), + body: request.body.clone(), + }; + self.inner.send(&HostPolicy::new(["127.0.0.1"]), &moved, max_bytes) + } +} + +/// The connectors for a test build that asks for the mock, or None for an ordinary start. +pub(super) fn connectors(paths: &Paths) -> Option { + let base = std::env::var(BASE).ok()?; + let base = base.trim_end_matches('/').to_owned(); + if !base.starts_with("http://127.0.0.1:") { + return None; + } + let mut parts = Parts::real(paths); + let store = Arc::new(MemoryStore::default()); + parts.store = store.clone(); + parts.http = Arc::new(Redirect { + inner: parts.http.clone(), + base, + }); + parts.offline = Arc::new(|| false); + let connectors = Connectors::with(parts); + for entry in std::env::var(CONNECTED).unwrap_or_default().split(',') { + let Some((id, account)) = entry.split_once(':') else { + continue; + }; + let Some(def) = registry::find(id.trim()) else { continue }; + let account = account.trim().to_owned(); + if store.put(&target_name(def.id, &account), &Secret::new(TOKEN)).is_err() { + continue; + } + let connection = Connection { + account, + scopes: def.scopes().into_iter().map(str::to_owned).collect(), + connected_unix: unix_now(), + holds: Holds::Access, + base_url: None, + expired: false, + last_used_unix: None, + }; + connectors.insert_test_connection(def.id, connection); + } + Some(connectors) +} + +impl Connectors { + fn insert_test_connection(&self, id: &str, connection: Connection) { + self.lock().connections.insert(id.to_owned(), connection); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::connectors::{ + http::UreqHttp, + mock::{MockServer, Reply}, + registry::Method, + }; + + fn redirect(server: &MockServer) -> Redirect { + Redirect { + inner: Arc::new(UreqHttp::default()), + base: server.url(""), + } + } + + #[test] + fn a_request_to_a_pinned_host_goes_to_the_mock_under_that_host() { + let server = MockServer::start(); + server.route( + "/graph.microsoft.com/v1.0/me/onenote/notebooks", + vec![Reply::json(200, serde_json::json!({ "value": [] }))], + ); + let policy = HostPolicy::new(["graph.microsoft.com"]); + let request = HttpRequest::new(Method::Get, "https://graph.microsoft.com/v1.0/me/onenote/notebooks?a=1") + .header("Authorization", "Bearer t"); + let reply = redirect(&server).send(&policy, &request, 1024).expect("answers"); + assert_eq!(reply.status, 200); + let seen = server.requests_to("/graph.microsoft.com/v1.0/me/onenote/notebooks"); + assert_eq!(seen.len(), 1); + assert!(seen[0].target.ends_with("?a=1")); + assert_eq!(seen[0].header("authorization"), Some("Bearer t")); + } + + #[test] + fn a_request_to_any_other_host_is_still_refused() { + let server = MockServer::start(); + let policy = HostPolicy::new(["graph.microsoft.com"]); + let request = HttpRequest::new(Method::Get, "https://evil.example/v1.0/me"); + assert_eq!( + redirect(&server).send(&policy, &request, 1024).err(), + Some(HttpError::ForeignHost) + ); + assert!(server.requests().is_empty()); + } +} diff --git a/app/src-tauri/src/connectors/tests/mod.rs b/app/src-tauri/src/connectors/tests/mod.rs new file mode 100644 index 00000000..ae641bf4 --- /dev/null +++ b/app/src-tauri/src/connectors/tests/mod.rs @@ -0,0 +1,260 @@ +//! The whole sign-in, from the card's Connect to the token on a request, against a local mock server and a browser +//! double. No real service, client ID, or token is involved: every value here is made up. `sign_in` covers signing +//! in, `session` covers renewing, requests, and disconnecting, and `pasted` covers pasted tokens and the catalog. + +use std::{ + fs, + io::{Read, Write}, + net::TcpStream, + path::Path, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, + }, + thread, + time::Duration, +}; + +use serde_json::json; +use url::Url; + +use super::{ + error::Failure, + files::CONNECTIONS_FILE, + http::{HostPolicy, Http, HttpError, HttpRequest, HttpResponse, UreqHttp}, + mock::{ + fixtures::{connector, oauth, registry}, + MockServer, Reply, + }, + registry::{Auth, ConnectorDef, Method, Placement, Revoke, TokenDef}, + secret::Secret, + service::{Connectors, Opener, Parts}, + store::{MemoryStore, SecretStore}, + view::{ConnectInput, RevokeOutcome, StateView}, +}; + +const CLIENT_FILE: &str = + r#"{"clients":{"example":{"clientId":"example-client-id","clientSecret":"example-client-secret"}}}"#; +const NOW: u64 = 1_790_000_000; + +/// What the person does in the browser. +#[derive(Clone, Copy)] +enum Visit { + Approve(&'static str), + Refuse(&'static str), + WrongState, + Silent, +} + +struct FakeBrowser { + visit: Mutex, + opened: Mutex>, +} + +impl Opener for FakeBrowser { + fn open(&self, url: &str) -> bool { + self.opened.lock().expect("lock").push(url.to_owned()); + let visit = *self.visit.lock().expect("lock"); + let parsed = Url::parse(url).expect("a URL"); + let pair = |name: &str| { + parsed + .query_pairs() + .find(|(key, _)| key == name) + .map(|(_, v)| v.into_owned()) + .expect(name) + }; + let (redirect, state) = (pair("redirect_uri"), pair("state")); + let query = match visit { + Visit::Silent => return true, + Visit::Approve(code) => format!("code={code}&state={state}"), + Visit::Refuse(reason) => format!("error={reason}&state={state}"), + Visit::WrongState => "code=stolen&state=not-this-one".to_owned(), + }; + thread::spawn(move || { + let target = Url::parse(&redirect).expect("a redirect"); + let port = target.port().expect("a port"); + let mut stream = TcpStream::connect(("127.0.0.1", port)).expect("the listener is open"); + let request = format!( + "GET {}?{query} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n", + target.path() + ); + stream.write_all(request.as_bytes()).expect("sends"); + let mut answer = String::new(); + let _ = stream.read_to_string(&mut answer); + }); + true + } +} + +/// An `Http` that answers from a script, and applies the host policy the way the real client does. +#[derive(Default)] +struct ScriptedHttp { + replies: Mutex>, + seen: Mutex>, +} + +/// One request the script saw: the hosts allowed for it, where it went, its headers, and its form body. +struct Seen { + hosts: Vec, + url: String, + headers: Vec<(String, String)>, + body: String, +} + +impl Http for ScriptedHttp { + fn send(&self, policy: &HostPolicy, request: &HttpRequest, _max: usize) -> Result { + if !policy.allows(&request.url) { + return Err(HttpError::ForeignHost); + } + let body = match &request.body { + Some(super::http::Body::Form(fields)) => super::http::encode_form(fields), + _ => String::new(), + }; + self.seen.lock().expect("lock").push(Seen { + hosts: policy.hosts().to_vec(), + url: request.url.clone(), + headers: request.headers.clone(), + body, + }); + let mut replies = self.replies.lock().expect("lock"); + let (status, body) = if replies.len() > 1 { + replies.remove(0) + } else { + replies.first().cloned().unwrap_or((500, json!({}))) + }; + Ok(HttpResponse { + status, + content_type: None, + body: body.to_string().into_bytes(), + location: None, + }) + } +} + +struct Rig { + dir: tempfile::TempDir, + connectors: Connectors, + store: Arc, + browser: Arc, + offline: Arc, + clock: Arc, + registry: &'static [ConnectorDef], + http: Arc, +} + +impl Rig { + fn new(registry: &'static [ConnectorDef], http: Arc, config: Option<&str>) -> Rig { + let dir = tempfile::tempdir().expect("a temporary folder"); + if let Some(config) = config { + fs::write(dir.path().join("connectors.json"), config).expect("writes the config"); + } + let mut rig = Rig { + dir, + connectors: Connectors::with(Parts { + config_file: "unused".into(), + connections_file: "unused".into(), + registry, + store: Arc::new(MemoryStore::default()), + http: Arc::clone(&http), + opener: Arc::new(FakeBrowser { + visit: Mutex::new(Visit::Silent), + opened: Mutex::default(), + }), + offline: Arc::new(|| false), + now: Arc::new(|| 0), + wait: Duration::from_secs(1), + }), + store: Arc::new(MemoryStore::default()), + browser: Arc::new(FakeBrowser { + visit: Mutex::new(Visit::Silent), + opened: Mutex::default(), + }), + offline: Arc::new(AtomicBool::new(false)), + clock: Arc::new(AtomicU64::new(NOW)), + registry, + http, + }; + rig.restart(Duration::from_secs(10)); + rig + } + + /// Starts the service again over the same files and credentials, as the app does at launch. + fn restart(&mut self, wait: Duration) { + let (offline, clock) = (Arc::clone(&self.offline), Arc::clone(&self.clock)); + self.connectors = Connectors::with(Parts { + config_file: self.dir.path().join("connectors.json"), + connections_file: self.dir.path().join(CONNECTIONS_FILE), + registry: self.registry, + store: Arc::clone(&self.store) as Arc, + http: Arc::clone(&self.http), + opener: Arc::clone(&self.browser) as Arc, + offline: Arc::new(move || offline.load(Ordering::Relaxed)), + now: Arc::new(move || clock.load(Ordering::Relaxed)), + wait, + }); + } + + fn visit(&self, visit: Visit) { + *self.browser.visit.lock().expect("lock") = visit; + } + + fn opened(&self) -> Vec { + self.browser.opened.lock().expect("lock").clone() + } + + fn state(&self, id: &str) -> StateView { + self.connectors.view(id).expect("a view").state + } + + /// Every byte of every file the app wrote. The connectors file is the owner's, not the app's. + fn files_text(&self) -> String { + fn walk(dir: &Path, out: &mut String) { + for entry in fs::read_dir(dir).expect("reads").flatten() { + let path = entry.path(); + if path.is_dir() { + walk(&path, out); + } else if path.file_name().is_some_and(|name| name != "connectors.json") { + out.push_str(&fs::read_to_string(&path).unwrap_or_default()); + } + } + } + let mut text = String::new(); + walk(self.dir.path(), &mut text); + text + } +} + +/// An OAuth connector on the mock server, signed in with a client from the connectors file. +fn oauth_rig(server: &MockServer) -> Rig { + let def = connector("example", Auth::OAuth(oauth(server))); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "access_token": "access-one", "refresh_token": "refresh-one", "expires_in": 3600, "scope": "scope-a scope-b" }), + )], + ); + server.route("/me", vec![Reply::json(200, json!({ "email": "sam@example.com" }))]); + server.route("/revoke", vec![Reply::empty(200)]); + server.route("/api/things", vec![Reply::json(200, json!({ "things": [] }))]); + Rig::new(registry(vec![def]), Arc::new(UreqHttp::default()), Some(CLIENT_FILE)) +} + +fn connect(rig: &Rig) -> Result { + rig.connectors + .connect("example", ConnectInput::default()) + .map(|view| view.state) + .map_err(|error| error.failure) +} + +fn connected(account: &str) -> StateView { + StateView::Connected { + account: account.into(), + connected_unix: NOW, + } +} + +mod pasted; +mod session; +mod sign_in; +mod transfer; diff --git a/app/src-tauri/src/connectors/tests/pasted.rs b/app/src-tauri/src/connectors/tests/pasted.rs new file mode 100644 index 00000000..fa27a056 --- /dev/null +++ b/app/src-tauri/src/connectors/tests/pasted.rs @@ -0,0 +1,277 @@ +//! Pasted tokens, school connectors, the real registry without client IDs, and the interface catalog fixture. + +use super::*; + +fn token_def( + needs_base_url: bool, + placement: Placement, + account_pointer: &'static str, + check_url: &'static str, +) -> TokenDef { + TokenDef { + needs_base_url, + placement, + // A form token goes in a post, as Moodle's does. + check_method: if matches!(placement, Placement::FormField(_)) { + Method::Post + } else { + Method::Get + }, + check_url, + check_form: &[], + account_pointer, + } +} + +fn scripted(replies: Vec<(u16, serde_json::Value)>) -> Arc { + let http = ScriptedHttp::default(); + *http.replies.lock().expect("lock") = replies; + Arc::new(http) +} + +fn token_connector(id: &'static str, def: TokenDef, hosts: &'static [&'static str]) -> &'static ConnectorDef { + let template = connector(id, Auth::Token(def)); + Box::leak(Box::new(ConnectorDef { hosts, ..*template })) +} + +#[test] +fn a_pasted_token_is_checked_stored_in_the_credential_store_and_used_with_its_scheme() { + let http = scripted(vec![(200, json!({ "name": "Sam Student" }))]); + let def = token_connector( + "tokened", + token_def( + false, + Placement::Header("Token"), + "/name", + "https://tokens.example/api/auth/", + ), + &["tokens.example"], + ); + let rig = Rig::new(registry(vec![def]), http.clone(), None); + assert_eq!( + rig.state("tokened"), + StateView::NotConnected, + "a token connector never needs setup" + ); + + let input = ConnectInput { + token: Some(" pasted-token-value-1234 ".into()), + base_url: None, + }; + let view = rig.connectors.connect("tokened", input).expect("connects"); + assert_eq!(view.state, connected("Sam Student")); + assert_eq!( + rig.store.get("OpenNote/tokened/Sam Student"), + Ok(Some(Secret::new("pasted-token-value-1234"))) + ); + assert!(!rig.files_text().contains("pasted-token-value-1234")); + assert_eq!( + http.seen.lock().expect("lock")[0].headers, + [("Authorization".to_owned(), "Token pasted-token-value-1234".to_owned())] + ); + + let request = HttpRequest::new(Method::Get, "https://tokens.example/api/highlights"); + rig.connectors.request("tokened", &[], request, 1000).expect("answers"); + assert_eq!( + http.seen.lock().expect("lock")[1].headers[0].1, + "Token pasted-token-value-1234" + ); + let foreign = HttpRequest::new(Method::Get, "https://other.example/api"); + assert_eq!( + rig.connectors + .request("tokened", &[], foreign, 1000) + .map(|_| ()) + .map_err(|e| e.failure), + Err(Failure::ForeignHost) + ); +} + +#[test] +fn a_token_the_service_refuses_is_not_kept() { + let http = scripted(vec![(401, json!({ "detail": "Invalid token." }))]); + let def = token_connector( + "tokened", + token_def( + false, + Placement::Header("Token"), + "", + "https://tokens.example/api/auth/", + ), + &["tokens.example"], + ); + let rig = Rig::new(registry(vec![def]), http, None); + let outcome = rig.connectors.connect( + "tokened", + ConnectInput { + token: Some("wrong-token-value-12".into()), + base_url: None, + }, + ); + assert_eq!(outcome.map(|_| ()).map_err(|e| e.failure), Err(Failure::Rejected)); + assert_eq!(rig.state("tokened"), StateView::NotConnected); + assert_eq!(rig.store.get("OpenNote/tokened/default"), Ok(None)); + for bad in [ + None, + Some(String::new()), + Some("short".to_owned()), + Some("has a space in it".to_owned()), + ] { + let outcome = rig.connectors.connect( + "tokened", + ConnectInput { + token: bad, + base_url: None, + }, + ); + assert_eq!(outcome.map(|_| ()).map_err(|e| e.failure), Err(Failure::BadInput)); + } +} + +#[test] +fn a_school_connector_talks_only_to_the_schools_own_host() { + let http = scripted(vec![(200, json!({ "fullname": "Sam Student" }))]); + let mut def = token_def( + true, + Placement::FormField("wstoken"), + "/fullname", + "/webservice/rest/server.php", + ); + def.check_form = &[("wsfunction", "core_webservice_get_site_info")]; + let def = token_connector("school", def, &[]); + let rig = Rig::new(registry(vec![def]), http.clone(), None); + + for bad in [ + "http://school.example.edu", + "https://127.0.0.1", + "https://school.example.edu@evil.example", + "", + ] { + let input = ConnectInput { + token: Some("school-token-abcdef12".into()), + base_url: Some(bad.into()), + }; + let outcome = rig + .connectors + .connect("school", input) + .map(|_| ()) + .map_err(|e| e.failure); + assert_eq!(outcome, Err(Failure::BadInput), "{bad}"); + } + assert!( + http.seen.lock().expect("lock").is_empty(), + "nothing was sent for a bad address" + ); + + let input = ConnectInput { + token: Some("school-token-abcdef12".into()), + base_url: Some("School.Example.edu/moodle/".into()), + }; + let view = rig.connectors.connect("school", input).expect("connects"); + assert_eq!(view.state, connected("Sam Student")); + assert_eq!(view.base_url.as_deref(), Some("https://school.example.edu/moodle")); + assert_eq!(view.hosts, ["school.example.edu"]); + { + let seen = http.seen.lock().expect("lock"); + assert_eq!(seen[0].hosts, ["school.example.edu"]); + assert_eq!( + seen[0].url, + "https://school.example.edu/moodle/webservice/rest/server.php" + ); + assert!(seen[0].body.contains("wstoken=school-token-abcdef12")); + assert!(seen[0].body.contains("wsfunction=core_webservice_get_site_info")); + } + assert!(rig.files_text().contains("https://school.example.edu/moodle")); + assert!(!rig.files_text().contains("school-token-abcdef12")); + + let foreign = HttpRequest::new(Method::Post, "https://other.example.edu/webservice/rest/server.php"); + assert_eq!( + rig.connectors + .request("school", &[], foreign, 1000) + .map(|_| ()) + .map_err(|e| e.failure), + Err(Failure::ForeignHost) + ); +} + +#[test] +fn the_real_registry_signs_in_nowhere_without_client_ids() { + let rig = Rig::new( + crate::connectors::registry::CONNECTORS, + Arc::new(UreqHttp::default()), + None, + ); + let list = rig.connectors.list(); + assert_eq!(list.len(), crate::connectors::registry::CONNECTORS.len()); + let oauth_ids = ["microsoft", "google", "slack", "dropbox", "box", "vimeo"]; + for view in &list { + let expected = if oauth_ids.contains(&view.id) { + StateView::NeedsSetup + } else { + StateView::NotConnected + }; + assert_eq!(view.state, expected, "{}", view.id); + } +} + +/// The interface's fake host builds its catalog from this file, so the web build and the component tests show +/// exactly what the real registry sends. Rewrite it after a deliberate change with `UPDATE_WIRE=1 cargo test`. +#[test] +fn the_interfaces_catalog_is_the_registrys_own_list() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../src/features/connectors/registry.fixture.json"); + let rig = Rig::new( + crate::connectors::registry::CONNECTORS, + Arc::new(UreqHttp::default()), + None, + ); + let actual = serde_json::to_value(rig.connectors.list()).expect("serializes"); + if std::env::var_os("UPDATE_WIRE").is_some() { + let text = serde_json::to_string_pretty(&actual).expect("serializes"); + fs::write(&path, format!("{text}\n")).expect("writes the fixture"); + } + let saved: serde_json::Value = + serde_json::from_str(&fs::read_to_string(&path).expect("the fixture exists")).expect("json"); + assert_eq!( + saved, + actual, + "run UPDATE_WIRE=1 cargo test -p opennote connectors to rewrite {}", + path.display() + ); +} + +#[test] +fn a_401_on_one_endpoint_expires_a_pasted_token_only_when_the_token_check_fails_too() { + let http = scripted(vec![ + (200, json!({ "name": "Sam" })), + (401, json!({ "errors": "Insufficient scopes on access token." })), + (200, json!({ "name": "Sam" })), + (401, json!({})), + (401, json!({})), + ]); + let def = token_connector( + "tokened", + token_def( + false, + Placement::Header("Bearer"), + "/name", + "https://tokens.example/api/me", + ), + &["tokens.example"], + ); + let rig = Rig::new(registry(vec![def]), http, None); + let input = ConnectInput { + token: Some("pasted-token-value-1234".into()), + base_url: None, + }; + rig.connectors.connect("tokened", input).expect("connects"); + let ask = || { + let request = HttpRequest::new(Method::Get, "https://tokens.example/api/analytics"); + rig.connectors + .request("tokened", &[], request, 1000) + .map(|_| ()) + .map_err(|e| e.failure) + }; + assert_eq!(ask(), Err(Failure::Rejected), "the token still checks out"); + assert_eq!(rig.state("tokened"), connected("Sam")); + assert_eq!(ask(), Err(Failure::Expired), "the token check is refused too"); + assert_ne!(rig.state("tokened"), connected("Sam")); +} diff --git a/app/src-tauri/src/connectors/tests/session.rs b/app/src-tauri/src/connectors/tests/session.rs new file mode 100644 index 00000000..f5cb3953 --- /dev/null +++ b/app/src-tauri/src/connectors/tests/session.rs @@ -0,0 +1,319 @@ +//! Using a connection: renewing tokens, requests, expiry, disconnecting, and changing accounts. + +use super::*; + +#[test] +fn a_token_is_renewed_before_it_ends_and_a_rotated_refresh_token_replaces_the_old_one() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + let token = |rig: &Rig| { + rig.connectors + .access_token("example", &["calendarRead"]) + .expect("a token") + .expose() + .to_owned() + }; + + assert_eq!(token(&rig), "access-one"); + assert_eq!( + server.requests_to("/token").len(), + 1, + "the one from sign-in is still good" + ); + + server.clear_routes(); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "access_token": "access-two", "refresh_token": "refresh-two", "expires_in": 3600 }), + )], + ); + rig.clock.store(NOW + 3600 - 60, Ordering::Relaxed); + assert_eq!(token(&rig), "access-two"); + let renewal = &server.requests_to("/token")[1]; + assert_eq!(renewal.field("grant_type").as_deref(), Some("refresh_token")); + assert_eq!(renewal.field("refresh_token").as_deref(), Some("refresh-one")); + assert_eq!( + rig.store.get("OpenNote/example/sam@example.com"), + Ok(Some(Secret::new("refresh-two"))) + ); + assert_eq!(token(&rig), "access-two"); + assert_eq!( + server.requests_to("/token").len(), + 2, + "one renewal, and the next call used it" + ); +} + +#[test] +fn a_restart_keeps_the_connection_and_renews_from_the_credential_store() { + let server = MockServer::start(); + let mut rig = oauth_rig(&server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + server.clear_routes(); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "access_token": "access-after-restart", "expires_in": 3600 }), + )], + ); + rig.restart(Duration::from_secs(5)); + assert_eq!(rig.state("example"), connected("sam@example.com")); + let token = rig.connectors.access_token("example", &[]).expect("a token"); + assert_eq!(token.expose(), "access-after-restart"); +} + +#[test] +fn access_the_connection_lacks_is_reported_not_guessed() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + server.clear_routes(); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "access_token": "a", "refresh_token": "r", "scope": "scope-a" }), + )], + ); + server.route("/me", vec![Reply::json(200, json!({ "email": "sam@example.com" }))]); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + assert!(rig.connectors.access_token("example", &["calendarRead"]).is_ok()); + let missing = rig + .connectors + .access_token("example", &["tasksWrite"]) + .map(|_| ()) + .map_err(|e| e.failure); + assert_eq!(missing, Err(Failure::MissingAccess)); + assert_eq!( + rig.connectors + .access_token("nothing", &[]) + .map(|_| ()) + .map_err(|e| e.failure), + Err(Failure::Unknown) + ); +} + +#[test] +fn a_refused_renewal_marks_the_sign_in_expired_until_the_person_reconnects() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + server.clear_routes(); + server.route("/token", vec![Reply::json(400, json!({ "error": "invalid_grant" }))]); + rig.clock.store(NOW + 4000, Ordering::Relaxed); + let outcome = rig + .connectors + .access_token("example", &[]) + .map(|_| ()) + .map_err(|e| e.failure); + assert_eq!(outcome, Err(Failure::Expired)); + assert_eq!( + rig.state("example"), + StateView::Expired { + account: "sam@example.com".into() + } + ); + assert!(!rig.connectors.is_connected("example")); + assert!(fs::read_to_string(rig.dir.path().join(CONNECTIONS_FILE)) + .expect("reads") + .contains("\"expired\": true")); + let again = rig + .connectors + .access_token("example", &[]) + .map(|_| ()) + .map_err(|e| e.failure); + assert_eq!(again, Err(Failure::Expired)); + assert_eq!( + server.requests_to("/token").len(), + 2, + "an expired sign-in isn't retried on every call" + ); + + server.clear_routes(); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "access_token": "fresh", "refresh_token": "fresh-refresh", "expires_in": 3600 }), + )], + ); + server.route("/me", vec![Reply::json(200, json!({ "email": "sam@example.com" }))]); + rig.visit(Visit::Approve("c2")); + assert!(matches!(connect(&rig), Ok(StateView::Connected { account, .. }) if account == "sam@example.com")); + assert!(rig.connectors.is_connected("example")); +} + +#[test] +fn requests_carry_the_token_and_go_only_to_allowed_hosts() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + let get = |url: String, headers: Vec<(&str, &str)>| { + let mut request = HttpRequest::new(Method::Get, url); + for (name, value) in headers { + request = request.header(name, value); + } + rig.connectors + .request("example", &["calendarRead"], request, 10_000) + .map_err(|e| e.failure) + }; + let answer = get(server.url("/api/things"), vec![("Accept", "application/json")]).expect("answers"); + assert_eq!(answer.status, 200); + let sent = &server.requests_to("/api/things")[0]; + assert_eq!(sent.header("authorization"), Some("Bearer access-one")); + assert_eq!(sent.header("accept"), Some("application/json")); + + assert_eq!( + get("https://evil.example/steal".into(), vec![]).map(|_| ()), + Err(Failure::ForeignHost) + ); + assert_eq!( + get("https://127.0.0.1.evil.example/".into(), vec![]).map(|_| ()), + Err(Failure::ForeignHost) + ); + assert_eq!( + get(server.url("/api/things"), vec![("Authorization", "Bearer mine")]).map(|_| ()), + Err(Failure::BadInput) + ); + assert_eq!( + server.requests_to("/api/things").len(), + 1, + "the refused ones were never sent" + ); +} + +#[test] +fn a_401_renews_the_token_once_and_a_second_401_expires_the_sign_in() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + server.clear_routes(); + server.route("/api/things", vec![Reply::empty(401), Reply::json(200, json!({}))]); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "access_token": "access-renewed", "expires_in": 3600 }), + )], + ); + let request = || HttpRequest::new(Method::Get, server.url("/api/things")); + let answer = rig + .connectors + .request("example", &[], request(), 1000) + .expect("answers"); + assert_eq!(answer.status, 200); + assert_eq!( + server.requests_to("/api/things")[1].header("authorization"), + Some("Bearer access-renewed") + ); + + server.clear_routes(); + server.route("/api/things", vec![Reply::empty(401)]); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "access_token": "access-again", "expires_in": 3600 }), + )], + ); + let outcome = rig + .connectors + .request("example", &[], request(), 1000) + .map(|_| ()) + .map_err(|e| e.failure); + assert_eq!(outcome, Err(Failure::Expired)); + assert_eq!( + rig.state("example"), + StateView::Expired { + account: "sam@example.com".into() + } + ); +} + +#[test] +fn disconnecting_revokes_deletes_the_credential_and_forgets_the_account() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + let done = rig.connectors.disconnect("example").expect("disconnects"); + assert_eq!(done.revoke, RevokeOutcome::Revoked); + assert_eq!(done.view.state, StateView::NotConnected); + assert_eq!( + server.requests_to("/revoke")[0].field("token").as_deref(), + Some("refresh-one") + ); + assert_eq!(rig.store.get("OpenNote/example/sam@example.com"), Ok(None)); + assert!(!rig.files_text().contains("sam@example.com")); + assert!(!rig.connectors.is_connected("example")); + assert_eq!( + rig.connectors.disconnect("example").expect("again").revoke, + RevokeOutcome::Nothing + ); +} + +#[test] +fn a_service_that_cannot_be_told_still_disconnects_here() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + server.clear_routes(); + server.route("/revoke", vec![Reply::empty(503)]); + let done = rig.connectors.disconnect("example").expect("disconnects"); + assert_eq!(done.revoke, RevokeOutcome::Failed); + assert_eq!(rig.store.get("OpenNote/example/sam@example.com"), Ok(None)); + assert_eq!(rig.state("example"), StateView::NotConnected); +} + +#[test] +fn a_service_with_no_revoke_endpoint_says_so() { + let server = MockServer::start(); + let mut settings = oauth(&server); + settings.revoke = Revoke::None; + let def = connector("example", Auth::OAuth(settings)); + server.route( + "/token", + vec![Reply::json(200, json!({ "access_token": "a", "refresh_token": "r" }))], + ); + server.route("/me", vec![Reply::json(200, json!({ "email": "sam@example.com" }))]); + let rig = Rig::new(registry(vec![def]), Arc::new(UreqHttp::default()), Some(CLIENT_FILE)); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + assert_eq!( + rig.connectors.disconnect("example").expect("disconnects").revoke, + RevokeOutcome::NotSupported + ); +} + +#[test] +fn signing_in_as_another_account_replaces_the_old_credential() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + server.clear_routes(); + server.route( + "/token", + vec![Reply::json( + 200, + json!({ "access_token": "a2", "refresh_token": "refresh-other" }), + )], + ); + server.route("/me", vec![Reply::json(200, json!({ "email": "lee@example.com" }))]); + assert_eq!(connect(&rig), Ok(connected("lee@example.com"))); + assert_eq!(rig.store.get("OpenNote/example/sam@example.com"), Ok(None)); + assert_eq!( + rig.store.get("OpenNote/example/lee@example.com"), + Ok(Some(Secret::new("refresh-other"))) + ); +} diff --git a/app/src-tauri/src/connectors/tests/sign_in.rs b/app/src-tauri/src/connectors/tests/sign_in.rs new file mode 100644 index 00000000..2950b037 --- /dev/null +++ b/app/src-tauri/src/connectors/tests/sign_in.rs @@ -0,0 +1,180 @@ +//! Signing in: the browser, the state check, the time limit, cancel, and what lands on disk. + +use super::*; + +#[test] +fn signs_in_through_the_browser_and_keeps_only_metadata_on_disk() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Approve("the-auth-code")); + assert_eq!(connect(&rig), Ok(connected("sam@example.com"))); + + // The browser was sent to the service's own page, with state, an S256 challenge, and the loopback redirect. + let opened = rig.opened(); + assert_eq!(opened.len(), 1); + let url = Url::parse(&opened[0]).expect("a URL"); + let query: std::collections::HashMap<_, _> = url.query_pairs().into_owned().collect(); + assert!(opened[0].starts_with(&server.url("/authorize"))); + assert_eq!(query["code_challenge_method"], "S256"); + assert!(query["redirect_uri"].starts_with("http://127.0.0.1:") && query["redirect_uri"].ends_with("/callback")); + assert_eq!(query["state"].len(), 32); + + // The code was traded with the verifier whose challenge the browser was given, and with the client. + let exchange = &server.requests_to("/token")[0]; + assert_eq!(exchange.field("code").as_deref(), Some("the-auth-code")); + let verifier = exchange.field("code_verifier").expect("a verifier"); + assert_eq!(crate::connectors::pkce::challenge(&verifier), query["code_challenge"]); + assert_eq!( + exchange.field("redirect_uri").as_deref(), + Some(query["redirect_uri"].as_str()) + ); + assert_eq!( + exchange.field("client_secret").as_deref(), + Some("example-client-secret") + ); + + // The refresh token is in the credential store under the documented name, and nowhere on disk. + assert_eq!( + rig.store.get("OpenNote/example/sam@example.com"), + Ok(Some(Secret::new("refresh-one"))) + ); + let on_disk = rig.files_text(); + for leak in [ + "refresh-one", + "access-one", + "the-auth-code", + verifier.as_str(), + "example-client-secret", + ] { + assert!(!on_disk.contains(leak), "{leak} is on disk"); + } + let saved: serde_json::Value = + serde_json::from_str(&fs::read_to_string(rig.dir.path().join(CONNECTIONS_FILE)).expect("reads")).expect("json"); + let entry = saved["connections"]["example"].as_object().expect("an entry").clone(); + let mut keys: Vec<_> = entry.keys().map(String::as_str).collect(); + keys.sort_unstable(); + assert_eq!(keys, ["account", "connectedUnix", "holds", "scopes"]); + assert_eq!(entry["scopes"], json!(["scope-a", "scope-b"])); + assert!(rig.connectors.is_connected("example")); + let listed = format!("{:?}", rig.connectors.list()); + assert!(!listed.contains("access-one") && !listed.contains("refresh-one")); +} + +#[test] +fn without_a_client_id_the_card_needs_setup_and_nothing_opens() { + let server = MockServer::start(); + let def = connector("example", Auth::OAuth(oauth(&server))); + let rig = Rig::new(registry(vec![def]), Arc::new(UreqHttp::default()), None); + assert_eq!(rig.state("example"), StateView::NeedsSetup); + rig.visit(Visit::Approve("x")); + assert_eq!(connect(&rig), Err(Failure::NotConfigured)); + assert!(rig.opened().is_empty() && server.requests().is_empty()); +} + +#[test] +fn work_offline_stops_every_network_use_with_a_clear_reason() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + let before = server.requests().len(); + + rig.offline.store(true, Ordering::Relaxed); + assert_eq!(connect(&rig), Err(Failure::Offline)); + assert_eq!( + rig.connectors + .access_token("example", &[]) + .map(|_| ()) + .map_err(|e| e.failure), + Err(Failure::Offline) + ); + let request = HttpRequest::new(Method::Get, server.url("/api/things")); + assert_eq!( + rig.connectors + .request("example", &[], request, 1000) + .map(|_| ()) + .map_err(|e| e.failure), + Err(Failure::Offline) + ); + assert_eq!(server.requests().len(), before, "nothing was sent while offline"); + + // Disconnect still works on this computer, and says the service wasn't told. + let done = rig.connectors.disconnect("example").expect("disconnects"); + assert_eq!(done.revoke, RevokeOutcome::SkippedOffline); + assert_eq!(rig.store.get("OpenNote/example/sam@example.com"), Ok(None)); + assert_eq!(server.requests().len(), before); +} + +#[test] +fn a_no_in_the_browser_connects_nothing() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Refuse("access_denied")); + assert_eq!(connect(&rig), Err(Failure::Denied)); + assert!(server.requests_to("/token").is_empty()); + assert_eq!(rig.state("example"), StateView::NotConnected); + assert!(!rig.connectors.view("example").expect("a view").pending); +} + +#[test] +fn an_answer_with_the_wrong_state_is_never_traded_for_tokens() { + let server = MockServer::start(); + let mut rig = oauth_rig(&server); + rig.restart(Duration::from_millis(300)); + rig.visit(Visit::WrongState); + // The stray answer is refused and the wait goes on until the real one, or the time limit. + assert_eq!(connect(&rig), Err(Failure::TimedOut)); + assert!(server.requests_to("/token").is_empty()); + assert_eq!(rig.state("example"), StateView::NotConnected); +} + +#[test] +fn a_sign_in_nobody_finishes_times_out_and_frees_the_connector() { + let server = MockServer::start(); + let mut rig = oauth_rig(&server); + rig.restart(Duration::from_millis(200)); + rig.visit(Visit::Silent); + assert_eq!(connect(&rig), Err(Failure::TimedOut)); + assert!(!rig.connectors.view("example").expect("a view").pending); + rig.visit(Visit::Approve("late")); + assert_eq!( + connect(&rig), + Ok(connected("sam@example.com")), + "the person can try again" + ); +} + +#[test] +fn a_waiting_sign_in_shows_as_pending_can_be_canceled_and_blocks_a_second() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + rig.visit(Visit::Silent); + let waiting = { + let connectors = rig.connectors.clone(); + thread::spawn(move || { + connectors + .connect("example", ConnectInput::default()) + .map(|_| ()) + .map_err(|e| e.failure) + }) + }; + while !rig.connectors.view("example").expect("a view").pending { + thread::sleep(Duration::from_millis(10)); + } + assert_eq!(connect(&rig), Err(Failure::Busy)); + rig.connectors.cancel("example"); + assert_eq!(waiting.join().expect("finished"), Err(Failure::Canceled)); + assert!(!rig.connectors.view("example").expect("a view").pending); +} + +#[test] +fn a_sign_in_whose_record_cannot_be_saved_leaves_no_credential_behind() { + let server = MockServer::start(); + let rig = oauth_rig(&server); + // A folder where the connections file goes makes every write of it fail. + fs::create_dir_all(rig.dir.path().join(CONNECTIONS_FILE)).expect("makes the folder"); + rig.visit(Visit::Approve("good")); + assert_eq!(connect(&rig), Err(Failure::Storage)); + assert_eq!(rig.store.get("OpenNote/example/sam@example.com"), Ok(None)); + assert_eq!(rig.state("example"), StateView::NotConnected); +} diff --git a/app/src-tauri/src/connectors/tests/transfer.rs b/app/src-tauri/src/connectors/tests/transfer.rs new file mode 100644 index 00000000..f1609dd7 --- /dev/null +++ b/app/src-tauri/src/connectors/tests/transfer.rs @@ -0,0 +1,149 @@ +//! The account calls that move files and state: a download lands in the transfer folder, an upload sends the parts +//! it was given with the connection's token, a resumable upload's `Location` comes back, and names outside the +//! folder are refused. + +use super::*; +use crate::connectors::accounts; + +fn call(rig: &Rig, name: &str, args: serde_json::Value) -> Result { + accounts::call(&rig.connectors, &rig.dir.path().join("accounts"), name, &args) +} + +fn signed_in(server: &MockServer) -> Rig { + let rig = oauth_rig(server); + rig.visit(Visit::Approve("c")); + connect(&rig).expect("connects"); + rig +} + +#[test] +fn a_download_is_written_to_the_transfer_folder_with_the_token() { + let server = MockServer::start(); + let rig = signed_in(&server); + server.route("/files/1", vec![Reply::json(200, json!({ "file": true }))]); + let answer = call( + &rig, + "transfer.download", + json!({ + "connector": "example", "access": ["calendarRead"], "file": "dl/one.json", + "request": { "method": "GET", "url": server.url("/files/1") }, + }), + ) + .expect("downloads"); + assert_eq!(answer["status"], 200); + let path = answer["path"].as_str().expect("a path"); + assert_eq!(fs::read_to_string(path).expect("reads"), r#"{"file":true}"#); + assert_eq!( + server.requests_to("/files/1")[0].header("authorization"), + Some("Bearer access-one") + ); +} + +#[test] +fn a_failed_download_writes_nothing_and_says_what_the_service_said() { + let server = MockServer::start(); + let rig = signed_in(&server); + server.route("/files/2", vec![Reply::json(404, json!({ "error": "gone" }))]); + let answer = call( + &rig, + "transfer.download", + json!({ + "connector": "example", "access": [], "file": "missing.bin", + "request": { "method": "GET", "url": server.url("/files/2") }, + }), + ) + .expect("answers"); + assert_eq!(answer["status"], 404); + assert!(answer["body"].as_str().expect("a body").contains("gone")); + assert!(!Path::new(answer["path"].as_str().expect("a path")).exists()); +} + +#[test] +fn an_upload_sends_the_parts_in_order_and_returns_the_location() { + let server = MockServer::start(); + let rig = signed_in(&server); + call( + &rig, + "temp.write", + json!({ "file": "up.bin", "part": { "kind": "text", "text": "0123456789" } }), + ) + .expect("writes"); + server.route( + "/upload", + vec![Reply { + status: 200, + body: "{}".into(), + headers: vec![("Location".into(), server.url("/session/9"))], + }], + ); + let answer = call( + &rig, + "transfer.upload", + json!({ + "connector": "example", "access": [], "contentType": "multipart/related; boundary=b", + "request": { "method": "POST", "url": server.url("/upload") }, + "parts": [ + { "kind": "text", "text": "--b\r\n" }, + { "kind": "file", "file": "up.bin", "offset": 3, "length": 4 }, + { "kind": "text", "text": "\r\n--b--" }, + ], + }), + ) + .expect("uploads"); + assert_eq!(answer["status"], 200); + assert_eq!(answer["location"], server.url("/session/9")); + let sent = &server.requests_to("/upload")[0]; + assert_eq!(sent.body, "--b\r\n3456\r\n--b--"); + assert_eq!(sent.header("content-type"), Some("multipart/related; boundary=b")); +} + +#[test] +fn the_folder_calls_refuse_names_outside_it_and_state_round_trips() { + let server = MockServer::start(); + let rig = signed_in(&server); + for file in ["../x", "/windows/system32/a.dll", r"a\b"] { + let refused = call(&rig, "temp.read", json!({ "file": file })); + assert!(refused.is_err(), "{file}"); + } + assert_eq!( + call(&rig, "state.get", json!({ "name": "readwise" })).expect("reads"), + serde_json::Value::Null + ); + call( + &rig, + "state.set", + json!({ "name": "readwise", "value": { "cursor": "c1" } }), + ) + .expect("writes"); + assert_eq!( + call(&rig, "state.get", json!({ "name": "readwise" })).expect("reads"), + json!({ "cursor": "c1" }) + ); + call(&rig, "state.delete", json!({ "name": "readwise" })).expect("deletes"); + assert_eq!( + call(&rig, "state.get", json!({ "name": "readwise" })).expect("reads"), + serde_json::Value::Null + ); +} + +#[test] +fn a_client_typed_on_a_card_is_saved_and_clears_needs_setup() { + let server = MockServer::start(); + let def = connector("example", Auth::OAuth(oauth(&server))); + let rig = Rig::new(registry(vec![def]), Arc::new(UreqHttp::default()), None); + assert_eq!(rig.state("example"), StateView::NeedsSetup); + let view = call( + &rig, + "client.set", + json!({ "connector": "example", "clientId": "typed-client-id" }), + ) + .expect("saves"); + assert_eq!(view["state"]["kind"], "notConnected"); + assert_eq!(rig.state("example"), StateView::NotConnected); + let refused = call( + &rig, + "client.set", + json!({ "connector": "example", "clientId": "has space" }), + ); + assert!(refused.is_err()); +} diff --git a/app/src-tauri/src/connectors/view.rs b/app/src-tauri/src/connectors/view.rs new file mode 100644 index 00000000..11837342 --- /dev/null +++ b/app/src-tauri/src/connectors/view.rs @@ -0,0 +1,242 @@ +//! What the Settings page is told about a connector: its name and group, the access it asks for in the registry's +//! terms, and its state. There is no token, no client ID, and no address with a query in any of it. + +use serde::{Deserialize, Serialize}; + +use super::{ + files::Connection, + registry::{AuthKind, ConnectorDef, Group}, +}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AccessView { + pub capability: &'static str, + pub writes: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum StateView { + NotConnected, + /// An OAuth connector that has no client ID yet. + NeedsSetup, + #[serde(rename_all = "camelCase")] + Connected { + account: String, + connected_unix: u64, + }, + /// The service refused to renew the sign-in. The person connects again. + #[serde(rename_all = "camelCase")] + Expired { + account: String, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ConnectorView { + pub id: &'static str, + pub name: &'static str, + pub group: Group, + pub auth: AuthKind, + /// The OpenNote features that use it, by ID. + pub features: Vec<&'static str>, + pub access: Vec, + /// The hosts OpenNote talks to for it. A school connector shows its school's host. + pub hosts: Vec, + pub state: StateView, + /// The school's address, for a connector that has one. + pub base_url: Option, + /// A sign-in is waiting for the browser. + pub pending: bool, + pub last_used_unix: Option, +} + +/// The access, one entry per capability. A capability that any of its scopes changes data counts as a change. +fn access_of(def: &ConnectorDef) -> Vec { + let mut access: Vec = Vec::new(); + for entry in def.access { + match access.iter_mut().find(|seen| seen.capability == entry.capability) { + Some(seen) => seen.writes |= entry.writes, + None => access.push(AccessView { + capability: entry.capability, + writes: entry.writes, + }), + } + } + access +} + +/// What `needs_setup` and `pending` say is passed in, because they come from files and from running sign-ins. +pub fn view_of(def: &ConnectorDef, connection: Option<&Connection>, needs_setup: bool, pending: bool) -> ConnectorView { + let state = match connection { + Some(connection) if connection.expired => StateView::Expired { + account: connection.account.clone(), + }, + Some(connection) => StateView::Connected { + account: connection.account.clone(), + connected_unix: connection.connected_unix, + }, + None if needs_setup => StateView::NeedsSetup, + None => StateView::NotConnected, + }; + let school_host = connection + .and_then(|connection| connection.base_url.as_deref()) + .and_then(|base| url::Url::parse(base).ok()) + .and_then(|base| base.host_str().map(str::to_owned)); + ConnectorView { + id: def.id, + name: def.name, + group: def.group, + auth: def.kind(), + features: def.features.to_vec(), + access: access_of(def), + hosts: def + .hosts + .iter() + .map(|host| (*host).to_owned()) + .chain(school_host) + .collect(), + state, + base_url: connection.and_then(|connection| connection.base_url.clone()), + pending, + last_used_unix: connection.and_then(|connection| connection.last_used_unix), + } +} + +/// What the person typed on the page to connect. A token appears only here, going in. +#[derive(Default, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct ConnectInput { + pub base_url: Option, + pub token: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum RevokeOutcome { + /// There was no connection to remove. + Nothing, + /// The service forgot the sign-in. + Revoked, + /// The service has no way to revoke from here. The sign-in is removed from this computer only. + NotSupported, + /// Work offline was on, so the service wasn't told. + SkippedOffline, + /// The service couldn't be told. The sign-in is removed from this computer only. + Failed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Disconnected { + pub view: ConnectorView, + pub revoke: RevokeOutcome, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::connectors::{files::Holds, registry}; + + fn connection(expired: bool) -> Connection { + Connection { + account: "sam@example.com".into(), + scopes: vec!["scope".into()], + connected_unix: 1_790_000_000, + holds: Holds::Refresh, + base_url: None, + expired, + last_used_unix: None, + } + } + + #[test] + fn states_follow_the_connection_and_the_setup() { + let google = registry::find("google").expect("google"); + let state = |connection: Option<&Connection>, setup| view_of(google, connection, setup, false).state; + assert_eq!(state(None, true), StateView::NeedsSetup); + assert_eq!(state(None, false), StateView::NotConnected); + assert_eq!( + state(Some(&connection(false)), true), + StateView::Connected { + account: "sam@example.com".into(), + connected_unix: 1_790_000_000 + } + ); + assert_eq!( + state(Some(&connection(true)), false), + StateView::Expired { + account: "sam@example.com".into() + } + ); + } + + #[test] + fn the_wire_form_has_no_secret_fields_and_uses_camel_case() { + let view = view_of( + registry::find("google").expect("google"), + Some(&connection(false)), + false, + true, + ); + let json = serde_json::to_value(&view).expect("serializes"); + assert_eq!(json["state"]["kind"], "connected"); + assert_eq!(json["state"]["connectedUnix"], 1_790_000_000); + assert_eq!(json["auth"], "oauth"); + assert_eq!(json["group"], "google"); + assert_eq!(json["pending"], true); + let text = json.to_string().to_lowercase(); + for word in ["token", "secret", "clientid", "verifier"] { + assert!(!text.contains(word), "the view mentions {word}"); + } + } + + #[test] + fn access_is_listed_once_per_capability() { + let view = view_of(registry::find("google").expect("google"), None, false, false); + let capabilities: Vec<_> = view.access.iter().map(|access| access.capability).collect(); + assert_eq!( + capabilities, + [ + "account", + "calendarRead", + "tasksWrite", + "classroomRead", + "driveFiles", + "youtubeRead", + "driveRead", + "youtubeUpload", + "youtubeCaptions", + "driveAppData" + ] + ); + assert!( + view.access + .iter() + .find(|a| a.capability == "tasksWrite") + .expect("tasks") + .writes + ); + assert!( + !view + .access + .iter() + .find(|a| a.capability == "calendarRead") + .expect("calendar") + .writes + ); + } + + #[test] + fn a_school_connector_shows_its_school_host() { + let canvas = registry::find("canvas").expect("canvas"); + assert!(view_of(canvas, None, false, false).hosts.is_empty()); + let mut connected = connection(false); + connected.base_url = Some("https://school.example.edu".into()); + let view = view_of(canvas, Some(&connected), false, false); + assert_eq!(view.hosts, ["school.example.edu"]); + assert_eq!(view.base_url.as_deref(), Some("https://school.example.edu")); + } +} diff --git a/app/src-tauri/src/core_bridge.rs b/app/src-tauri/src/core_bridge.rs index 777849d2..fe9c4cd9 100644 --- a/app/src-tauri/src/core_bridge.rs +++ b/app/src-tauri/src/core_bridge.rs @@ -1,21 +1,24 @@ -//! Phase 3's core behind the page commands (core plan 11.1; Phase 4 PLAN.md section 3.13). Phase 3 planned this -//! bridge for its notes service; WP0 of Phase 4 builds the part pages need, so typed text reaches the core and -//! survives a restart. +// checks-disable-file modifiability: one module whose parts share its state; split it when it grows again + +//! Phase 3's core behind the app's notes and page commands (core plan 11.1; Phase 4 PLAN.md section 3.13). //! -//! Until the storage-backed notes service lands (`storage.core`), the interface's pages come from Phase 2's notes -//! snapshot, whose IDs the core doesn't know. The bridge keeps their content in one notebook of its own, under -//! `%LOCALAPPDATA%\OpenNote\phase4`, and a small map from each interface page ID to the core page that holds it. -//! The core starts the first time a page opens, so start-up doesn't wait for it. +//! One core serves both. The notes commands in [`crate::notes`] keep the tree of notebooks, section groups, +//! sections, and pages in the notes folder that setup chose, in the note format: one folder per notebook with its +//! `notebook.json`, its sections, and its pages (docs/format/README.md section 3). The page commands here open and +//! edit those pages by their IDs, which are the tree's node IDs. Only the core's device-local data, such as the +//! journals and the library's order of notebooks, stays under `%LOCALAPPDATA%\OpenNote\core`. //! -//! WP2 adds the history commands and turns the core's events into `core:*` events for the interface, with the -//! interface's page IDs. +//! The core starts with the first command, which is the tree's first load, and a beta 1 profile is migrated then +//! (see [`crate::notes::migrate`]). use std::{ - collections::{BTreeMap, HashMap}, - fs, io, + borrow::Cow, + collections::HashMap, + fs, + panic::Location, path::{Path, PathBuf}, - sync::{Arc, Mutex, OnceLock, PoisonError}, - time::Duration, + sync::{mpsc, Arc, Mutex, OnceLock, PoisonError}, + time::{Duration, Instant}, }; use opennote_core::{ @@ -24,42 +27,66 @@ use opennote_core::{ resolve::{Edit, TxnRequest}, CoalesceKey, }, + session::notebook::NotebookHandle, session::{ core::{Core, CoreConfig}, events::{CoreEvent, EventSink}, - notebook::{NodePlacement, NotebookHandle, ParentRef}, page::{PageHandle, RestoreResult, TxnAck}, }, - BlockId, ClientId, CoreError, PageId, RevisionId, SectionId, + BlockId, ClientId, CoreError, NotebookId, PageId, RevisionId, }; use serde_json::Value; use tauri::{ipc::Response, AppHandle, Emitter, Manager, State}; use crate::{ ipc::{codes, IpcError, IpcResult}, + notes::NotesState, paths::Paths, - settings::file::write_atomic, + settings::SettingsStore, }; +// Phase 8: search and linking over this core. One command carries every method. +mod held; +pub mod search; +// Beta 5: history retention, Delete history, and the other storage commands over this core. +pub mod admin; + +use held::{called_from, Door, Held, Holding, COMMAND_WAIT, HELD_WARNING}; +pub use held::{BUSY, RESPONSIVE_EVENT, STALLED_EVENT}; + /// How long the exit flush may take before the journals keep the rest for the next start (core plan 9.3). const EXIT_FLUSH: Duration = Duration::from_secs(5); -/// The longest page ID the commands take. The interface's page IDs are far shorter. +/// The longest page ID the commands take. Page IDs are 26 characters. const MAX_PAGE_ID: usize = 128; /// What the interface hears when the core can't start. The cause, which can name local paths, goes to the log. -const NOT_STARTED: &str = "OpenNote couldn't open its pages. Try again in a moment."; +const NOT_STARTED: &str = "OpenNote couldn't open its notes. Try again in a moment."; /// Sends one event to the interface. -type Emit = Box; +pub type Emit = Box; -/// Where the core's page events go: the app, once a page has opened, under the interface's page IDs. It holds the -/// app behind a closure, so unit tests never link the windowing code an `AppHandle` drags in. +/// Where the core's events go: the app, once the interface has called a command. It holds the app behind a +/// closure, so unit tests and the notes harness never link the windowing code an `AppHandle` drags in. Tree +/// changes the core makes on its own, such as a page's title copy after a save, go to a worker that turns them +/// into notes events. #[derive(Default)] -struct Relay { +pub struct Relay { emit: OnceLock, - /// Core page ID to interface page ID. - pages: Mutex>, + /// The search indexer, which hears every core event and every save. + index: OnceLock, + trees: Mutex>>, + /// Hears each saved page's ID, for the local API's webhooks. + saved: OnceLock, +} + +impl Relay { + /// Sends an event to the interface, if it is listening. + pub(crate) fn send(&self, name: &'static str, payload: Value) { + if let Some(emit) = self.emit.get() { + emit(name, payload); + } + } } /// The core's events, as `core:*` events for the interface (core plan 11.2). @@ -67,51 +94,123 @@ struct AppEvents(Arc); impl EventSink for AppEvents { fn emit(&self, event: CoreEvent) { + if let Some(index) = self.0.index.get() { + index.on_event(&event); + } + if let (CoreEvent::Saved { page, .. }, Some(saved)) = (&event, self.0.saved.get()) { + saved(&page.to_string()); + } if let CoreEvent::SaveFailed { .. } = &event { ::log::warn!("The core couldn't save a page: {event:?}"); } - let (name, page) = match &event { - CoreEvent::TxnApplied { page, .. } => ("core:txn-applied", *page), - CoreEvent::ExternalChange { page, .. } => ("core:external-change", *page), - CoreEvent::ReadOnly { page, .. } => ("core:read-only", *page), - CoreEvent::Saved { page, .. } => ("core:saved", *page), - CoreEvent::SaveFailed { page, .. } => ("core:save-failed", *page), - _ => return, - }; - let Some(emit) = self.0.emit.get() else { - return; - }; - let pages = self.0.pages.lock().unwrap_or_else(PoisonError::into_inner); - let Some(ui) = pages.get(&page).cloned() else { + if let CoreEvent::TreeChanged { notebook } = &event { + if let Some(trees) = self.0.trees.lock().unwrap_or_else(PoisonError::into_inner).as_ref() { + let _ = trees.send(*notebook); + } return; + } + let name = match &event { + CoreEvent::TxnApplied { .. } => "core:txn-applied", + CoreEvent::ExternalChange { .. } => "core:external-change", + CoreEvent::ReadOnly { .. } => "core:read-only", + CoreEvent::Saved { .. } => "core:saved", + CoreEvent::SaveFailed { .. } => "core:save-failed", + _ => return, }; - drop(pages); - let Ok(mut payload) = serde_json::to_value(&event) else { + let Ok(payload) = serde_json::to_value(&event) else { return; }; - payload["page"] = Value::String(ui); - emit(name, payload); + self.0.send(name, payload); } } -/// The managed state behind the page commands. +type SavedHook = Box; + +/// The managed state behind the notes and page commands. Cloning shares it, so a command can take it onto a +/// blocking thread (see [`CoreBridge::run`]). +#[derive(Clone)] pub struct CoreBridge { + /// This device's files, `%LOCALAPPDATA%\OpenNote`: the core's own data and a beta 1 profile's files. root: PathBuf, /// The started bridge. A start that failed leaves it empty, so the next command tries again. - state: Mutex>, + state: Arc>>, + /// Where the commands wait for the core; see [`CoreBridge::take_core_within`]. + door: Door, relay: Arc, + search: Arc, + /// The command that holds the core now, for the watchdog and the exit. + held: Held, + /// After how long a held core is reported: [`HELD_WARNING`], shorter in tests. + warn_after: Duration, + /// How long a command waits for the core: [`COMMAND_WAIT`], shorter in tests. + command_wait: Duration, + /// Set once the watchdog runs. It starts with the first command, before the core does, so a start that + /// never returns is reported too. + watched: Arc>, + /// Which opening of each window label is the current one. Kept apart from `state`, so a window that goes can + /// retire its opening at once even while a long command holds the bridge. + windows: Arc, + /// How long page versions are kept, from `settings.editing.history.keep`. It reaches the core when it starts. + retention: Arc>, } -struct Bridge { - core: Core, - notebook: NotebookHandle, - section: SectionId, - map_file: PathBuf, - /// Interface page ID to core page ID. - map: BTreeMap, - /// Open pages by core page and client. - open: HashMap<(PageId, ClientId), PageHandle>, - relay: Arc, +/// One opening of a window: its label and which time that label was opened. A page window closed and opened again +/// keeps its label, so the label alone can't tell the sessions of the old window from those of the new one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WindowOwner { + pub label: String, + pub generation: u64, +} + +/// The current opening of each window label. +#[derive(Default)] +pub struct Windows(Mutex>); + +impl Windows { + fn map(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.0.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// The window labeled `label` as it is open now. + pub fn current(&self, label: &str) -> WindowOwner { + let generation = self.map().get(label).copied().unwrap_or(0); + WindowOwner { + label: label.to_owned(), + generation, + } + } + + /// Whether `owner` is still the open window of its label. + pub fn is_current(&self, owner: &WindowOwner) -> bool { + self.map().get(&owner.label).copied().unwrap_or(0) == owner.generation + } + + /// The window labeled `label` went: returns the opening that went, and the label's next window is a new one. + pub fn retire(&self, label: &str) -> WindowOwner { + let mut map = self.map(); + let generation = map.entry(label.to_owned()).or_insert(0); + let gone = WindowOwner { + label: label.to_owned(), + generation: *generation, + }; + *generation += 1; + gone + } +} + +/// The started core with what the commands keep beside it. +pub struct Bridge { + pub(crate) core: Core, + /// This device's files. + pub(crate) root: PathBuf, + /// Open pages by page and client. + pub(crate) open: HashMap<(PageId, ClientId), PageHandle>, + /// The notebook each open page was opened in. + homes: HashMap<(PageId, ClientId), NotebookId>, + /// The window each open page's client lives in, so a window that goes closes the sessions it left open. + owners: HashMap<(PageId, ClientId), WindowOwner>, + pub(crate) notes: NotesState, + pub(crate) relay: Arc, } fn internal(error: impl std::fmt::Display) -> IpcError { @@ -123,7 +222,7 @@ fn invalid(field: &str, error: impl std::fmt::Display) -> IpcError { } /// A core error with the code the page service reads: an edit's own code, `readOnly`, `notFound`, or internal. -fn core_error(error: CoreError) -> IpcError { +pub(crate) fn core_error(error: CoreError) -> IpcError { let code = match &error { CoreError::Edit(edit) => edit.code(), CoreError::ReadOnly(_) => "readOnly", @@ -137,9 +236,8 @@ fn revision_id(text: &str) -> IpcResult { RevisionId::parse(text).map_err(|error| invalid("revision", error)) } -/// Checks a page argument: a short, plain ID, as the interface's page IDs are, so nothing else becomes a key of the -/// page map. -fn check_page(page: &str) -> IpcResult<()> { +/// Checks a page argument: a short, plain ID, as page IDs are. +pub(crate) fn check_page(page: &str) -> IpcResult<()> { let plain = page .bytes() .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'); @@ -149,110 +247,131 @@ fn check_page(page: &str) -> IpcResult<()> { Ok(()) } -/// The page map, or an empty one when there is no file yet. Any other failure stops the start, so a map that can't -/// be read now is never replaced by one without its pages. -fn read_map(file: &Path) -> Result, IpcError> { - match fs::read_to_string(file) { - Ok(text) => serde_json::from_str(&text) - .map_err(|error| internal(format!("The page map at {} can't be read: {error}", file.display()))), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(BTreeMap::new()), - Err(error) => Err(internal(format!( - "The page map at {} can't be read: {error}", - file.display() - ))), +/// The core's device-local data folder. A beta 1 profile kept it under `phase4`; it moves to `core` once, so the +/// device, the journals, and the library stay as they were. +fn data_dir(root: &Path) -> PathBuf { + let data = root.join("core"); + let old = root.join("phase4").join("core"); + if !data.exists() && old.is_dir() { + if let Err(error) = fs::rename(&old, &data) { + ::log::warn!("Couldn't move the core's data from phase4: {error}"); + return old; + } } + data } impl Bridge { - fn start(root: &Path, relay: Arc) -> Result { - // The map goes first, so a map that can't be read stops the start before the core holds the notebook. - let map_file = root.join("pages.json"); - let map = read_map(&map_file)?; - let config = - CoreConfig::production(root.join("core"), env!("CARGO_PKG_VERSION").to_owned()).map_err(internal)?; - let core = Core::start(config, Arc::new(AppEvents(relay.clone())), None).map_err(internal)?; - let dir = root.join("Pages"); - let notebook = if dir.join("notebook.json").exists() { - core.open_notebook(&dir) - } else { - core.create_notebook(root, "Pages") - } - .map_err(internal)?; - let section = match notebook.tree().sections.first() { - Some(section) => section.id, - None => { - let at = NodePlacement { - parent: ParentRef::Notebook, - before: None, - }; - notebook.create_section("Pages", at).map_err(internal)? - } - }; + fn start(root: &Path, relay: Arc, search: &search::Hub) -> Result { + let config = CoreConfig::production(data_dir(root), env!("CARGO_PKG_VERSION").to_owned()).map_err(internal)?; + // The index starts first, because the core's save hook and events need its handle. + search.spawn(root, &relay)?; + let core = Core::start(config, Arc::new(AppEvents(relay.clone())), search.sink()).map_err(internal)?; + search.attach(&core); Ok(Bridge { core, - notebook, - section, - map_file, - map, + root: root.to_path_buf(), open: HashMap::new(), + homes: HashMap::new(), + owners: HashMap::new(), + notes: NotesState::default(), relay, }) } - /// Lets the core's events for a core page reach the interface under its page ID. - fn relay_page(&self, ui: &str, core: PageId) { - let mut pages = self.relay.pages.lock().unwrap_or_else(PoisonError::into_inner); - pages.entry(core).or_insert_with(|| ui.to_owned()); - } - - /// The core page that holds an interface page, made the first time it opens. The map holds a new page only once - /// the file does, so a page whose mapping wasn't written isn't typed into. - fn core_page(&mut self, page: &str) -> IpcResult { - check_page(page)?; - if let Some(id) = self.map.get(page) { - return PageId::parse(id).map_err(|error| invalid("page", error)); - } - let at = NodePlacement { - parent: ParentRef::Section(self.section), - before: None, - }; - let id = self.notebook.create_page(self.section, at).map_err(internal)?; - let mut next = self.map.clone(); - next.insert(page.to_owned(), id.to_string()); - let json = serde_json::to_vec_pretty(&next).map_err(internal)?; - write_atomic(&self.map_file, &json)?; - self.map = next; - Ok(id) - } - - /// The client's open session of a page, for every command but page_open: only page_open makes a page. - fn open_handle(&self, page: &str, client: &str) -> IpcResult { + /// The client's open session of a page, for every command but page_open. + pub(crate) fn open_handle(&self, page: &str, client: &str) -> IpcResult { check_page(page)?; let not_open = || IpcError::new("notFound", "This page isn't open."); - let id = self - .map - .get(page) - .and_then(|id| PageId::parse(id).ok()) - .ok_or_else(not_open)?; + let id = PageId::parse(page).map_err(|_| not_open())?; let client = ClientId::parse(client).map_err(|error| invalid("client", error))?; self.open.get(&(id, client)).cloned().ok_or_else(not_open) } - /// The client's session of a page, opened and made first when needed (page_open). - fn handle(&mut self, page: &str, client: &str) -> IpcResult { - let id = self.core_page(page)?; - self.relay_page(page, id); + /// The client's session of a page, opened when needed (page_open). The page must be in an open notebook. + pub(crate) fn handle(&mut self, page: &str, client: &str) -> IpcResult { + check_page(page)?; + let missing = || IpcError::new("notFound", "This page isn't in any notebook."); + let id = PageId::parse(page).map_err(|_| missing())?; let client = ClientId::parse(client).map_err(|error| invalid("client", error))?; if let Some(handle) = self.open.get(&(id, client.clone())) { return Ok(handle.clone()); } - let handle = self.notebook.open_page(id, client.clone()).map_err(internal)?; + let (notebook, _) = self.core.find_node(id.0).ok_or_else(missing)?; + let handle = notebook.open_page(id, client.clone()).map_err(core_error)?; + self.homes.insert((id, client.clone()), notebook.id()); self.open.insert((id, client), handle.clone()); Ok(handle) } + /// Closes the client's session of a page, if it has one. The core saves the page in the background. + pub(crate) fn close_handle(&mut self, page: &str, client: &str) -> IpcResult<()> { + let Ok(id) = PageId::parse(page) else { + return Ok(()); + }; + let client = ClientId::parse(client).map_err(|error| invalid("client", error))?; + self.close_client(&(id, client)) + } + + /// [`Bridge::handle`] for a client that lives in the window `owner`. A client belongs to one window: another + /// window that names the same client is refused rather than handed its session, whose sequence, undo, and + /// close would then be shared. A window that went, whose open arrives late, is refused too, as nothing would + /// close the session it opened. + pub(crate) fn handle_in( + &mut self, + page: &str, + client: &str, + owner: &WindowOwner, + windows: &Windows, + ) -> IpcResult { + if !windows.is_current(owner) { + return Err(IpcError::new("notFound", "This window has closed.")); + } + if let (Ok(id), Ok(client_id)) = (PageId::parse(page), ClientId::parse(client)) { + if self.owners.get(&(id, client_id)).is_some_and(|held| held != owner) { + return Err(invalid("client", "This client is open in another window.")); + } + } + let handle = self.handle(page, client)?; + self.owners + .insert((handle.id(), handle.client().clone()), owner.clone()); + Ok(handle) + } + + /// Closes a client's session of a page: the page saves, and the session forgets the client. + fn close_client(&mut self, key: &(PageId, ClientId)) -> IpcResult<()> { + self.homes.remove(key); + self.owners.remove(key); + match self.open.remove(key) { + Some(handle) => handle.close(&key.1).map_err(internal), + None => Ok(()), + } + } + + /// Closes the sessions a window left open. A window that closes goes without unmounting its pages, so their + /// clients would stay in the session for good. Only that opening's sessions close, not those of a window + /// opened since with the same label. Returns how many it closed. + pub(crate) fn close_window(&mut self, window: &WindowOwner) -> usize { + let keys: Vec<(PageId, ClientId)> = self + .owners + .iter() + .filter(|(_, owner)| *owner == window) + .map(|(key, _)| key.clone()) + .collect(); + for key in &keys { + if let Err(error) = self.close_client(key) { + ::log::warn!( + "Couldn't close a page the {} window left open: {}", + window.label, + error.message + ); + } + } + keys.len() + } + fn open_handles(&self, page: &str) -> Vec { - let Some(id) = self.map.get(page).and_then(|id| PageId::parse(id).ok()) else { + let Ok(id) = PageId::parse(page) else { return Vec::new(); }; self.open @@ -261,28 +380,185 @@ impl Bridge { .map(|(_, handle)| handle.clone()) .collect() } + + /// Forgets the open pages of a notebook that closed and opened again, as after its folder was renamed, and + /// returns them so the interface opens them again. + pub(crate) fn forget_notebook_pages(&mut self, notebook: NotebookId) -> Vec { + let gone: Vec<(PageId, ClientId)> = self + .homes + .iter() + .filter(|(_, home)| **home == notebook) + .map(|(key, _)| key.clone()) + .collect(); + for key in &gone { + self.open.remove(key); + self.homes.remove(key); + } + let mut pages: Vec = gone.into_iter().map(|(page, _)| page).collect(); + pages.sort_unstable_by_key(ToString::to_string); + pages.dedup(); + pages + } + + /// Forgets the sessions of pages that left their notebook, such as pages moved to Trash or to another notebook. + /// The interface opens a page again when it needs it. + pub(crate) fn drop_stale_handles(&mut self) { + let core = &self.core; + let homes = &self.homes; + let stale: Vec<(PageId, ClientId)> = self + .open + .keys() + .filter(|key| { + let home = homes.get(*key).copied(); + !core + .find_node(key.0 .0) + .is_some_and(|(notebook, node)| node.is_some() && Some(notebook.id()) == home) + }) + .cloned() + .collect(); + for key in stale { + self.homes.remove(&key); + self.owners.remove(&key); + if let Some(handle) = self.open.remove(&key) { + let _ = handle.close(&key.1); + } + } + } } impl CoreBridge { pub fn new(paths: &Paths) -> CoreBridge { - CoreBridge::at(paths.local.join("phase4")) + CoreBridge::at(paths.local.clone()) } - fn at(root: PathBuf) -> CoreBridge { + /// A bridge whose device-local files are under `root`, for tests and the notes harness. + pub fn at(root: PathBuf) -> CoreBridge { CoreBridge { root, - state: Mutex::new(None), + state: Arc::new(Mutex::new(None)), + door: Arc::default(), relay: Arc::default(), + search: Arc::default(), + windows: Arc::default(), + held: Arc::default(), + warn_after: HELD_WARNING, + command_wait: COMMAND_WAIT, + watched: Arc::default(), + retention: Arc::default(), + } + } + + /// The same bridge, reporting a held core after `warn_after`. For tests. + #[cfg(test)] + pub fn warning_after(mut self, warn_after: Duration) -> CoreBridge { + self.warn_after = warn_after; + self + } + + /// The same bridge, with commands giving up on a held core after `command_wait`. For tests. + #[cfg(test)] + pub fn waiting_at_most(mut self, command_wait: Duration) -> CoreBridge { + self.command_wait = command_wait; + self + } + + /// Runs `work` on a blocking thread and answers when it is done. Every command that touches the core goes + /// through it. The core serves one command at a time, so a command waits for the one before it. That wait + /// used to sit on one of the async runtime's few worker threads. One command that never returned then took + /// every worker in turn, and nothing the interface asked for came back, core or not (beta 4's T2-3). On a + /// blocking thread a wait costs nothing else: unrelated commands keep answering, and the watchdog says what + /// the core is busy with. + pub async fn run( + &self, + work: impl FnOnce(&CoreBridge) -> IpcResult + Send + 'static, + ) -> IpcResult { + let bridge = self.clone(); + tauri::async_runtime::spawn_blocking(move || work(&bridge)) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))? + } + + /// Calls `saved` with each saved page's ID. The first call wins. + pub(crate) fn on_saved(&self, saved: Box) { + let _ = self.relay.saved.set(saved); + } + + /// Sends the core's events through `emit`. The first call wins. + pub fn listen(&self, emit: Emit) { + let _ = self.relay.emit.set(emit); + } + + /// Lets the core's events reach the app's window. + fn listen_app(&self, app: &AppHandle) { + if self.relay.emit.get().is_some() { + return; } + let app = app.clone(); + self.listen(Box::new(move |name, payload| { + if let Err(error) = app.emit(name, payload) { + ::log::warn!("Couldn't send {name} to the interface: {error}"); + } + })); + } + + /// The windows' openings, which page_open records each session's owner by. + pub fn windows(&self) -> &Windows { + &self.windows + } + + /// A window went: retires its opening at once, so nothing opens in its name any more, and returns it for + /// [`CoreBridge::close_window`], which may wait for the bridge. + pub fn window_went(&self, label: &str) -> WindowOwner { + self.windows.retire(label) + } + + /// Closes the page sessions of a window that went, if the core has started. See [`Bridge::close_window`]. + pub fn close_window(&self, window: &WindowOwner) -> usize { + let Some(mut state) = self.take_core_within(self.command_wait) else { + ::log::warn!( + "Couldn't close the pages the {} window left open: the core is busy ({}).", + window.label, + self.holder() + ); + return 0; + }; + state.as_mut().map_or(0, |bridge| bridge.close_window(window)) } /// Runs `work` on the bridge, starting the core first if it hasn't started. A start that fails is logged and - /// tried again by the next command. - fn with(&self, work: impl FnOnce(&mut Bridge) -> IpcResult) -> IpcResult { - let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + /// tried again by the next command. The watchdog knows the command by where it was called from. + #[track_caller] + pub fn with(&self, work: impl FnOnce(&mut Bridge) -> IpcResult) -> IpcResult { + self.with_named(called_from(Location::caller()), work) + } + + /// [`CoreBridge::with`], with the command's name for the watchdog should it hold the core too long. + pub fn with_named( + &self, + what: impl Into>, + work: impl FnOnce(&mut Bridge) -> IpcResult, + ) -> IpcResult { + self.watched.get_or_init(|| self.watch_held()); + let what = what.into(); + let Some(mut state) = self.take_core_within(self.command_wait) else { + let holder = self.holder(); + ::log::error!("{what} gave up waiting for the core: {holder}"); + return Err(IpcError::new( + BUSY, + format!("OpenNote's core is busy ({holder}). Try again in a moment."), + )); + }; + *self.held.lock().unwrap_or_else(PoisonError::into_inner) = Some((Instant::now(), what)); + let _holding = Holding(self.held.clone()); if state.is_none() { - match Bridge::start(&self.root, self.relay.clone()) { - Ok(bridge) => *state = Some(bridge), + match Bridge::start(&self.root, self.relay.clone(), &self.search) { + Ok(bridge) => { + bridge + .core + .set_retention(*self.retention.lock().unwrap_or_else(PoisonError::into_inner)); + *state = Some(bridge); + self.watch_trees(); + } Err(error) => { ::log::error!("The core couldn't start: {}", error.message); return Err(IpcError::new(codes::INTERNAL, NOT_STARTED)); @@ -295,18 +571,138 @@ impl CoreBridge { } } + /// Runs a notes command in the notes folder `folder`, after the folder's notebooks are open and a beta 1 + /// profile is migrated, and sends the notes events for what it changed. + #[track_caller] + pub fn notes(&self, folder: Option, work: impl FnOnce(&mut Bridge) -> IpcResult) -> IpcResult { + self.notes_named(called_from(Location::caller()), folder, work) + } + + /// [`CoreBridge::notes`], with the command's name for the watchdog. + pub fn notes_named( + &self, + what: impl Into>, + folder: Option, + work: impl FnOnce(&mut Bridge) -> IpcResult, + ) -> IpcResult { + self.with_named(what, |bridge| { + bridge.use_folder(folder); + let result = work(bridge); + bridge.drop_stale_handles(); + // Notebooks this command opened or closed reach the index now, with the tree events. + self.search.notebooks_changed(&bridge.core); + bridge.send_tree_events(); + result + }) + } + + /// Starts the worker that turns tree changes the core makes on its own into notes events. + fn watch_trees(&self) { + let (sender, receiver) = mpsc::channel::(); + *self.relay.trees.lock().unwrap_or_else(PoisonError::into_inner) = Some(sender); + let state = Arc::downgrade(&self.state); + let spawned = std::thread::Builder::new() + .name("opennote-tree-events".into()) + .spawn(move || { + while receiver.recv().is_ok() { + // A burst of changes needs one pass. + while receiver.try_recv().is_ok() {} + let Some(state) = state.upgrade() else { + return; + }; + let mut state = state.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(bridge) = state.as_mut() { + bridge.send_tree_events(); + } + } + }); + if let Err(error) = spawned { + ::log::warn!("Couldn't start the tree events worker: {error}"); + } + } + + /// The open notebooks, which are the notebook folders in the notes folder. It starts the core and opens the + /// library if that has not happened. The answer is empty when the core can't start. The self-check reads the + /// notebooks through their own `verify`. + pub fn notebooks(&self) -> Vec { + self.notes(None, |bridge| Ok(bridge.core.notebooks())) + .unwrap_or_default() + .into_iter() + .filter(|notebook| !notebook.is_backup()) + .collect() + } + + /// Runs a search the way the interface's search box does, for the local API. + pub(crate) fn search_query(&self, args: Value) -> IpcResult { + self.with(|_| Ok(()))?; + self.search.call("query", args) + } + /// Saves every page and stops the core. The app calls it on exit. pub fn shutdown(&self) { - let state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + self.shutdown_within(EXIT_FLUSH); + } + + /// [`CoreBridge::shutdown`], waiting at most `timeout` for a command that holds the core. A command that + /// never returns must not keep the window open with nothing to show for it: the exit goes on without the + /// final save, and the journals keep every edit for the next start (spec 20), except those of pages in unlocked + /// protected sections, whose journals live only in memory (ADR 0042). + pub fn shutdown_within(&self, timeout: Duration) { + *self.relay.trees.lock().unwrap_or_else(PoisonError::into_inner) = None; + let Some(state) = self.take_core_within(timeout) else { + ::log::error!( + "The core is still busy at exit ({}), so OpenNote exits without a final save. \ + The journals keep the edits for the next start, except the unsaved edits of pages in \ + unlocked protected sections, whose journals are only in memory and are lost (ADR 0042).", + self.holder() + ); + return; + }; if let Some(bridge) = state.as_ref() { if let Err(error) = bridge.core.flush_all(EXIT_FLUSH) { ::log::error!("Couldn't save the open pages: {error}"); } + // The index follows the saves just flushed, then closes before the core goes. + self.search.close(); bridge.core.shutdown(EXIT_FLUSH); } } } +/// The notes folder that settings name, if setup has chosen one. A test build asked for the sample library uses +/// the folder setup proposes until then. +pub(crate) fn notes_folder(app: &AppHandle) -> Option { + let chosen = app + .state::() + .get() + .storage + .notes_folder + .filter(|folder| !folder.is_empty()) + .map(PathBuf::from); + chosen.or_else(|| crate::notes::migrate::seeding().then(|| app.state::().documents.join("OpenNote"))) +} + +/// Runs a notes command for the app: its events go to the window, in the notes folder from settings. +#[track_caller] +pub(crate) fn run_notes( + app: &AppHandle, + bridge: &CoreBridge, + work: impl FnOnce(&mut Bridge) -> IpcResult, +) -> IpcResult { + run_notes_named(app, bridge, called_from(Location::caller()), work) +} + +/// [`run_notes`], with the command's name for the watchdog. +pub(crate) fn run_notes_named( + app: &AppHandle, + bridge: &CoreBridge, + what: impl Into>, + work: impl FnOnce(&mut Bridge) -> IpcResult, +) -> IpcResult { + bridge.listen_app(app); + bridge.notes_named(what, notes_folder(app), work) +} + /// An open page of the core, for Phase 4's image commands (P3-8). pub fn page_handle(app: &AppHandle, page: PageId) -> Option { let bridge = app.state::(); @@ -321,14 +717,9 @@ pub fn page_handle(app: &AppHandle, page: PageId) -> Option { } } -/// The core page that holds an interface page, if it was ever opened. -pub fn core_page_id(app: &AppHandle, page: &str) -> Option { - let bridge = app.state::(); - let state = bridge.state.lock().unwrap_or_else(PoisonError::into_inner); - match state.as_ref() { - Some(bridge) => bridge.map.get(page).and_then(|id| PageId::parse(id).ok()), - _ => None, - } +/// The core page an interface page ID names: they are the same ID. +pub fn core_page_id(_app: &AppHandle, page: &str) -> Option { + PageId::parse(page).ok() } fn edit_error(error: opennote_core::EditError) -> IpcError { @@ -339,23 +730,23 @@ fn edit_error(error: opennote_core::EditError) -> IpcError { #[tauri::command] pub async fn page_open( app: AppHandle, + window: tauri::WebviewWindow, bridge: State<'_, CoreBridge>, page: String, client: String, viewport: Option, ) -> IpcResult { - bridge.relay.emit.get_or_init(|| { - Box::new(move |name, payload| { - if let Err(error) = app.emit(name, payload) { - ::log::warn!("Couldn't send {name} to the interface: {error}"); - } + bridge + .run(move |bridge| { + let windows = bridge.windows(); + let owner = windows.current(window.label()); + run_notes_named(&app, bridge, "page_open", |bridge| { + let handle = bridge.handle_in(&page, &client, &owner, windows)?; + let envelope = handle.envelope(viewport).map_err(internal)?; + Ok(Response::new(envelope.bytes)) + }) }) - }); - bridge.with(|bridge| { - let handle = bridge.handle(&page, &client)?; - let envelope = handle.envelope(viewport).map_err(internal)?; - Ok(Response::new(envelope.bytes)) - }) + .await } /// One transaction from the interface, in the client's sequence (core plan 11.4). @@ -369,69 +760,84 @@ pub async fn page_apply( ui: Option, edits: Vec, ) -> IpcResult { - bridge.with(|bridge| { - let handle = bridge.open_handle(&page, &client)?; - let request = TxnRequest { - page: handle.id(), - client: handle.client().clone(), - client_seq, - coalesce, - ui, - edits, - }; - handle.apply(request).map_err(edit_error) - }) + bridge + .run(move |bridge| { + bridge.with_named("page_apply", |bridge| { + let handle = bridge.open_handle(&page, &client)?; + let request = TxnRequest { + page: handle.id(), + client: handle.client().clone(), + client_seq, + coalesce, + ui, + edits, + }; + handle.apply(request).map_err(edit_error) + }) + }) + .await } /// The applied-changes frame of the undone step, or no bytes when there is nothing to undo. #[tauri::command] pub async fn page_undo(bridge: State<'_, CoreBridge>, page: String, client: String) -> IpcResult { - bridge.with(|bridge| { - let handle = bridge.open_handle(&page, &client)?; - let frame = handle.undo(handle.client()).map_err(edit_error)?; - Ok(Response::new(frame.map(|frame| frame.bytes).unwrap_or_default())) - }) + bridge + .run(move |bridge| { + bridge.with_named("page_undo", |bridge| { + let handle = bridge.open_handle(&page, &client)?; + let frame = handle.undo(handle.client()).map_err(edit_error)?; + Ok(Response::new(frame.map(|frame| frame.bytes).unwrap_or_default())) + }) + }) + .await } #[tauri::command] pub async fn page_redo(bridge: State<'_, CoreBridge>, page: String, client: String) -> IpcResult { - bridge.with(|bridge| { - let handle = bridge.open_handle(&page, &client)?; - let frame = handle.redo(handle.client()).map_err(edit_error)?; - Ok(Response::new(frame.map(|frame| frame.bytes).unwrap_or_default())) - }) + bridge + .run(move |bridge| { + bridge.with_named("page_redo", |bridge| { + let handle = bridge.open_handle(&page, &client)?; + let frame = handle.redo(handle.client()).map_err(edit_error)?; + Ok(Response::new(frame.map(|frame| frame.bytes).unwrap_or_default())) + }) + }) + .await } /// Saves the page now, as Ctrl+S does. #[tauri::command] pub async fn page_save_now(bridge: State<'_, CoreBridge>, page: String) -> IpcResult<()> { - bridge.with(|bridge| { - for handle in bridge.open_handles(&page) { - handle.save_now().map_err(internal)?; - } - Ok(()) - }) + bridge + .run(move |bridge| { + bridge.with_named("page_save_now", |bridge| { + for handle in bridge.open_handles(&page) { + handle.save_now().map_err(internal)?; + } + Ok(()) + }) + }) + .await } /// Closes the client's session of the page. The core saves it in the background. #[tauri::command] pub async fn page_close(bridge: State<'_, CoreBridge>, page: String, client: String) -> IpcResult<()> { - bridge.with(|bridge| { - let Some(id) = bridge.map.get(&page).and_then(|id| PageId::parse(id).ok()) else { - return Ok(()); - }; - let client = ClientId::parse(&client).map_err(|error| invalid("client", error))?; - if let Some(handle) = bridge.open.remove(&(id, client.clone())) { - handle.close(&client).map_err(internal)?; - } - Ok(()) - }) + bridge + .run(move |bridge| bridge.with_named("page_close", |bridge| bridge.close_handle(&page, &client))) + .await } /// The page's saved versions, newest first (core plan 8). #[tauri::command] pub async fn history_list(bridge: State<'_, CoreBridge>, page: String, client: String) -> IpcResult> { - bridge.with(|bridge| bridge.open_handle(&page, &client)?.history().map_err(core_error)) + bridge + .run(move |bridge| { + bridge.with_named("history_list", |bridge| { + bridge.open_handle(&page, &client)?.history().map_err(core_error) + }) + }) + .await } /// A saved version as a read-only page envelope. @@ -442,28 +848,37 @@ pub async fn history_open( client: String, revision: String, ) -> IpcResult { - bridge.with(|bridge| { - let handle = bridge.open_handle(&page, &client)?; - let envelope = handle.open_version(revision_id(&revision)?).map_err(core_error)?; - Ok(Response::new(envelope.bytes)) - }) + bridge + .run(move |bridge| { + bridge.with_named("history_open", |bridge| { + let handle = bridge.open_handle(&page, &client)?; + let envelope = handle.open_version(revision_id(&revision)?).map_err(core_error)?; + Ok(Response::new(envelope.bytes)) + }) + }) + .await } -/// Restores a version in place, or as a new page. +/// Restores a version in place, or as a new page, which the notes events then add to the tree. #[tauri::command] pub async fn history_restore( + app: AppHandle, bridge: State<'_, CoreBridge>, page: String, client: String, revision: String, as_copy: bool, ) -> IpcResult { - bridge.with(|bridge| { - let handle = bridge.open_handle(&page, &client)?; - handle - .restore_version(revision_id(&revision)?, as_copy) - .map_err(core_error) - }) + bridge + .run(move |bridge| { + run_notes_named(&app, bridge, "history_restore", |bridge| { + let handle = bridge.open_handle(&page, &client)?; + handle + .restore_version(revision_id(&revision)?, as_copy) + .map_err(core_error) + }) + }) + .await } /// Brings blocks back from a version as one transaction of the client (P3-7). @@ -476,12 +891,16 @@ pub async fn history_restore_blocks( revision: String, blocks: Vec, ) -> IpcResult { - bridge.with(|bridge| { - let handle = bridge.open_handle(&page, &client)?; - handle - .restore_blocks(client_seq, revision_id(&revision)?, &blocks) - .map_err(edit_error) - }) + bridge + .run(move |bridge| { + bridge.with_named("history_restore_blocks", |bridge| { + let handle = bridge.open_handle(&page, &client)?; + handle + .restore_blocks(client_seq, revision_id(&revision)?, &blocks) + .map_err(edit_error) + }) + }) + .await } /// Names a version, or marks it to keep forever. @@ -494,163 +913,18 @@ pub async fn history_name( name: Option, keep: bool, ) -> IpcResult<()> { - bridge.with(|bridge| { - let handle = bridge.open_handle(&page, &client)?; - handle - .name_version(revision_id(&revision)?, name, keep) - .map_err(core_error) - }) + bridge + .run(move |bridge| { + bridge.with_named("history_name", |bridge| { + let handle = bridge.open_handle(&page, &client)?; + handle + .name_version(revision_id(&revision)?, name, keep) + .map_err(core_error) + }) + }) + .await } #[cfg(test)] -mod tests { - use super::*; - use opennote_core::ops::resolve::Edit; - - fn text_of(handle: &PageHandle) -> String { - let envelope = handle.envelope(None).expect("an envelope"); - let decoded = opennote_core::wire::envelope::decode(&envelope.bytes).expect("decodes"); - let page: Value = serde_json::from_slice(decoded.page_json).expect("page JSON"); - page["blocks"][0]["data"]["markdown"] - .as_str() - .unwrap_or_default() - .to_owned() - } - - fn insert(handle: &PageHandle, seq: u64, markdown: &str) { - let block: opennote_core::ops::resolve::NewBlock = serde_json::from_value(serde_json::json!({ - "id": "01k6f00000000000000000b001", "type": "text", "data": { "markdown": markdown } - })) - .expect("a new block"); - let request = TxnRequest { - page: handle.id(), - client: handle.client().clone(), - client_seq: seq, - coalesce: None, - ui: None, - edits: vec![Edit::InsertBlock { - block, - after: None, - before: None, - }], - }; - handle.apply(request).expect("applies"); - } - - #[test] - fn typed_text_survives_a_restart_of_the_core() { - let dir = tempfile::tempdir().expect("a temp folder"); - let bridge = CoreBridge::at(dir.path().to_owned()); - bridge - .with(|bridge| { - let handle = bridge.handle("p-mitosis", "main-1")?; - insert(&handle, 1, "Cells divide"); - Ok(()) - }) - .expect("the first run"); - bridge.shutdown(); - let again = CoreBridge::at(dir.path().to_owned()); - let text = again - .with(|bridge| Ok(text_of(&bridge.handle("p-mitosis", "main-1")?))) - .expect("the second run"); - again.shutdown(); - assert_eq!(text, "Cells divide"); - } - - #[test] - fn each_interface_page_gets_its_own_core_page() { - let dir = tempfile::tempdir().expect("a temp folder"); - let bridge = CoreBridge::at(dir.path().to_owned()); - let (one, two, again) = bridge - .with(|bridge| Ok((bridge.core_page("a")?, bridge.core_page("b")?, bridge.core_page("a")?))) - .expect("pages"); - bridge.shutdown(); - assert_ne!(one, two); - assert_eq!(one, again); - } - - #[test] - fn events_name_the_interface_page_and_history_keeps_closed_versions() { - let dir = tempfile::tempdir().expect("a temp folder"); - let bridge = CoreBridge::at(dir.path().to_owned()); - let (core, versions) = bridge - .with(|bridge| { - let handle = bridge.handle("p-history", "main-1")?; - insert(&handle, 1, "Kept when closed"); - let client = handle.client().clone(); - bridge.open.remove(&(handle.id(), client.clone())); - handle.close(&client).map_err(internal)?; - let again = bridge.handle("p-history", "main-1")?; - Ok((again.id(), again.history().map_err(core_error)?)) - }) - .expect("a saved page"); - let named = bridge.relay.pages.lock().expect("the relay").get(&core).cloned(); - bridge.shutdown(); - assert_eq!(named.as_deref(), Some("p-history")); - assert!(!versions.is_empty()); - assert_eq!(core_error(CoreError::NotFound("version".into())).code, "notFound"); - } - - #[test] - fn an_unreadable_page_map_stops_the_start_and_is_never_replaced() { - let dir = tempfile::tempdir().expect("a temp folder"); - let map_file = dir.path().join("pages.json"); - fs::write(&map_file, b"{ not json").expect("a damaged map"); - let bridge = CoreBridge::at(dir.path().to_owned()); - let error = bridge - .with(|bridge| bridge.handle("p-damaged", "main-1").map(|_| ())) - .expect_err("the start fails"); - assert_eq!(error.code, codes::INTERNAL); - assert!(!error.message.contains(&dir.path().display().to_string())); - assert_eq!(fs::read(&map_file).expect("the map"), b"{ not json"); - // A start that failed is tried again by the next command, once the file can be read. - fs::write(&map_file, br#"{ "p-kept": "01k6f00000000000000000p001" }"#).expect("a mended map"); - let kept = bridge - .with(|bridge| Ok(bridge.map.get("p-kept").cloned())) - .expect("the second start"); - bridge.shutdown(); - assert_eq!(kept.as_deref(), Some("01k6f00000000000000000p001")); - } - - #[test] - fn a_page_is_mapped_only_once_its_map_is_written() { - let dir = tempfile::tempdir().expect("a temp folder"); - let bridge = CoreBridge::at(dir.path().to_owned()); - let (first, second, mapped) = bridge - .with(|bridge| { - // A folder where the map goes makes every write fail. - bridge.map_file = dir.path().join("blocked"); - fs::create_dir(&bridge.map_file).map_err(internal)?; - let first = bridge.core_page("p-new").is_err(); - let second = bridge.core_page("p-new").is_err(); - Ok((first, second, bridge.map.contains_key("p-new"))) - }) - .expect("the bridge"); - bridge.shutdown(); - assert!(first && second); - assert!(!mapped); - } - - #[test] - fn only_page_open_makes_a_page_and_page_ids_are_checked() { - let dir = tempfile::tempdir().expect("a temp folder"); - let bridge = CoreBridge::at(dir.path().to_owned()); - let (unopened, opened) = bridge - .with(|bridge| { - let unopened = bridge.open_handle("p-never-opened", "main-1").map(|_| ()).unwrap_err(); - bridge.handle("p-opened", "main-1")?; - let opened = bridge.open_handle("p-opened", "main-1").is_ok(); - Ok((unopened, opened)) - }) - .expect("the bridge"); - let map = fs::read_to_string(dir.path().join("pages.json")).expect("the map"); - bridge.shutdown(); - assert_eq!(unopened.code, "notFound"); - assert!(opened); - assert!(!map.contains("p-never-opened")); - for bad in ["", "a/b", "..", "p 1", &"p".repeat(MAX_PAGE_ID + 1)] { - assert_eq!(check_page(bad).expect_err(bad).code, codes::INVALID); - } - assert!(check_page("01k6f00000000000000000p001").is_ok()); - } -} +#[path = "core_bridge_tests.rs"] +mod tests; diff --git a/app/src-tauri/src/core_bridge/admin.rs b/app/src-tauri/src/core_bridge/admin.rs new file mode 100644 index 00000000..d64d2e3f --- /dev/null +++ b/app/src-tauri/src/core_bridge/admin.rs @@ -0,0 +1,249 @@ +//! Storage administration over the core: how long history is kept (`settings.editing.history.keep`) and the +//! commands that delete history. + +use std::sync::PoisonError; + +use opennote_core::{ + session::notebook::{HistoryDeleted, HistoryScope, NotebookHandle}, + store::history::Retention, + CoreError, PageId, SectionId, +}; +use serde::Deserialize; +use tauri::State; + +use super::{core_error, CoreBridge}; +use crate::ipc::{IpcError, IpcResult}; +use crate::settings::editing::HistoryKeep; + +/// Which pages Delete history reaches, as the interface names them: `{ kind, id }`, where a notebook's `id` is the +/// notebook's own ID. +#[derive(Clone, Debug, Deserialize)] +pub struct ScopeArg { + pub kind: String, + pub id: String, +} + +impl ScopeArg { + /// The core's scope. Every ID is parsed, so nothing but an ID reaches the core. + fn scope(&self) -> IpcResult { + let bad = || IpcError::invalid("scope", "isn't a page, section, or notebook"); + match self.kind.as_str() { + "page" => PageId::parse(&self.id).map(HistoryScope::Page).map_err(|_| bad()), + "section" => SectionId::parse(&self.id).map(HistoryScope::Section).map_err(|_| bad()), + "notebook" if !self.id.is_empty() && self.id.len() <= 64 => Ok(HistoryScope::Notebook), + _ => Err(bad()), + } + } +} + +/// The core's retention for the history setting (spec 13.3). +pub fn retention_for(keep: HistoryKeep) -> Retention { + match keep { + HistoryKeep::Month => Retention::Days30, + HistoryKeep::Year => Retention::Year1, + HistoryKeep::Forever => Retention::Forever, + } +} + +impl CoreBridge { + /// Sets how long page versions are kept. The setting is held until the core starts, and reaches a started + /// core at once. Versions are thinned to it page by page, when each page is tidied after it closes. + pub fn set_retention(&self, keep: HistoryKeep) { + let retention = retention_for(keep); + *self.retention.lock().unwrap_or_else(PoisonError::into_inner) = retention; + let state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(bridge) = state.as_ref() { + bridge.core.set_retention(retention); + } + } + + /// Forgets what search keeps beside the index for pages deleted for good: the words read from their pictures, + /// handwriting, and recordings, which include transcripts. + pub fn forget_pages(&self, pages: &[opennote_core::PageId]) { + self.search.forget_pages(pages); + } + + /// Delete history (A2-3, A4-12): deletes the saved versions of a page, a section, or a notebook. With + /// `keep_named`, named versions and versions marked to keep stay. Pages' current content is untouched. A page or + /// section is looked for in `notebook` first, then in the other open notebooks, since the interface may not know + /// which notebook holds it. + pub fn delete_history(&self, notebook: &str, scope: &ScopeArg, keep_named: bool) -> IpcResult { + let parsed = scope.scope()?; + let notebooks = self.notebooks(); + let named = |id: &str| notebooks.iter().find(|nb| nb.id().to_string() == id); + if parsed == HistoryScope::Notebook { + let target = named(&scope.id).ok_or_else(|| IpcError::new("notFound", "That notebook isn't open."))?; + return target.delete_history(parsed, keep_named).map_err(core_error); + } + let mut order: Vec<&NotebookHandle> = named(notebook).into_iter().collect(); + order.extend(notebooks.iter().filter(|nb| nb.id().to_string() != notebook)); + for candidate in order { + match candidate.delete_history(parsed, keep_named) { + Err(CoreError::NotFound(_)) => continue, + other => return other.map_err(core_error), + } + } + Err(IpcError::new( + "notFound", + "That page or section isn't in an open notebook.", + )) + } + + /// How long page versions are kept, as the bridge holds it. + pub fn retention(&self) -> Retention { + *self.retention.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +/// Delete history for a page, a section, or a notebook (`history_delete`). +#[tauri::command] +pub async fn history_delete( + bridge: State<'_, CoreBridge>, + notebook: String, + scope: ScopeArg, + keep_named: bool, +) -> IpcResult { + bridge.delete_history(¬ebook, &scope, keep_named) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core_bridge::edit_error; + + #[test] + fn each_setting_maps_to_the_cores_retention() { + assert_eq!(retention_for(HistoryKeep::Month), Retention::Days30); + assert_eq!(retention_for(HistoryKeep::Year), Retention::Year1); + assert_eq!(retention_for(HistoryKeep::Forever), Retention::Forever); + assert_eq!(retention_for(HistoryKeep::default()), Retention::default()); + } + + #[test] + fn the_setting_reaches_the_core_before_and_after_it_starts() { + let dir = tempfile::tempdir().expect("a temp folder"); + let bridge = CoreBridge::at(dir.path().join("local")); + bridge.set_retention(HistoryKeep::Month); + let started = bridge + .with(|bridge| Ok(bridge.core.retention())) + .expect("the core starts"); + assert_eq!( + started, + Retention::Days30, + "a setting made before start-up is applied when the core starts" + ); + bridge.set_retention(HistoryKeep::Forever); + let changed = bridge + .with(|bridge| Ok(bridge.core.retention())) + .expect("the core runs"); + assert_eq!(changed, Retention::Forever, "a change reaches the running core"); + bridge.shutdown(); + } + + /// A notebook, a section, and a page made through the notes commands, with two saved versions of the page, + /// the first one named. Returns the notebook, section, and page IDs. + fn a_page_with_versions(bridge: &CoreBridge, notes: &std::path::Path) -> (String, String, String) { + use opennote_core::ops::resolve::{Edit, NewBlock, TxnRequest}; + use serde_json::json; + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + let mut create = |kind: &str, parent: Option| { + let input = json!({ "kind": kind, "placement": { "parentId": parent, "beforeId": null } }); + let node = bridge.dispatch("notes_create", &json!({ "input": input }))?; + Ok::<_, IpcError>(node["id"].as_str().unwrap_or_default().to_owned()) + }; + let notebook = create("notebook", None)?; + let section = create("section", Some(notebook.clone()))?; + let page = create("page", Some(section.clone()))?; + let handle = bridge.handle(&page, "main-1")?; + for (seq, id) in [(1_u64, "01k6f00000000000000000b001"), (2, "01k6f00000000000000000b002")] { + let block: NewBlock = serde_json::from_value(json!({ + "id": id, "type": "text", "data": { "markdown": format!("Version {seq}") } + })) + .map_err(|e| IpcError::new("io", e.to_string()))?; + let request = TxnRequest { + page: handle.id(), + client: handle.client().clone(), + client_seq: seq, + coalesce: None, + ui: None, + edits: vec![Edit::InsertBlock { + block, + after: None, + before: None, + }], + }; + handle.apply(request).map_err(edit_error)?; + if seq == 1 { + let saved = handle.save_now().map_err(core_error)?; + handle + .name_version(saved.revision, Some("Named".into()), false) + .map_err(core_error)?; + } + } + // The first edit keeps the page as it was, as an unnamed version. Closing the page saves the second edit as another. + let client = handle.client().clone(); + handle.close(&client).map_err(core_error)?; + Ok((notebook, section, page)) + }) + .expect("a page with versions") + } + + fn versions(bridge: &CoreBridge, page: &str) -> Vec> { + bridge + .with(|bridge| { + let handle = bridge.open_handle(page, "main-1")?; + Ok(handle + .history() + .map_err(core_error)? + .into_iter() + .map(|v| v.name) + .collect()) + }) + .expect("the history") + } + + #[test] + fn delete_history_keeps_named_versions_when_asked_and_checks_its_scope() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let (notebook, section, page) = a_page_with_versions(&bridge, ¬es); + // The first edit keeps the page as it was, then the named save, then the close. + assert_eq!(versions(&bridge, &page).len(), 3, "three saved versions"); + + let scope = |kind: &str, id: &str| ScopeArg { + kind: kind.into(), + id: id.into(), + }; + // The page is found even when the interface names no notebook. + let deleted = bridge.delete_history("", &scope("page", &page), true).expect("deletes"); + assert_eq!((deleted.pages, deleted.versions), (1, 2)); + assert_eq!( + versions(&bridge, &page), + [Some("Named".to_owned())], + "the named version stays" + ); + + let deleted = bridge + .delete_history(¬ebook, &scope("section", §ion), false) + .expect("deletes"); + assert_eq!(deleted.versions, 1); + assert!(versions(&bridge, &page).is_empty(), "nothing is left"); + assert!(bridge + .delete_history(¬ebook, &scope("notebook", ¬ebook), true) + .is_ok()); + + for bad in [scope("page", "..\\x"), scope("folder", &page), scope("notebook", "")] { + assert!(bridge.delete_history(¬ebook, &bad, true).is_err(), "{bad:?}"); + } + assert_eq!( + bridge + .delete_history(¬ebook, &scope("notebook", "01k6f00000000000000000n999"), true) + .unwrap_err() + .code, + "notFound" + ); + bridge.shutdown(); + } +} diff --git a/app/src-tauri/src/core_bridge/held.rs b/app/src-tauri/src/core_bridge/held.rs new file mode 100644 index 00000000..bb2f9a8c --- /dev/null +++ b/app/src-tauri/src/core_bridge/held.rs @@ -0,0 +1,179 @@ +//! How a command takes the core and how long it waits for one that holds it, and the watchdog that names a +//! command holding it too long. The core serves one command at a time; beta 4's T2-3 was one that never +//! returned, with nothing to say so. This module bounds every wait for the core and reports the holder. + +use std::{ + borrow::Cow, + panic::Location, + sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError}, + time::{Duration, Instant}, +}; + +use super::{Bridge, CoreBridge}; + +/// How long one command may hold the core before the log and the interface hear which one, and how often they +/// hear it again while it goes on. The core serves one command at a time, so a command that never returns stops +/// every later one; beta 4's T2-3 was such a stop, and nothing said what the core was doing. Now the watchdog +/// names the holder, the interface shows that the core isn't responding, and the exit goes on without it. +pub(super) const HELD_WARNING: Duration = Duration::from_secs(10); + +/// How long a command waits for the core before it fails as [`BUSY`] instead of joining the queue behind a +/// command that never returns. The interface sends its commands over the webview's IPC channel, which carries +/// only a handful at a time: once that many wait on the core, nothing else gets through, not even the window's +/// Close or a log line (beta 4's T2-3). A command that fails after this wait keeps the channel open, and the +/// interface says the core isn't responding instead of saying "Saving" for the rest of the session. It is longer +/// than the core's journal `OPEN_TIMEOUT` (10 s, journal_thread.rs), the one bounded wait a command makes on +/// another thread while it holds the core, so a stuck journal open gives up before the commands behind it do. +/// +/// The channel's limit stays: the interface keeps one edit in flight per page, but with many pages open their +/// edits and the tree's and search's commands can still fill the channel for one such wait while the core is +/// held. The watchdog reports the holder after [`HELD_WARNING`] all the same. +pub(super) const COMMAND_WAIT: Duration = Duration::from_secs(15); + +/// The error code of a command that gave up waiting for the core. The message names what holds it. +pub const BUSY: &str = "coreBusy"; + +/// The event that says a command has held the core for too long, with `what` holds it and for how many +/// `seconds`, and the one that says the core answers again. +pub const STALLED_EVENT: &str = "core:stalled"; +pub const RESPONSIVE_EVENT: &str = "core:responsive"; + +/// What holds the core now: since when, and which command. +pub(super) type Held = Arc)>>>; + +/// The door the commands queue at: whether the core is taken, and the condition the waiters sleep on until +/// the holder leaves. Serializing the commands here, not on the core's own lock, is what lets a command give up +/// after a bounded wait without polling. +pub(super) type Door = Arc<(Mutex, Condvar)>; + +/// The taken core: the core's state, given back through the door when dropped (after the state's own guard). +pub(super) struct CoreGuard<'a> { + state: MutexGuard<'a, Option>, + _turn: Turn<'a>, +} + +impl std::ops::Deref for CoreGuard<'_> { + type Target = Option; + fn deref(&self) -> &Option { + &self.state + } +} + +impl std::ops::DerefMut for CoreGuard<'_> { + fn deref_mut(&mut self) -> &mut Option { + &mut self.state + } +} + +/// Frees the door and wakes the next waiter when a command returns, or unwinds. +struct Turn<'a>(&'a Door); + +impl Drop for Turn<'_> { + fn drop(&mut self) { + let (taken, freed) = &**self.0; + *taken.lock().unwrap_or_else(PoisonError::into_inner) = false; + freed.notify_one(); + } +} + +/// Clears the holder when the command returns, or unwinds. +pub(super) struct Holding(pub(super) Held); + +impl Drop for Holding { + fn drop(&mut self) { + *self.0.lock().unwrap_or_else(PoisonError::into_inner) = None; + } +} + +/// Where a command was called from, for the watchdog: the file and line, with the workspace path trimmed. +pub(super) fn called_from(location: &Location<'_>) -> String { + let file = location.file().replace('\\', "/"); + let file = file.rsplit_once("/src/").map_or(file.as_str(), |(_, rest)| rest); + format!("the command at {file}:{}", location.line()) +} + +impl CoreBridge { + /// Takes the core, waiting at most `timeout` for the command that holds it. `None` when it is still held + /// after that. Commands queue at the door: a waiter sleeps on its condition variable until the holder + /// leaves, not on a poll, so a contended command starts the moment the one before it ends. + pub(super) fn take_core_within(&self, timeout: Duration) -> Option> { + let deadline = Instant::now().checked_add(timeout); + let (taken, freed) = &*self.door; + let mut taken = taken.lock().unwrap_or_else(PoisonError::into_inner); + while *taken { + let left = deadline.map_or(Duration::MAX, |d| d.saturating_duration_since(Instant::now())); + if left.is_zero() { + return None; + } + taken = freed + .wait_timeout(taken, left) + .unwrap_or_else(PoisonError::into_inner) + .0; + } + *taken = true; + drop(taken); + let state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + Some(CoreGuard { + state, + _turn: Turn(&self.door), + }) + } + + /// What holds the core now, for the log. + pub(super) fn holder(&self) -> String { + let holder = self.held.lock().unwrap_or_else(PoisonError::into_inner); + holder + .as_ref() + .map_or("nothing holds it now".to_owned(), |(since, what)| { + format!("{what} has held it for {} s", since.elapsed().as_secs()) + }) + } + + /// Starts the watchdog that reports a command holding the core for longer than `warn_after`: an error in + /// the log, and [`STALLED_EVENT`] to the interface, again every `warn_after` while it goes on, and then + /// [`RESPONSIVE_EVENT`] once the command returns. It can't free the core, but the log then says what to + /// look at, and the title bar says the core isn't responding, where a silent hang said "Saving" forever. + pub(super) fn watch_held(&self) { + let held = Arc::downgrade(&self.held); + let relay = Arc::downgrade(&self.relay); + let warn_after = self.warn_after; + let spawned = std::thread::Builder::new() + .name("opennote-core-watchdog".into()) + .spawn(move || { + // The hold last reported, by its start, and when it was reported. + let mut reported: Option<(Instant, Instant)> = None; + loop { + std::thread::sleep(warn_after / 4); + let Some(held) = held.upgrade() else { return }; + let now = held.lock().unwrap_or_else(PoisonError::into_inner).clone(); + let send = |name, payload| { + if let Some(relay) = relay.upgrade() { + relay.send(name, payload); + } + }; + let Some((since, what)) = now else { + if reported.take().is_some() { + ::log::info!("The core answers again"); + send(RESPONSIVE_EVENT, serde_json::json!({})); + } + continue; + }; + let elapsed = since.elapsed(); + let due = match reported { + Some((hold, at)) if hold == since => at.elapsed() >= warn_after, + _ => elapsed >= warn_after, + }; + if !due { + continue; + } + reported = Some((since, Instant::now())); + let seconds = elapsed.as_secs(); + ::log::error!("{what} has held the core for {seconds} s, and every other command waits for it"); + send(STALLED_EVENT, serde_json::json!({ "what": what, "seconds": seconds })); + } + }); + if let Err(error) = spawned { + ::log::warn!("Couldn't start the core watchdog: {error}"); + } + } +} diff --git a/app/src-tauri/src/core_bridge/search.rs b/app/src-tauri/src/core_bridge/search.rs new file mode 100644 index 00000000..c0eeb85d --- /dev/null +++ b/app/src-tauri/src/core_bridge/search.rs @@ -0,0 +1,213 @@ +//! Search and linking for the interface (Phase 8; crates/search). One command, `search_call`, carries every +//! method, so the app's command list, the handler list, and the capability file each gain one line. +//! +//! The index follows the core the bridge starts. The core's save hook and events feed it, and it reads pages and +//! their titles from the notebooks the core has open, which are the notebook folders in the notes folder. The +//! interface's page IDs are the core's, so nothing maps between them, and the interface sends the index nothing. +//! The bridge tells the hub which notebooks are open after every notes command, so a notebook that opens (made, +//! restored, imported, or found in a copied notes folder) is indexed and a notebook that closes leaves the index. +//! The index file is a cache on this device (`search.db` beside the core's data), and the next start builds it +//! again if it is gone. + +mod args; +mod calls; +mod events; +mod extras; +#[cfg(test)] +mod tests; + +use std::{ + collections::BTreeSet, + path::Path, + sync::{Arc, Mutex, MutexGuard, PoisonError}, +}; + +use opennote_core::{ + session::{core::Core, events::IndexSink}, + NotebookId, PageId, +}; +use opennote_search::{ + BackgroundIndexer, CorePageSource, CoreSlot, IndexerConfig, IndexerHandle, MediaText, SearchError, SearchIndex, + SharedIndex, Switcher, +}; +use serde_json::Value; +use tauri::{AppHandle, State}; + +use self::events::announce; +use super::{CoreBridge, Relay}; +use crate::ipc::{codes, IpcError, IpcResult}; + +/// The event that tells the interface what the index changed. +pub const UPDATED_EVENT: &str = "search:updated"; + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn lock_index(index: &SharedIndex) -> MutexGuard<'_, SearchIndex> { + index.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn internal(error: impl std::fmt::Display) -> IpcError { + IpcError::new(codes::INTERNAL, error.to_string()) +} + +fn invalid(field: &str, error: impl std::fmt::Display) -> IpcError { + IpcError::invalid(field, &error.to_string()) +} + +fn search_error(error: SearchError) -> IpcError { + match error { + SearchError::Pattern(message) => IpcError::invalid("pattern", &message), + other => internal(other), + } +} + +struct Run { + /// Taken at close. + indexer: Option, + handle: IndexerHandle, + /// The notebooks the indexer has been told are open. `None` until the first start. + open: Option>, +} + +/// The index, the way the bridge reaches it. The bridge owns one; its commands go through it. +#[derive(Default)] +pub(super) struct Hub { + run: Mutex>, + slot: CoreSlot, + switcher: Mutex, + /// Words read from pictures, handwriting, and recordings, kept beside the index. + media: MediaText, +} + +impl Hub { + /// Forgets the words read from the media of pages deleted for good: their pictures, handwriting, and + /// transcripts. The index drops the pages themselves when their notebooks are read again. + pub(super) fn forget_pages(&self, pages: &[PageId]) { + for page in pages { + self.media.forget_page(*page); + } + } + + /// Opens the index file and starts the indexer, before the core starts, because the core's save hook and + /// events need its handle. A file that can't be opened leaves an index in memory, which the next start builds + /// again, so search still works. + pub(super) fn spawn(&self, root: &Path, relay: &Arc) -> Result { + let mut run = lock(&self.run); + if let Some(run) = run.as_ref() { + return Ok(run.handle.clone()); + } + let index = SearchIndex::open(&root.join("search.db")) + .or_else(|error| { + ::log::warn!("The search index file couldn't be opened, so search starts empty: {error}"); + SearchIndex::open_in_memory() + }) + .map_err(internal)?; + self.media.attach(&root.join("media-text.json")); + let source = CorePageSource::production(self.slot.clone()).with_media(self.media.clone()); + let observer = { + let relay = relay.clone(); + Box::new(move |event| announce(&relay, event)) + }; + let indexer = BackgroundIndexer::spawn(index, source, IndexerConfig::default(), Some(observer)); + let handle = indexer.handle(); + let _ = relay.index.set(handle.clone()); + *run = Some(Run { + indexer: Some(indexer), + handle: handle.clone(), + open: None, + }); + Ok(handle) + } + + /// The hook the core calls after each save. + pub(super) fn sink(&self) -> Option> { + let handle = lock(&self.run).as_ref().map(|run| run.handle.clone())?; + Some(Arc::new(handle)) + } + + /// The core is running: the indexer may read its notebooks. + pub(super) fn attach(&self, core: &Core) { + self.slot.set(core.clone()); + } + + /// Tells the indexer which notebooks are open. The first call starts the index with them, which drops any + /// notebook the index holds that is not open and compares the rest with the files. Later calls reconcile a + /// notebook that opened and drop one that closed. A backup notebook is not indexed. + pub(super) fn notebooks_changed(&self, core: &Core) { + // Safe mode turns the index's background work off (Phase 13): nothing is indexed while it is on. + if crate::hardening::safe_mode() { + return; + } + let now: BTreeSet = core + .notebooks() + .into_iter() + .filter(|notebook| !notebook.is_backup()) + .map(|notebook| notebook.id()) + .collect(); + let mut run = lock(&self.run); + let Some(run) = run.as_mut() else { + return; + }; + match &run.open { + None => run.handle.start(now.iter().copied().collect()), + Some(before) => { + for closed in before.difference(&now) { + run.handle.notebook_closed(*closed); + } + for opened in now.difference(before) { + run.handle.tree_changed(*opened); + } + } + } + run.open = Some(now); + } + + /// Finishes the waiting jobs for a moment and closes the index file cleanly. The app calls it on exit. + pub(super) fn close(&self) { + let indexer = lock(&self.run).as_mut().and_then(|run| run.indexer.take()); + if let Some(indexer) = indexer { + if let Err(error) = indexer.close() { + ::log::warn!("The search index didn't close cleanly: {error}"); + } + } + } + + fn handle(&self) -> IpcResult<(IndexerHandle, Vec)> { + lock(&self.run) + .as_ref() + .map(|run| { + let open = run.open.iter().flatten().copied().collect(); + (run.handle.clone(), open) + }) + .ok_or_else(|| internal("The search index isn't running.")) + } + + fn core_page(&self, page: &str) -> IpcResult { + PageId::parse(page).map_err(|error| invalid("page", error)) + } + + /// Runs one method of the interface's search client. + pub(super) fn call(&self, method: &str, args: Value) -> IpcResult { + let (handle, notebooks) = self.handle()?; + self.dispatch(&handle, ¬ebooks, method, &args) + } +} + +/// One method of the search client. +#[tauri::command] +pub async fn search_call( + app: AppHandle, + bridge: State<'_, CoreBridge>, + method: String, + args: Value, +) -> IpcResult { + bridge.listen_app(&app); + // The first call starts the core, because the index follows it. + bridge + .run(|bridge| bridge.with_named("search_call", |_| Ok(()))) + .await?; + let hub = bridge.search.clone(); + hub.call(&method, args) +} diff --git a/app/src-tauri/src/core_bridge/search/args.rs b/app/src-tauri/src/core_bridge/search/args.rs new file mode 100644 index 00000000..eb150d63 --- /dev/null +++ b/app/src-tauri/src/core_bridge/search/args.rs @@ -0,0 +1,99 @@ +//! The arguments of each method of the interface's search client. + +use opennote_search::Query; +use serde::Deserialize; +use serde_json::Value; + +use super::invalid; +use crate::ipc::IpcResult; + +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +pub(super) struct QueryArgs { + pub(super) text: String, + pub(super) regex: bool, + pub(super) filters: Query, + pub(super) limit: Option, + pub(super) offset: Option, + pub(super) utc_offset_minutes: i32, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct SwitcherArgs { + pub(super) query: String, + #[serde(default)] + pub(super) recent: Vec, + #[serde(default)] + pub(super) current: Option, + #[serde(default)] + pub(super) limit: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct SuggestArgs { + pub(super) prefix: String, + #[serde(default)] + pub(super) limit: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct PageArgs { + pub(super) page: String, + #[serde(default)] + pub(super) limit: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct LinkArg { + pub(super) title: String, + #[serde(default)] + pub(super) heading: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ResolveArgs { + #[serde(default)] + pub(super) from: Option, + pub(super) links: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct PreviewArgs { + #[serde(default)] + pub(super) page: Option, + #[serde(default)] + pub(super) title: Option, + #[serde(default)] + pub(super) fragment: Option, + #[serde(default)] + pub(super) from: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct MentionArgs { + pub(super) markdown: String, + pub(super) title: String, + #[serde(default)] + pub(super) target: Option, + #[serde(default)] + pub(super) which: Option>, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct TagArgs { + pub(super) from: String, + #[serde(default)] + pub(super) to: Option, +} + +pub(super) fn parse Deserialize<'de>>(args: &Value) -> IpcResult { + serde_json::from_value(args.clone()).map_err(|error| invalid("args", error)) +} diff --git a/app/src-tauri/src/core_bridge/search/calls.rs b/app/src-tauri/src/core_bridge/search/calls.rs new file mode 100644 index 00000000..983adcf3 --- /dev/null +++ b/app/src-tauri/src/core_bridge/search/calls.rs @@ -0,0 +1,220 @@ +//! The methods of the interface's search client, one function each, and the match that picks one by name. + +use std::time::Duration; + +use opennote_core::{NotebookId, PageId}; +use opennote_search::{ + mentions, pattern, syntax, IndexerHandle, PlaceList, Query, SearchError, SearchIndex, SearchLimits, SwitchContext, + TextMode, +}; +use serde::Serialize; +use serde_json::{json, Value}; + +use super::args::{ + parse, MentionArgs, PageArgs, PreviewArgs, QueryArgs, ResolveArgs, SuggestArgs, SwitcherArgs, TagArgs, +}; +use super::events::now; +use super::{internal, lock, lock_index, search_error, Hub}; +use crate::ipc::IpcResult; + +fn to_value(value: &T) -> IpcResult { + serde_json::to_value(value).map_err(internal) +} + +impl Hub { + /// Runs the method `method` on the index. + pub(super) fn dispatch( + &self, + handle: &IndexerHandle, + notebooks: &[NotebookId], + method: &str, + args: &Value, + ) -> IpcResult { + let shared = handle.index(); + let index = || lock_index(&shared); + match method { + "query" => self.query(handle, notebooks, args), + "switcher" => self.switch(&index(), args), + "suggestPages" => suggest(&index(), args), + "headings" => self.on_page(args, |page, _| index().headings(page)), + "backlinks" => self.on_page(args, |page, _| index().backlinks(page)), + "outgoing" => self.on_page(args, |page, _| index().outgoing_links(page)), + "repairEdits" => self.on_page(args, |page, _| index().repair_edits(page)), + "unlinkedMentions" => self.on_page(args, |page, a| index().unlinked_mentions(page, a.limit.unwrap_or(50))), + "resolve" => self.resolve(&index(), args), + "linkPreview" => self.link_preview(&index(), args), + "findMentions" => find_mentions(args), + "linkMentions" => self.link_mentions(args), + "tagTree" => to_value(&index().tag_tree().map_err(search_error)?), + "planTagRename" | "planTagDelete" => plan_tag(&index(), method, args), + "titleSettled" => self.settle(handle, args), + "rebuild" => rebuild(handle, notebooks), + "status" => status(handle, &index()), + "flush" => Ok(json!(handle.flush(Duration::from_secs(20)))), + other => self.extra(handle, notebooks, other, args), + } + } + + /// Runs a read that takes one page. + fn on_page( + &self, + args: &Value, + read: impl FnOnce(PageId, &PageArgs) -> Result, + ) -> IpcResult { + let request: PageArgs = parse(args)?; + let page = self.core_page(&request.page)?; + to_value(&read(page, &request).map_err(search_error)?) + } + + /// The names of the open notebooks and their sections, for the `in:` operator. + fn places(&self, notebooks: &[NotebookId]) -> PlaceList { + let mut places = PlaceList::new(); + for notebook in notebooks.iter().filter_map(|id| self.slot.notebook(*id)) { + let tree = notebook.tree(); + places.notebook(&tree.title, tree.notebook); + for section in &tree.sections { + places.section(§ion.title, section.id); + } + } + places + } + + fn switch(&self, index: &SearchIndex, args: &Value) -> IpcResult { + let request: SwitcherArgs = parse(args)?; + let mut switcher = lock(&self.switcher); + switcher.refresh(index).map_err(search_error)?; + let context = SwitchContext { + recent: request.recent.iter().filter_map(|ui| self.core_page(ui).ok()).collect(), + current: request.current.as_deref().and_then(|ui| self.core_page(ui).ok()), + scope: None, + limit: request.limit.unwrap_or(0), + }; + to_value(&switcher.find(&request.query, &context)) + } + + fn resolve(&self, index: &SearchIndex, args: &Value) -> IpcResult { + let request: ResolveArgs = parse(args)?; + let from = request.from.as_deref().and_then(|ui| self.core_page(ui).ok()); + let mut out = Vec::new(); + for link in &request.links { + let found = index.resolve_title(&link.title, link.heading.as_deref(), from); + out.push(found.map_err(search_error)?); + } + to_value(&out) + } + + fn link_preview(&self, index: &SearchIndex, args: &Value) -> IpcResult { + let request: PreviewArgs = parse(args)?; + let fragment = request.fragment.as_deref(); + let page = match (&request.page, &request.title) { + (Some(ui), _) => Some(self.core_page(ui)?), + (None, Some(title)) => { + let from = request.from.as_deref().and_then(|ui| self.core_page(ui).ok()); + let resolved = index.resolve_title(title, fragment, from).map_err(search_error)?; + resolved.first().map(|target| target.page) + } + (None, None) => None, + }; + let preview = match page { + Some(page) => index.link_preview(page, fragment).map_err(search_error)?, + None => None, + }; + to_value(&preview) + } + + fn link_mentions(&self, args: &Value) -> IpcResult { + let request: MentionArgs = parse(args)?; + let target = self.core_page(request.target.as_deref().unwrap_or_default())?; + let linked = mentions::link_mentions(&request.markdown, &request.title, target, request.which.as_deref()); + Ok(match linked { + Some(linked) => json!({ "markdown": linked.markdown, "count": linked.linked.len() }), + None => Value::Null, + }) + } + + fn settle(&self, handle: &IndexerHandle, args: &Value) -> IpcResult { + let request: PageArgs = parse(args)?; + handle.title_settled(self.core_page(&request.page)?); + Ok(Value::Null) + } + + /// Reads the search box as the person typed it, and runs it. A pattern that is not valid comes back as a + /// message for the line under the box, and a typed operator that can't be used as a note. + pub(super) fn query(&self, handle: &IndexerHandle, notebooks: &[NotebookId], args: &Value) -> IpcResult { + let request: QueryArgs = parse(args)?; + let (mut query, notes) = match typed_query(&request, &self.places(notebooks)) { + Ok(typed) => typed, + Err(message) => return Ok(json!({ "hits": [], "complete": true, "notes": [], "patternError": message })), + }; + let filters = request.filters; + query.tags.extend(filters.tags); + query.block_types.extend(filters.block_types); + if filters.date.is_some() { + query.date = filters.date; + } + query.title_only |= filters.title_only; + query.limit = request.limit.unwrap_or(20).min(100); + query.offset = request.offset.unwrap_or(0); + let results = handle + .search_within(&query, &SearchLimits::default()) + .map_err(search_error)?; + Ok(json!({ + "hits": to_value(&results.hits)?, + "complete": results.complete, + "notes": notes, + })) + } +} + +/// The query for the text of the box, and a note for each typed operator it could not use. A regular expression +/// that is not valid is the error. +fn typed_query(request: &QueryArgs, places: &PlaceList) -> Result<(Query, Vec), String> { + if request.regex || request.filters.mode == TextMode::Regex { + pattern::check(&request.text)?; + return Ok((Query::regex(request.text.clone()), Vec::new())); + } + let context = syntax::SyntaxContext { + now: now(), + utc_offset_minutes: request.utc_offset_minutes, + places, + }; + let typed = syntax::parse(&request.text, &context); + let notes = typed + .notes + .iter() + .map(|note| json!({ "message": note.message() })) + .collect(); + Ok((typed.query, notes)) +} + +fn suggest(index: &SearchIndex, args: &Value) -> IpcResult { + let request: SuggestArgs = parse(args)?; + let found = index.suggest_pages(&request.prefix, request.limit.unwrap_or(8)); + to_value(&found.map_err(search_error)?) +} + +fn find_mentions(args: &Value) -> IpcResult { + let request: MentionArgs = parse(args)?; + to_value(&mentions::find_mentions(&request.markdown, &request.title)) +} + +fn plan_tag(index: &SearchIndex, method: &str, args: &Value) -> IpcResult { + let request: TagArgs = parse(args)?; + let plan = if method == "planTagDelete" { + index.plan_tag_delete(&request.from) + } else { + index.plan_tag_rename(&request.from, request.to.as_deref().unwrap_or_default()) + }; + to_value(&plan.map_err(search_error)?) +} + +fn rebuild(handle: &IndexerHandle, notebooks: &[NotebookId]) -> IpcResult { + handle.rebuild(notebooks.to_vec()); + Ok(Value::Null) +} + +fn status(handle: &IndexerHandle, index: &SearchIndex) -> IpcResult { + let stats = handle.stats(); + let pages = index.page_count().map_err(search_error)?; + Ok(json!({ "pages": pages, "failures": handle.failures().len(), "rebuilds": stats.rebuilds })) +} diff --git a/app/src-tauri/src/core_bridge/search/events.rs b/app/src-tauri/src/core_bridge/search/events.rs new file mode 100644 index 00000000..15845cda --- /dev/null +++ b/app/src-tauri/src/core_bridge/search/events.rs @@ -0,0 +1,57 @@ +//! What the indexer tells the window. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use opennote_core::{PageId, Timestamp}; +use opennote_search::{IndexEvent, IndexUpdate}; +use serde_json::{json, Value}; + +use super::{Relay, UPDATED_EVENT}; + +/// The current time, for words such as `today` in a typed search. +pub(super) fn now() -> Timestamp { + let ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0); + Timestamp::from_unix_ms(ms) +} + +/// Tells the interface what the indexer did. +pub(super) fn announce(relay: &Relay, event: IndexEvent) { + let Some(emit) = relay.emit.get() else { + return; + }; + let payload = match event { + IndexEvent::Updated(update) => update_payload(&update), + IndexEvent::Rebuilt => json!({ "rebuilt": true }), + IndexEvent::Failed { page, message } => json!({ "failed": { "page": page.to_string(), "message": message } }), + IndexEvent::Damaged { message } => json!({ "damaged": message }), + IndexEvent::CaughtUp => return, + }; + emit(UPDATED_EVENT, payload); +} + +fn update_payload(update: &IndexUpdate) -> Value { + let pages = |list: &[PageId]| list.iter().map(ToString::to_string).collect::>(); + let renames: Vec = update + .renames + .iter() + .map(|plan| { + json!({ + "page": plan.rename.page.to_string(), + "oldTitle": plan.rename.old_title, + "newTitle": plan.rename.new_title, + "edits": serde_json::to_value(&plan.edits).unwrap_or(Value::Null), + "otherPages": pages(&plan.other_pages()), + }) + }) + .collect(); + json!({ + "generation": update.generation, + "added": pages(&update.added), + "updated": pages(&update.updated), + "removed": pages(&update.removed), + "renames": renames, + }) +} diff --git a/app/src-tauri/src/core_bridge/search/extras.rs b/app/src-tauri/src/core_bridge/search/extras.rs new file mode 100644 index 00000000..490a0b9b --- /dev/null +++ b/app/src-tauri/src/core_bridge/search/extras.rs @@ -0,0 +1,315 @@ +//! The search methods added in Beta 4. They answer for the link a launch carried, for page facts (collections, the +//! calendar, and the graph filters), for tagged lines, and for the link graph. They are methods of the same +//! `search_call` command, so the command list stays as it was. Reading a page file here never opens a page +//! session, so it cannot disturb a page being edited, and a page in an encrypted section is never read. + +use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; + +use opennote_core::format::CanonicalCodec; +use opennote_core::limits::{Limits, Timings}; +use opennote_core::model::{BlockData, Page}; +use opennote_core::seams::Codec; +use opennote_core::session::page::read_page_dir; +use opennote_core::store::fs::Fs; +use opennote_core::store::layout::NotebookLayout; +use opennote_core::store::std_fs::StdFs; +use opennote_core::{BlockId, NotebookId, PageId, SectionId}; +use opennote_search::sync::Job; +use opennote_search::{IndexerHandle, MediaKind, PageFact, Scope}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use super::{internal, invalid, lock_index, search_error, Hub}; +use crate::ipc::IpcResult; + +/// The most tagged blocks one request returns. +const MAX_TAGGED: usize = 4000; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ScopeArgs { + scope: Scope, +} + +#[derive(Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct NotebookArgs { + notebook: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct FindArgs { + needle: String, + limit: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct MediaArgs { + page: String, + block: Option, + kind: Option, + text: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct NeighborArgs { + page: String, + depth: Option, +} + +/// A text block that carries line tags or an open checkbox. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct TaggedBlock { + page: PageId, + title: String, + notebook: NotebookId, + section: SectionId, + modified: i64, + block: String, + markdown: String, + ids: Vec, + tags: BTreeMap>, + checked: Vec, +} + +fn args Deserialize<'de>>(value: &Value) -> IpcResult { + serde_json::from_value(value.clone()).map_err(|error| invalid("args", error)) +} + +fn notebook_of(args: &NotebookArgs) -> IpcResult> { + args.notebook + .as_deref() + .map(|text| NotebookId::parse(text).map_err(|error| invalid("notebook", error))) + .transpose() +} + +/// Reads page files of open notebooks that are not encrypted. +struct PageReader<'a> { + hub: &'a Hub, + fs: Arc, + codec: Arc, + limits: Limits, + /// The encrypted sections of each notebook already looked at, or `None` for a notebook that can't be read. + locked: BTreeMap>>, +} + +impl<'a> PageReader<'a> { + fn new(hub: &'a Hub) -> Self { + PageReader { + hub, + fs: Arc::new(StdFs::new(&Timings::default())), + codec: Arc::new(CanonicalCodec), + limits: Limits::default(), + locked: BTreeMap::new(), + } + } + + fn read(&mut self, notebook: NotebookId, section: SectionId, page: PageId) -> Option { + let handle = self.hub.slot.notebook(notebook)?; + let locked = self.locked.entry(notebook).or_insert_with(|| { + (!handle.is_backup()).then(|| { + let tree = handle.tree(); + tree.sections.iter().filter(|s| s.encrypted).map(|s| s.id).collect() + }) + }); + if locked.as_ref()?.contains(§ion) { + return None; + } + let dir = NotebookLayout::new(handle.path()).page_dir(section, page); + read_page_dir(self.fs.as_ref(), self.codec.as_ref(), &dir, &self.limits) + .ok() + .map(|loaded| loaded.page) + } +} + +/// Whether Markdown has an unchecked task item. +fn has_open_box(markdown: &str) -> bool { + markdown.lines().any(|line| { + let line = line.trim_start(); + let rest = line + .strip_prefix("- ") + .or_else(|| line.strip_prefix("* ")) + .or_else(|| line.strip_prefix("+ ")) + .or_else(|| { + let digits = line.trim_start_matches(|c: char| c.is_ascii_digit()); + digits.strip_prefix(". ").or_else(|| digits.strip_prefix(") ")) + }); + rest.is_some_and(|rest| rest.starts_with("[ ]")) + }) +} + +impl Hub { + /// Runs a method that only Beta 4 added, or says the method is not known. + pub(super) fn extra( + &self, + handle: &IndexerHandle, + _notebooks: &[NotebookId], + method: &str, + value: &Value, + ) -> IpcResult { + let shared = handle.index(); + match method { + "launchLink" => Ok(json!(crate::deeplink::take_launch())), + "pageFacts" => { + let request: NotebookArgs = args(value).unwrap_or_default(); + let facts = lock_index(&shared) + .page_facts(notebook_of(&request)?) + .map_err(search_error)?; + self.facts_json(facts) + } + "taggedBlocks" => { + let request: ScopeArgs = args(value)?; + let candidates = lock_index(&shared) + .tag_candidates(request.scope) + .map_err(search_error)?; + Ok(self.tagged_blocks(&candidates)) + } + "setMediaText" => { + let request: MediaArgs = args(value)?; + let page = self.core_page(&request.page)?; + let (Some(block), Some(kind)) = (request.block.as_deref(), request.kind) else { + return Err(invalid("args", "setMediaText needs a block and a kind")); + }; + let block = BlockId::parse(block).map_err(|error| invalid("block", error))?; + // A page the index does not hold is locked, deleted, or unknown, so nothing of it is kept. + let held = lock_index(&shared).indexed_page(page).map_err(search_error)?.is_some(); + let changed = held + && self + .media + .set(page, block, kind, request.text.as_deref().unwrap_or_default()); + if changed { + handle.submit(Job::Reload { page, notebook: None }); + } + Ok(json!(changed)) + } + "mediaBlocks" => { + let request: MediaArgs = args(value)?; + let page = self.core_page(&request.page)?; + let blocks: Vec = self + .media + .blocks(page) + .into_iter() + .map(|(block, kind)| json!({ "block": block, "kind": kind })) + .collect(); + Ok(Value::Array(blocks)) + } + "findText" => { + let request: FindArgs = args(value)?; + if request.needle.trim().is_empty() { + return Ok(json!([])); + } + let limit = request.limit.unwrap_or(2000).min(5000); + let hits = lock_index(&shared) + .blocks_containing(&request.needle, limit) + .map_err(search_error)?; + serde_json::to_value(hits).map_err(internal) + } + "graph" => { + let request: NotebookArgs = args(value).unwrap_or_default(); + let graph = lock_index(&shared) + .link_graph(notebook_of(&request)?) + .map_err(search_error)?; + Ok(json!({ + "pages": graph.pages(), + "edges": graph.edges(), + "orphans": graph.orphans().iter().map(|page| page.page).collect::>(), + "broken": graph.broken().len(), + })) + } + "neighbors" | "connections" => { + let request: NeighborArgs = args(value)?; + let page = self.core_page(&request.page)?; + let graph = lock_index(&shared).link_graph(None).map_err(search_error)?; + if method == "connections" { + serde_json::to_value(graph.connections(page)).map_err(internal) + } else { + let depth = request + .depth + .unwrap_or(1) + .clamp(1, opennote_search::linkgraph::MAX_DEPTH); + serde_json::to_value(graph.neighbors(page, depth)).map_err(internal) + } + } + other => Err(invalid("method", format!("unknown search method {other}"))), + } + } + + /// The facts as JSON, with the properties of the pages that have them read from their files. + fn facts_json(&self, facts: Vec) -> IpcResult { + let mut reader = PageReader::new(self); + let mut out = Vec::with_capacity(facts.len()); + for fact in facts { + let properties = if fact.has_properties { + reader + .read(fact.notebook, fact.section, fact.page) + .and_then(|page| page.view.extra.get("properties").cloned()) + } else { + None + }; + let mut row = serde_json::to_value(&fact).map_err(internal)?; + if let (Some(map), Some(properties)) = (row.as_object_mut(), properties) { + map.insert("properties".into(), properties); + } + out.push(row); + } + Ok(Value::Array(out)) + } + + /// The text blocks of the candidate pages that carry line tags or an open checkbox. + fn tagged_blocks(&self, candidates: &[PageFact]) -> Value { + let mut reader = PageReader::new(self); + let mut out: Vec = Vec::new(); + 'pages: for fact in candidates { + let Some(page) = reader.read(fact.notebook, fact.section, fact.page) else { + continue; + }; + for id in page.reading_order() { + let Some(block) = page.blocks.get(id) else { continue }; + let BlockData::Text(data) = &block.data else { continue }; + let tagged = data.tags.values().any(|tags| !tags.is_empty()); + if !tagged && !has_open_box(&data.markdown) { + continue; + } + out.push(TaggedBlock { + page: fact.page, + title: fact.title.clone(), + notebook: fact.notebook, + section: fact.section, + modified: fact.modified, + block: block.id.to_string(), + markdown: data.markdown.to_string(), + ids: data.ids.iter().map(ToString::to_string).collect(), + tags: data + .tags + .iter() + .filter(|(_, tags)| !tags.is_empty()) + .map(|(id, tags)| (id.to_string(), tags.clone())) + .collect(), + checked: data.checked.iter().map(ToString::to_string).collect(), + }); + if out.len() >= MAX_TAGGED { + break 'pages; + } + } + } + json!(out) + } +} + +#[cfg(test)] +mod tests { + use super::has_open_box; + + #[test] + fn finds_unchecked_task_items() { + assert!(has_open_box("Intro\n\n- [ ] buy milk\n- [x] done")); + assert!(has_open_box("1. [ ] first")); + assert!(!has_open_box("- [x] done\n- plain")); + assert!(!has_open_box("a [ ] in the middle")); + } +} diff --git a/app/src-tauri/src/core_bridge/search/tests.rs b/app/src-tauri/src/core_bridge/search/tests.rs new file mode 100644 index 00000000..b23b86d0 --- /dev/null +++ b/app/src-tauri/src/core_bridge/search/tests.rs @@ -0,0 +1,308 @@ +//! The search commands against a real core and real notebooks: typed text, titles, links, Trash, a restart, and a +//! notebook that arrives in the notes folder. + +use std::{ + path::Path, + time::{Duration, Instant}, +}; + +use opennote_core::ops::resolve::{Edit, NewBlock, TxnRequest}; +use serde_json::{json, Value}; + +use super::{internal, CoreBridge}; +use crate::{core_bridge::Bridge, ipc::codes, ipc::IpcError}; + +fn create(bridge: &mut Bridge, kind: &str, parent: Option<&str>, title: &str) -> Result { + let input = json!({ "kind": kind, "placement": { "parentId": parent, "beforeId": null }, "title": title }); + let node = bridge.dispatch("notes_create", &json!({ "input": input }))?; + Ok(node["id"].as_str().unwrap_or_default().to_owned()) +} + +/// A notebook with a section and one page for each title; returns the notebook, the section, and the pages. +fn notebook(bridge: &CoreBridge, notes: &Path, name: &str, section: &str, pages: &[&str]) -> Vec { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + let notebook = create(bridge, "notebook", None, name)?; + let section = create(bridge, "section", Some(¬ebook), section)?; + let mut made = vec![notebook, section.clone()]; + for title in pages { + made.push(create(bridge, "page", Some(§ion), title)?); + } + Ok(made) + }) + .expect("a notebook") +} + +fn type_text(bridge: &CoreBridge, notes: &Path, page: &str, markdown: &str) { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + let handle = bridge.handle(page, "main-1")?; + let block: NewBlock = serde_json::from_value(json!({ + "id": format!("01k6f0000000000000000{:05}", page.len() * 7 + markdown.len()), + "type": "text", + "data": { "markdown": markdown } + })) + .expect("a new block"); + let request = TxnRequest { + page: handle.id(), + client: handle.client().clone(), + client_seq: 1, + coalesce: None, + ui: None, + edits: vec![Edit::InsertBlock { + block, + after: None, + before: None, + }], + }; + handle.apply(request).expect("applies"); + handle.save_now().map_err(internal) + }) + .expect("typed text"); +} + +/// Runs a method until its answer satisfies `done`, because the index follows saves in the background. +fn eventually(bridge: &CoreBridge, method: &str, args: Value, done: impl Fn(&Value) -> bool) -> Value { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let answer = bridge.search.call(method, args.clone()).expect("a search answer"); + if done(&answer) || Instant::now() > deadline { + return answer; + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn titles(answer: &Value) -> Vec { + answer["hits"] + .as_array() + .map(|hits| { + hits.iter() + .filter_map(|hit| hit["title"].as_str().map(str::to_owned)) + .collect() + }) + .unwrap_or_default() +} + +fn found(answer: &Value) -> bool { + !titles(answer).is_empty() +} + +struct World { + _dir: tempfile::TempDir, + local: std::path::PathBuf, + notes: std::path::PathBuf, + bridge: CoreBridge, +} + +fn start() -> World { + let dir = tempfile::tempdir().expect("a temp folder"); + let (local, notes) = (dir.path().join("local"), dir.path().join("Notes")); + let bridge = CoreBridge::at(local.clone()); + World { + _dir: dir, + local, + notes, + bridge, + } +} + +#[test] +fn typed_text_is_found_under_the_page_and_its_title() { + let world = start(); + let made = notebook( + &world.bridge, + &world.notes, + "Biology", + "Lectures", + &["Cell biology", "Shopping"], + ); + type_text(&world.bridge, &world.notes, &made[2], "Mitochondria make energy"); + let answer = eventually(&world.bridge, "query", json!({ "text": "mitochondria" }), found); + assert_eq!(answer["hits"][0]["page"], made[2].as_str()); + assert_eq!(titles(&answer), ["Cell biology"]); + let by_title = eventually(&world.bridge, "query", json!({ "text": "shopping" }), found); + assert_eq!(by_title["hits"][0]["page"], made[3].as_str()); + world.bridge.shutdown(); +} + +#[test] +fn every_notebook_of_the_notes_folder_is_searched() { + let world = start(); + let first = notebook(&world.bridge, &world.notes, "Biology", "Lectures", &["Cells"]); + let second = notebook(&world.bridge, &world.notes, "Cooking", "Recipes", &["Bread"]); + type_text(&world.bridge, &world.notes, &first[2], "Osmosis moves water"); + type_text(&world.bridge, &world.notes, &second[2], "Osmosis in brine"); + let answer = eventually(&world.bridge, "query", json!({ "text": "osmosis" }), |answer| { + titles(answer).len() == 2 + }); + let mut names = titles(&answer); + names.sort(); + assert_eq!(names, ["Bread", "Cells"]); + // The in: operator names a notebook or a section of the real tree. + let scoped = eventually( + &world.bridge, + "query", + json!({ "text": "osmosis in:cooking" }), + |answer| titles(answer).len() == 1, + ); + assert_eq!(titles(&scoped), ["Bread"]); + world.bridge.shutdown(); +} + +#[test] +fn links_resolve_by_title_and_show_as_backlinks() { + let world = start(); + let made = notebook(&world.bridge, &world.notes, "Biology", "Lectures", &["Alpha", "Beta"]); + type_text( + &world.bridge, + &world.notes, + &made[3], + "See [[Alpha]] and [[Nowhere]] for more", + ); + let backlinks = eventually(&world.bridge, "backlinks", json!({ "page": made[2] }), |answer| { + answer.as_array().is_some_and(|list| !list.is_empty()) + }); + assert_eq!(backlinks[0]["source"], made[3].as_str()); + assert_eq!(backlinks[0]["sourceTitle"], "Beta"); + let resolved = world + .bridge + .search + .call( + "resolve", + json!({ "from": made[3], "links": [{ "title": "Alpha" }, { "title": "Nowhere" }] }), + ) + .expect("resolutions"); + assert_eq!(resolved[0]["status"], "resolved"); + assert_eq!(resolved[0]["targets"][0]["page"], made[2].as_str()); + assert_eq!(resolved[1]["status"], "broken"); + let suggestions = world + .bridge + .search + .call("suggestPages", json!({ "prefix": "al" })) + .expect("suggestions"); + assert_eq!(suggestions[0]["page"], made[2].as_str()); + world.bridge.shutdown(); +} + +#[test] +fn a_page_in_trash_leaves_the_index_and_comes_back_with_restore() { + let world = start(); + let made = notebook(&world.bridge, &world.notes, "Biology", "Lectures", &["Keep", "Gone"]); + eventually(&world.bridge, "query", json!({ "text": "gone" }), found); + let receipt = world + .bridge + .notes(Some(world.notes.clone()), |bridge| { + bridge.dispatch("notes_trash", &json!({ "ids": [made[3]] })) + }) + .expect("trashed"); + let answer = eventually(&world.bridge, "query", json!({ "text": "gone" }), |answer| { + !found(answer) + }); + assert!(!found(&answer), "a page in Trash is not found"); + world + .bridge + .notes(Some(world.notes.clone()), |bridge| { + bridge.dispatch("notes_restore", &json!({ "receiptId": receipt["id"] })) + }) + .expect("restored"); + let back = eventually(&world.bridge, "query", json!({ "text": "gone" }), found); + assert_eq!(titles(&back), ["Gone"]); + world.bridge.shutdown(); +} + +#[test] +fn a_renamed_page_is_found_by_its_new_title() { + let world = start(); + let made = notebook(&world.bridge, &world.notes, "Biology", "Lectures", &["Draft"]); + eventually(&world.bridge, "query", json!({ "text": "draft" }), found); + world + .bridge + .notes(Some(world.notes.clone()), |bridge| { + bridge.dispatch("notes_rename", &json!({ "id": made[2], "title": "Final copy" })) + }) + .expect("renamed"); + let answer = eventually(&world.bridge, "query", json!({ "text": "final" }), found); + assert_eq!(titles(&answer), ["Final copy"]); + world.bridge.shutdown(); +} + +#[test] +fn the_index_follows_the_notes_after_a_restart_and_a_notebook_that_arrives() { + let world = start(); + let made = notebook(&world.bridge, &world.notes, "Biology", "Lectures", &["Cells"]); + type_text(&world.bridge, &world.notes, &made[2], "Ribosomes build proteins"); + eventually(&world.bridge, "query", json!({ "text": "ribosomes" }), found); + world.bridge.shutdown(); + drop(world.bridge); + + // Another computer's notebook arrives in the notes folder while the app is closed. + let other = start_at( + &world.local.with_file_name("other-local"), + &world.notes.with_file_name("Elsewhere"), + ); + let arrived = notebook(&other.bridge, &other.notes, "Chemistry", "Labs", &["Titration"]); + type_text(&other.bridge, &other.notes, &arrived[2], "Ribosomes of acids"); + other.bridge.shutdown(); + drop(other.bridge); + let folder = std::fs::read_dir(&other.notes) + .expect("the other notes folder") + .flatten() + .map(|entry| entry.path()) + .find(|path| path.join("notebook.json").is_file()) + .expect("the notebook's folder"); + copy_dir(&folder, &world.notes.join("Chemistry")); + + let again = CoreBridge::at(world.local.clone()); + again + .notes(Some(world.notes.clone()), |_| Ok(())) + .expect("the notebooks open"); + let answer = eventually(&again, "query", json!({ "text": "ribosomes" }), |answer| { + titles(answer).len() == 2 + }); + let mut names = titles(&answer); + names.sort(); + assert_eq!(names, ["Cells", "Titration"]); + again.shutdown(); +} + +struct Other { + notes: std::path::PathBuf, + bridge: CoreBridge, +} + +fn start_at(local: &Path, notes: &Path) -> Other { + Other { + notes: notes.to_path_buf(), + bridge: CoreBridge::at(local.to_path_buf()), + } +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).expect("a folder"); + for entry in std::fs::read_dir(from).expect("a listing").flatten() { + let target = to.join(entry.file_name()); + if entry.path().is_dir() { + copy_dir(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), target).expect("a copy"); + } + } +} + +#[test] +fn a_bad_pattern_is_reported_and_an_unknown_method_is_refused() { + let world = start(); + world.bridge.with(|_| Ok(())).expect("the core starts"); + let answer = world + .bridge + .search + .call("query", json!({ "text": "(unclosed", "regex": true })) + .expect("an answer"); + assert!(answer["patternError"] + .as_str() + .is_some_and(|message| !message.is_empty())); + let error = world.bridge.search.call("nothing", json!({})).expect_err("refused"); + assert_eq!(error.code, codes::INVALID); + world.bridge.shutdown(); +} diff --git a/app/src-tauri/src/core_bridge_tests.rs b/app/src-tauri/src/core_bridge_tests.rs new file mode 100644 index 00000000..7df99266 --- /dev/null +++ b/app/src-tauri/src/core_bridge_tests.rs @@ -0,0 +1,568 @@ +//! Tests of the page commands' plumbing over pages the notes commands made. + +use std::fs; + +use opennote_core::ops::resolve::{Edit, TxnRequest}; +use serde_json::{json, Value}; + +use super::*; + +fn text_of(handle: &PageHandle) -> String { + let envelope = handle.envelope(None).expect("an envelope"); + let decoded = opennote_core::wire::envelope::decode(&envelope.bytes).expect("decodes"); + let page: Value = serde_json::from_slice(decoded.page_json).expect("page JSON"); + page["blocks"][0]["data"]["markdown"] + .as_str() + .unwrap_or_default() + .to_owned() +} + +fn insert(handle: &PageHandle, seq: u64, markdown: &str) { + insert_block(handle, seq, "01k6f00000000000000000b001", markdown); +} + +/// Inserts a text block whose ID is made from `n`, so a page can take many. +fn insert_block(handle: &PageHandle, seq: u64, id: &str, markdown: &str) { + let block: opennote_core::ops::resolve::NewBlock = serde_json::from_value(json!({ + "id": id, "type": "text", "data": { "markdown": markdown } + })) + .expect("a new block"); + let request = TxnRequest { + page: handle.id(), + client: handle.client().clone(), + client_seq: seq, + coalesce: None, + ui: None, + edits: vec![Edit::InsertBlock { + block, + after: None, + before: None, + }], + }; + handle.apply(request).expect("applies"); +} + +/// The ID of the `n`th block a test inserts. +fn block_id(n: u64) -> String { + format!("01k6f0000000000000000{n:05}") +} + +/// A notebook with a section and a page in `notes`; returns the page's ID. +fn a_page(bridge: &CoreBridge, notes: &Path) -> String { + pages(bridge, notes, 1).remove(0) +} + +/// A notebook with a section and `count` pages in `notes`; returns the pages' IDs. +fn pages(bridge: &CoreBridge, notes: &Path, count: usize) -> Vec { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + let create = |bridge: &mut Bridge, kind: &str, parent: Option| { + let input = json!({ "kind": kind, "placement": { "parentId": parent, "beforeId": null } }); + let node = bridge.dispatch("notes_create", &json!({ "input": input }))?; + Ok::<_, IpcError>(node["id"].as_str().unwrap_or_default().to_owned()) + }; + let notebook = create(bridge, "notebook", None)?; + let section = create(bridge, "section", Some(notebook))?; + (0..count) + .map(|_| create(bridge, "page", Some(section.clone()))) + .collect() + }) + .expect("pages") +} + +#[test] +fn typed_text_survives_a_restart_of_the_core() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (local, notes) = (dir.path().join("local"), dir.path().join("Notes")); + let bridge = CoreBridge::at(local.clone()); + let page = a_page(&bridge, ¬es); + bridge + .notes(Some(notes.clone()), |bridge| { + insert(&bridge.handle(&page, "main-1")?, 1, "Cells divide"); + Ok(()) + }) + .expect("the first run"); + bridge.shutdown(); + let again = CoreBridge::at(local); + let text = again + .notes(Some(notes), |bridge| Ok(text_of(&bridge.handle(&page, "main-1")?))) + .expect("the second run"); + again.shutdown(); + assert_eq!(text, "Cells divide"); +} + +/// A command that holds the core for too long is reported, to the log and to the interface, with its name and +/// for how long; and the interface hears when the core answers again. Beta 4's T2-3 hung with "Saving" in the +/// title bar and nothing in the log, and nobody could tell what the core was doing. +#[test] +fn a_command_that_holds_the_core_too_long_is_reported_and_the_core_answers_again_after() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")).warning_after(Duration::from_millis(100)); + let heard: Arc>> = Arc::default(); + let seen = heard.clone(); + bridge.listen(Box::new(move |name, payload| { + seen.lock().expect("the events").push((name, payload)); + })); + a_page(&bridge, ¬es); + // The first command started the core, which may itself have been reported: the events are looked for + // after the ones heard so far, by what they say. + let event = |name: &str, after: usize, what: Option<&str>| -> Option<(usize, Value)> { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while std::time::Instant::now() < deadline { + let found = heard + .lock() + .expect("the events") + .iter() + .enumerate() + .skip(after) + .find_map(|(i, (n, p))| { + (*n == name && what.is_none_or(|what| p["what"] == what)).then(|| (i, p.clone())) + }); + if found.is_some() { + return found; + } + std::thread::sleep(Duration::from_millis(10)); + } + None + }; + let mut seen_until = 0; + let (started, ready) = std::sync::mpsc::channel(); + let (release, released) = std::sync::mpsc::channel::<()>(); + std::thread::scope(|scope| { + let stuck = &bridge; + scope.spawn(move || { + let _ = stuck.with_named("a stuck command", |_| { + let _ = started.send(()); + let _ = released.recv_timeout(Duration::from_secs(5)); + Ok(()) + }); + }); + ready.recv().expect("the command runs"); + let (at, stalled) = + event(STALLED_EVENT, 0, Some("a stuck command")).expect("the interface hears that the core stalled"); + seen_until = at + 1; + assert!(stalled["seconds"].is_u64(), "{stalled}"); + assert!( + bridge.holder().starts_with("a stuck command has held it for"), + "{}", + bridge.holder() + ); + let _ = release.send(()); + }); + event(RESPONSIVE_EVENT, seen_until, None).expect("the interface hears that the core answers again"); + assert_eq!(bridge.holder(), "nothing holds it now"); + bridge + .with_named("a command after it", |_| Ok(())) + .expect("the core is free"); + bridge.shutdown(); +} + +/// Every command through `with` is known to the watchdog by where it was called from, so the log names the +/// holder without each caller naming itself. +#[test] +fn the_watchdog_knows_a_command_by_where_it_was_called_from() { + let dir = tempfile::tempdir().expect("a temp folder"); + let bridge = CoreBridge::at(dir.path().join("local")); + let holder = bridge.with(|_| Ok(bridge.holder())).expect("runs"); + assert!(holder.starts_with("the command at core_bridge_tests.rs:"), "{holder}"); + let holder = bridge.notes(None, |_| Ok(bridge.holder())).expect("runs"); + assert!(holder.starts_with("the command at core_bridge_tests.rs:"), "{holder}"); + bridge.shutdown(); +} + +/// Closes a client's session of a page, as `page_close` does. +fn close_page(bridge: &CoreBridge, page: &str, client: &str) { + bridge + .with(|bridge| { + let id = PageId::parse(page).expect("a page ID"); + let client = ClientId::parse(client).expect("a client ID"); + bridge.homes.remove(&(id, client.clone())); + if let Some(handle) = bridge.open.remove(&(id, client.clone())) { + handle.close(&client).map_err(internal)?; + } + Ok(()) + }) + .expect("closes"); +} + +/// The lock order of the core, under the load beta 4's T2-3 session had: typing, undo, saves, renames, pages +/// opening and closing, and a page going to Trash and back, all at once with the saver and maintenance threads. +/// A command that waits for another forever would hang this test, so a watchdog of its own ends the process +/// with the holder's name after a minute. +#[test] +fn concurrent_edits_saves_renames_and_closes_never_wait_for_each_other_forever() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let ids = pages(&bridge, ¬es, 3); + let (a, b, c) = (ids[0].clone(), ids[1].clone(), ids[2].clone()); + let open = |page: &str, client: &str| { + bridge + .notes(Some(notes.clone()), |bridge| bridge.handle(page, client)) + .expect("opens") + }; + let typing = open(&a, "typing"); + let saving = open(&a, "saving"); + let done = Arc::new(std::sync::atomic::AtomicBool::new(false)); + { + let (done, bridge) = (done.clone(), bridge.clone()); + std::thread::spawn(move || { + for _ in 0..60 { + std::thread::sleep(Duration::from_secs(1)); + if done.load(std::sync::atomic::Ordering::SeqCst) { + return; + } + } + eprintln!("The core deadlocked: {}", bridge.holder()); + std::process::abort(); + }); + } + let until = std::time::Instant::now() + Duration::from_millis(1500); + let running = || std::time::Instant::now() < until; + std::thread::scope(|scope| { + scope.spawn(|| { + let (mut seq, mut n) = (0, 0); + while running() { + seq += 1; + n += 1; + insert_block(&typing, seq, &block_id(n), "typed"); + if n % 3 == 0 { + typing.undo(typing.client()).expect("undoes"); + } + } + }); + scope.spawn(|| { + while running() { + saving.save_now().expect("saves"); + } + }); + scope.spawn(|| { + let mut n = 0; + while running() { + n += 1; + let title = format!("Title {n}"); + bridge + .notes(Some(notes.clone()), |bridge| { + bridge.dispatch("notes_rename", &json!({ "id": a, "title": title })) + }) + .expect("renames"); + } + }); + scope.spawn(|| { + let mut n = 1_000; + while running() { + n += 1; + let handle = open(&b, "window"); + insert_block(&handle, 1, &block_id(n), "in a window"); + close_page(&bridge, &b, "window"); + } + }); + scope.spawn(|| { + while running() { + let receipt = bridge + .notes(Some(notes.clone()), |bridge| { + bridge.dispatch("notes_trash", &json!({ "ids": [c] })) + }) + .expect("trashes"); + let receipt = receipt["id"].as_str().unwrap_or_default().to_owned(); + bridge + .notes(Some(notes.clone()), |bridge| { + bridge.dispatch("notes_restore", &json!({ "receiptId": receipt })) + }) + .expect("restores"); + } + }); + }); + done.store(true, std::sync::atomic::Ordering::SeqCst); + close_page(&bridge, &a, "typing"); + close_page(&bridge, &a, "saving"); + bridge.shutdown(); +} + +/// A command behind one that never returns fails as busy after its wait, naming the holder, instead of joining +/// the queue. The webview's IPC channel carries only a few commands at once, so a queue of them froze every +/// command. Close and the log froze with them (beta 4's T2-3). The core serves the next command once the holder +/// returns. +#[test] +fn a_command_gives_up_on_a_held_core_and_names_the_holder() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")).waiting_at_most(Duration::from_millis(100)); + a_page(&bridge, ¬es); + let (started, ready) = std::sync::mpsc::channel(); + let (release, released) = std::sync::mpsc::channel::<()>(); + std::thread::scope(|scope| { + let stuck = &bridge; + scope.spawn(move || { + let _ = stuck.with_named("a stuck command", |_| { + let _ = started.send(()); + let _ = released.recv_timeout(Duration::from_secs(5)); + Ok(()) + }); + }); + ready.recv().expect("the command runs"); + let at = std::time::Instant::now(); + let error = bridge + .with_named("the next command", |_| Ok(())) + .expect_err("the next command gives up"); + assert!(at.elapsed() < Duration::from_secs(2), "it waited {:?}", at.elapsed()); + assert_eq!(error.code, BUSY); + assert!( + error.message.contains("a stuck command"), + "the message names the holder: {}", + error.message + ); + release.send(()).expect("the holder is released"); + }); + bridge + .with_named("after", |_| Ok(())) + .expect("the core answers once the holder returns"); + bridge.shutdown(); +} + +#[test] +fn the_exit_gives_up_on_a_command_that_holds_the_core_too_long() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + a_page(&bridge, ¬es); + let (started, ready) = std::sync::mpsc::channel(); + let took = std::thread::scope(|scope| { + scope.spawn(|| { + let _ = bridge.with_named("a stuck command", |_| { + let _ = started.send(()); + std::thread::sleep(Duration::from_millis(900)); + Ok(()) + }); + }); + ready.recv().expect("the command runs"); + let at = std::time::Instant::now(); + bridge.shutdown_within(Duration::from_millis(150)); + at.elapsed() + }); + assert!( + took < Duration::from_millis(800), + "the exit waited {took:?} for the command" + ); + assert!( + bridge.held.lock().expect("the holder").is_none(), + "the holder is cleared once the command returns" + ); + bridge.shutdown(); +} + +#[test] +fn only_pages_of_open_notebooks_open_and_page_ids_are_checked() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let page = a_page(&bridge, ¬es); + let (unknown, unopened, opened) = bridge + .notes(Some(notes), |bridge| { + let unknown = bridge + .handle("01k6f00000000000000000p001", "main-1") + .map(|_| ()) + .unwrap_err(); + let unopened = bridge.open_handle(&page, "main-1").map(|_| ()).unwrap_err(); + bridge.handle(&page, "main-1")?; + Ok((unknown, unopened, bridge.open_handle(&page, "main-1").is_ok())) + }) + .expect("the bridge"); + bridge.shutdown(); + assert_eq!(unknown.code, "notFound"); + assert_eq!(unopened.code, "notFound"); + assert!(opened); + for bad in ["", "a/b", "..", "p 1", "..\\..\\x", &"p".repeat(MAX_PAGE_ID + 1)] { + assert_eq!(check_page(bad).expect_err(bad).code, codes::INVALID); + } + assert!(check_page("01k6f00000000000000000p001").is_ok()); + assert_eq!(core_error(CoreError::NotFound("version".into())).code, "notFound"); +} + +/// F3-3: a page window that closed went without closing its pages, so its client stayed in the session for good. +#[test] +fn a_window_that_goes_closes_the_page_sessions_it_left_open_and_no_others() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let page = a_page(&bridge, ¬es); + let main = bridge.windows().current("main"); + let first = bridge.windows().current("page-x"); + bridge + .notes(Some(notes.clone()), |inner| { + inner.handle_in(&page, "main-1", &main, bridge.windows())?; + inner.handle_in(&page, "wabc12345-1", &first, bridge.windows())?; + Ok(()) + }) + .expect("the bridge"); + let gone = bridge.window_went("page-x"); + assert_eq!(gone, first); + // The same page opens in its own window again before the old window's sessions are closed. + let second = bridge.windows().current("page-x"); + assert_ne!(second, first, "a window opened again with the same label is a new one"); + let late = bridge.notes(Some(notes.clone()), |inner| { + inner.handle_in(&page, "wabc12345-2", &first, bridge.windows()) + }); + assert_eq!( + late.map(|_| ()).unwrap_err().code, + "notFound", + "an open from the window that went is refused" + ); + bridge + .notes(Some(notes.clone()), |inner| { + inner.handle_in(&page, "wdef67890-1", &second, bridge.windows())?; + Ok(()) + }) + .expect("the new window opens the page"); + assert_eq!(bridge.close_window(&gone), 1); + assert_eq!(bridge.close_window(&gone), 0, "it closes each session once"); + let open = bridge + .notes(Some(notes), |inner| { + Ok(( + inner.open_handle(&page, "main-1").is_ok(), + inner.open_handle(&page, "wabc12345-1").is_ok(), + inner.open_handle(&page, "wabc12345-2").is_ok(), + inner.open_handle(&page, "wdef67890-1").is_ok(), + )) + }) + .expect("the bridge"); + bridge.shutdown(); + assert_eq!( + open, + (true, false, false, true), + "the new window's session outlives the old window's close" + ); +} + +/// F3-1: a second window that named a client another window holds was handed that window's session. +#[test] +fn a_client_belongs_to_one_window() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let page = a_page(&bridge, ¬es); + let main = bridge.windows().current("main"); + let other = bridge.windows().current("page-x"); + let (again, taken) = bridge + .notes(Some(notes), |inner| { + inner.handle_in(&page, "main-1", &main, bridge.windows())?; + let again = inner.handle_in(&page, "main-1", &main, bridge.windows()).is_ok(); + let taken = inner + .handle_in(&page, "main-1", &other, bridge.windows()) + .map(|_| ()) + .unwrap_err(); + Ok((again, taken)) + }) + .expect("the bridge"); + assert!(again, "the window that holds the client opens it again"); + assert_eq!(taken.code, "invalid", "{}", taken.message); + assert_eq!(bridge.close_window(&other), 0, "the session stays with its window"); + assert_eq!(bridge.close_window(&main), 1); + bridge.shutdown(); +} + +#[test] +fn a_page_moved_to_trash_loses_its_open_session() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let page = a_page(&bridge, ¬es); + bridge + .notes(Some(notes.clone()), |bridge| { + bridge.handle(&page, "main-1")?; + bridge.dispatch("notes_trash", &json!({ "ids": [page] })) + }) + .expect("the bridge"); + let still_open = bridge + .notes(Some(notes), |bridge| Ok(bridge.open_handle(&page, "main-1").is_ok())) + .expect("the bridge"); + bridge.shutdown(); + assert!(!still_open); +} + +#[test] +fn a_beta_one_profile_keeps_its_core_data_under_core() { + let dir = tempfile::tempdir().expect("a temp folder"); + let old = dir.path().join("phase4").join("core"); + fs::create_dir_all(&old).expect("the old data folder"); + fs::write(old.join("marker"), "kept").expect("a file"); + assert_eq!(data_dir(dir.path()), dir.path().join("core")); + assert_eq!( + fs::read_to_string(dir.path().join("core").join("marker")).expect("moved"), + "kept" + ); + assert!(!old.exists()); +} + +/// Phase 9 keeps a recording's entry in the page's view, because the core can't edit a block of a type it doesn't +/// know, and the time stamps of typed text in the data of a text block. Both must come back after a restart. +#[test] +fn a_recording_entry_and_text_marks_survive_a_restart() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (local, notes) = (dir.path().join("local"), dir.path().join("Notes")); + let edits = |value: Value| -> Vec { serde_json::from_value(value).expect("edits") }; + let apply = |handle: &PageHandle, seq: u64, edits: Vec| { + let request = TxnRequest { + page: handle.id(), + client: handle.client().clone(), + client_seq: seq, + coalesce: None, + ui: None, + edits, + }; + handle.apply(request).expect("applies"); + }; + let bridge = CoreBridge::at(local.clone()); + let page = a_page(&bridge, ¬es); + bridge + .notes(Some(notes.clone()), |bridge| { + let handle = bridge.handle(&page, "main-1")?; + insert(&handle, 1, "Osmosis"); + let entry = json!({ "id": "r1", "state": "recording", "flags": [{ "id": "f" }], "tracks": [] }); + apply( + &handle, + 2, + edits(json!([ + { "edit": "insertBlock", "after": "01k6f00000000000000000b001", "block": { + "id": "01k6f00000000000000000b002", "type": "ext:org.opennote/recording", + "data": { "recording": "r1" }, "fallback": { "markdown": "Audio recording" } } }, + { "edit": "setPage", "view": { "recordings": { "r1": entry } } } + ])), + ); + let done = json!({ "id": "r1", "state": "complete", "flags": null, "tracks": [] }); + let marks = json!({ "recordings": ["r1"], "marks": [{ + "from": 1, "to": 8, "recording": 0, "startNs": 5, "endNs": 9 + }] }); + apply( + &handle, + 3, + edits(json!([ + { "edit": "setPage", "view": { "recordings": { "r1": done } } }, + { "edit": "patchBlock", "block": "01k6f00000000000000000b001", "data": { "marks": marks } } + ])), + ); + Ok(()) + }) + .expect("the first run"); + bridge.shutdown(); + let again = CoreBridge::at(local); + let page_json = again + .notes(Some(notes), |bridge| { + let envelope = bridge.handle(&page, "main-1")?.envelope(None).map_err(internal)?; + let decoded = opennote_core::wire::envelope::decode(&envelope.bytes).map_err(internal)?; + serde_json::from_slice::(decoded.page_json).map_err(internal) + }) + .expect("the second run"); + again.shutdown(); + let blocks = &page_json["blocks"]; + assert_eq!(blocks[0]["data"]["markdown"], "Osmosis"); + assert_eq!(blocks[0]["data"]["marks"]["marks"][0]["startNs"], 5); + assert_eq!(blocks[1]["type"], "ext:org.opennote/recording"); + assert_eq!(blocks[1]["data"]["recording"], "r1"); + assert_eq!(page_json["view"]["recordings"]["r1"]["state"], "complete"); + assert!( + page_json["view"]["recordings"]["r1"].get("flags").is_none(), + "null removes a member" + ); +} diff --git a/app/src-tauri/src/deeplink.rs b/app/src-tauri/src/deeplink.rs new file mode 100644 index 00000000..2aa0c702 --- /dev/null +++ b/app/src-tauri/src/deeplink.rs @@ -0,0 +1,153 @@ +//! `opennote://` links (Phase 8, Links to paragraphs). A link names a page, and a heading or paragraph in it, by +//! IDs, so it survives renames and moves. Windows starts `OpenNote.exe "opennote://page/"` when a person +//! follows one from Outlook, Word, Teams, or a browser. A second launch reaches the running window through the +//! single-instance forwarding (`window://forwarded-args`); the first launch's link waits here until the interface +//! asks for it with the `launchLink` search method. A link only ever opens a page: this module reads nothing else +//! from it. +//! +//! The protocol is registered for the current user (`HKCU\Software\Classes\opennote`), not for the machine. +//! Only a release build registers it, and only when it does not run from a portable folder. A development build +//! never takes the link away from the installed app, and a portable copy leaves nothing behind. + +use std::sync::Mutex; + +/// The scheme, with its separator. +const PREFIX: &str = "opennote://"; + +/// The longest link accepted. A longer one is not a link this app made. +const MAX_LINK_LEN: usize = 512; + +static LAUNCH: Mutex> = Mutex::new(None); + +/// Whether an argument is an OpenNote link: the scheme, then only characters a page or element ID link uses. +pub fn is_link(arg: &str) -> bool { + let arg = arg.trim(); + arg.len() <= MAX_LINK_LEN + && arg.len() > PREFIX.len() + && arg[..PREFIX.len()].eq_ignore_ascii_case(PREFIX) + && arg[PREFIX.len()..] + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '-' | '_' | '%' | '#' | '?' | '=' | '&' | '.')) +} + +/// The first OpenNote link among the arguments. +pub fn find_link(args: &[String]) -> Option { + args.iter().find(|arg| is_link(arg)).map(|arg| arg.trim().to_owned()) +} + +/// Keeps the link a first launch was started with, for the interface to take once. +pub fn remember_launch(args: &[String]) { + if let Some(link) = find_link(args) { + *LAUNCH.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(link); + } +} + +/// The link the app was started with. It is given out once. +pub fn take_launch() -> Option { + LAUNCH.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take() +} + +/// Registers `opennote://` for the current user so Windows starts this exe for it. Best effort: a failure is +/// logged and the app goes on. +pub fn register_protocol() { + #[cfg(all(windows, not(debug_assertions)))] + if let Err(error) = registry::register() { + log::warn!("Couldn't register opennote:// links: {error}"); + } +} + +#[cfg(all(windows, not(debug_assertions)))] +mod registry { + use windows::{ + core::PCWSTR, + Win32::System::Registry::{ + RegCloseKey, RegCreateKeyExW, RegSetValueExW, HKEY, HKEY_CURRENT_USER, KEY_WRITE, REG_OPTION_NON_VOLATILE, + REG_SZ, + }, + }; + + fn wide(text: &str) -> Vec { + text.encode_utf16().chain(std::iter::once(0)).collect() + } + + fn set(path: &str, name: Option<&str>, value: &str) -> Result<(), String> { + let path = wide(path); + let value = wide(value); + let name = name.map(wide); + let mut key = HKEY::default(); + // SAFETY: the strings are NUL-terminated and outlive the calls, and the key is closed before returning. + unsafe { + RegCreateKeyExW( + HKEY_CURRENT_USER, + PCWSTR(path.as_ptr()), + None, + PCWSTR::null(), + REG_OPTION_NON_VOLATILE, + KEY_WRITE, + None, + &mut key, + None, + ) + .ok() + .map_err(|error| error.to_string())?; + let bytes = std::slice::from_raw_parts(value.as_ptr().cast::(), value.len() * 2); + let result = RegSetValueExW( + key, + name.as_ref().map_or(PCWSTR::null(), |name| PCWSTR(name.as_ptr())), + None, + REG_SZ, + Some(bytes), + ); + let _ = RegCloseKey(key); + result.ok().map_err(|error| error.to_string()) + } + } + + pub(super) fn register() -> Result<(), String> { + let exe = std::env::current_exe().map_err(|error| error.to_string())?; + if exe.parent().is_some_and(|dir| dir.join("portable").exists()) { + return Ok(()); + } + let command = format!("\"{}\" \"%1\"", exe.display()); + let root = "Software\\Classes\\opennote"; + set(root, None, "URL:OpenNote link")?; + set(root, Some("URL Protocol"), "")?; + set(&format!("{root}\\shell\\open\\command"), None, &command) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_page_and_paragraph_links() { + assert!(is_link("opennote://page/01hzx3k9q7m2v5c8d4e6f0abcd")); + assert!(is_link("OpenNote://page/abc/def?x=1#y")); + assert!(is_link( + "opennote://clip?url=https%3A%2F%2Fexample.org%2Fa&title=A%20page" + )); + } + + #[test] + fn accepts_section_and_notebook_links() { + assert!(is_link("opennote://section/01hzx3k9q7m2v5c8d4e6f0abcd")); + assert!(is_link("opennote://notebook/01hzx3k9q7m2v5c8d4e6f0abcd")); + } + + #[test] + fn refuses_everything_else() { + assert!(!is_link("https://example.com")); + assert!(!is_link("opennote://")); + assert!(!is_link("opennote://page/a b")); + assert!(!is_link("opennote://page/\"; calc")); + assert!(!is_link(&format!("opennote://{}", "a".repeat(600)))); + } + + #[test] + fn finds_the_link_among_arguments() { + let args = vec!["--x".to_owned(), " opennote://page/abc ".to_owned()]; + assert_eq!(find_link(&args).as_deref(), Some("opennote://page/abc")); + assert_eq!(find_link(&["notes.onepkg".to_owned()]), None); + } +} diff --git a/app/src-tauri/src/early.rs b/app/src-tauri/src/early.rs index c9b2cb29..a93b1280 100644 --- a/app/src-tauri/src/early.rs +++ b/app/src-tauri/src/early.rs @@ -10,7 +10,7 @@ use crate::{ args::{self, Args}, boot, instance::{self, InstanceGuard, InstanceOutcome}, - paths::{Paths, PROFILE_DIR_VAR}, + paths::{self, Paths, PROFILE_DIR_VAR}, perf, updater::{self, GuardOutcome}, window::webview::{self, RuntimeCheck}, @@ -61,7 +61,13 @@ impl Steps for System { } fn resolve_paths(&self) -> std::io::Result { - let paths = Paths::resolve(std::env::var_os(PROFILE_DIR_VAR).map(Into::into))?; + let program_dir = std::env::current_exe() + .ok() + .and_then(|exe| exe.parent().map(std::path::Path::to_path_buf)); + let paths = Paths::resolve(paths::profile_override( + std::env::var_os(PROFILE_DIR_VAR).map(Into::into), + program_dir.as_deref(), + ))?; // Logging starts as soon as it has a folder, so the steps after this one are on record. let profile = std::env::var_os("USERPROFILE") .map(std::path::PathBuf::from) @@ -94,6 +100,10 @@ pub fn run() -> EarlyOutcome { if let Some(marker) = updater::TEST_ENDPOINTS_MARKER { log::warn!("{marker}: this build accepts test update endpoints and a test key."); } + // `--uninstall` from Installed apps, and the copy that finishes it, never start the app. + if let Some(code) = crate::install::uninstall::run_from_args() { + return EarlyOutcome::Exit(code); + } run_with(&System) } @@ -110,6 +120,9 @@ pub fn run_with(steps: &impl Steps) -> EarlyOutcome { return EarlyOutcome::Exit(EXIT_FAILURE); } }; + // A "Share to OpenNote" launch saves what was shared first, because a second launch exits at the lock below. + // Any other launch returns at once. + crate::share::receive_launch(&paths); let instance = match steps.acquire_instance(&paths, &args) { InstanceOutcome::Owner(guard) => guard, InstanceOutcome::Forwarded => return EarlyOutcome::Exit(0), diff --git a/app/src-tauri/src/hardening/commands.rs b/app/src-tauri/src/hardening/commands.rs new file mode 100644 index 00000000..88d0be40 --- /dev/null +++ b/app/src-tauri/src/hardening/commands.rs @@ -0,0 +1,446 @@ +//! The commands behind `DiagnosticsClient` (app/src/features/diagnostics/client.ts). Each wraps one call into +//! `crates/crashreport` or `crates/diagnostics`. Errors carry a code and a fixed sentence, never text from a +//! report, so a failure can't leak what the report would have held. + +use opennote_core::{session::notebook::NotebookHandle, store::verify::VerifyReport, CoreError}; +use opennote_crashreport::{prepare, send, Consent, Report, SendError, StoreError, Summary, Transport}; +use opennote_diagnostics::{ + selfcheck::{self, Inputs}, + Bundle, BundleError, BundleInputs, BundleOptions, NotebookCounts, SelfCheck, SessionStats, StartReport, SystemDisk, + SystemFacts, +}; +use std::sync::PoisonError; + +use serde::{Deserialize, Serialize}; +use tauri::{AppHandle, Manager, State, WebviewWindow}; + +use super::{now_unix, transport::HttpTransport, Hardening}; +use crate::{ + boot::channel_for, + core_bridge::CoreBridge, + ipc::{codes, IpcError, IpcResult}, + lifecycle::{request_exit, ExitReason, ExitState, Relaunch}, + paths::Paths, +}; + +/// Error codes of `DiagnosticsError` in the interface. +mod errors { + pub const NOT_OPTED_IN: &str = "notOptedIn"; + pub const NO_ADDRESS: &str = "noAddress"; + pub const MISSING: &str = "missing"; + pub const NOT_REVIEWED: &str = "notReviewed"; + pub const CANCELED: &str = "canceled"; + pub const OFFLINE: &str = "offline"; + pub const IO: &str = "io"; +} + +fn failure(code: &str) -> IpcError { + IpcError::new(code, code) +} + +pub(super) fn send_error(error: &SendError) -> IpcError { + failure(match error { + SendError::NotOptedIn => errors::NOT_OPTED_IN, + SendError::NotConfigured | SendError::InvalidEndpoint => errors::NO_ADDRESS, + SendError::NotReviewed | SendError::WrongAgreement => errors::NOT_REVIEWED, + SendError::Store(StoreError::BadId(_) | StoreError::Missing(_)) => errors::MISSING, + SendError::Store(_) | SendError::Transport(_) => errors::IO, + }) +} + +async fn blocking(work: impl FnOnce() -> IpcResult + Send + 'static) -> IpcResult { + tauri::async_runtime::spawn_blocking(work) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))? +} + +#[tauri::command] +pub fn crash_consent_get(hardening: State<'_, Hardening>) -> IpcResult { + Ok(hardening.consent()) +} + +/// Stores the person's yes or no. A yes turns saving on; a no turns it off. Nothing is sent either way. +#[tauri::command] +pub fn crash_consent_set(hardening: State<'_, Hardening>, consent: Consent) -> IpcResult<()> { + hardening + .update(|privacy| privacy.crash_reports.consent = consent) + .map(|_| ()) + .map_err(|_| failure(errors::IO)) +} + +/// A made-up report in the real format, for the consent screen. +#[tauri::command] +pub fn crash_example() -> IpcResult { + Ok(Report::example(env!("CARGO_PKG_VERSION"), now_unix())) +} + +#[tauri::command] +pub fn crash_list(hardening: State<'_, Hardening>) -> IpcResult> { + Ok(hardening.store.list()) +} + +/// What the review shows: the exact text that would be sent, where it would go, and its digest. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Pending { + id: String, + endpoint: String, + payload: String, + digest: String, +} + +fn prepared(hardening: &Hardening, id: &str) -> Result { + let settings = hardening.privacy().crash_reports; + let scrubber = hardening + .scrubber + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + prepare(&hardening.store, id, &settings, &scrubber) +} + +/// Reads one report for review. Nothing is sent. +#[tauri::command] +pub fn crash_prepare(hardening: State<'_, Hardening>, id: String) -> IpcResult { + let pending = prepared(&hardening, &id).map_err(|error| send_error(&error))?; + Ok(Pending { + id: pending.id().to_owned(), + endpoint: pending.endpoint().to_owned(), + payload: pending.payload().to_owned(), + digest: pending.digest().to_owned(), + }) +} + +/// Sends a report the person reviewed. The report is read again, so what goes out is the text with the digest +/// the person saw, or nothing. +pub fn send_reviewed(hardening: &Hardening, transport: &dyn Transport, id: &str, digest: &str) -> IpcResult<()> { + if super::offline() { + return Err(failure(errors::OFFLINE)); + } + let pending = prepared(hardening, id).map_err(|error| send_error(&error))?; + let agreement = pending.agree(digest).map_err(|error| send_error(&error))?; + send(&pending, agreement, transport).map_err(|error| { + ::log::warn!("A crash report was not sent: {error}"); + send_error(&error) + })?; + hardening + .update(|privacy| privacy.report_sent_unix = Some(now_unix())) + .map(|_| ()) + .map_err(|_| failure(errors::IO)) +} + +#[tauri::command] +pub async fn crash_send(app: AppHandle, id: String, digest: String) -> IpcResult<()> { + blocking(move || send_reviewed(&app.state::(), &HttpTransport, &id, &digest)).await +} + +#[tauri::command] +pub fn crash_delete(hardening: State<'_, Hardening>, id: String) -> IpcResult<()> { + hardening.store.delete(&id).map_err(|error| match error { + StoreError::BadId(_) | StoreError::Missing(_) => failure(errors::MISSING), + _ => failure(errors::IO), + }) +} + +#[tauri::command] +pub fn crash_delete_all(hardening: State<'_, Hardening>) -> IpcResult { + Ok(hardening.store.delete_all()) +} + +/// The open notebooks as one notebook for the self-check: the notes folder for the disk check, each notebook's own +/// check for the file check, and any unsaved change in any of them. +pub struct OpenNotebooks { + folder: std::path::PathBuf, + handles: Vec, +} + +impl OpenNotebooks { + /// `None` when no notebook is open. + pub fn new(handles: Vec) -> Option { + let first = handles.first()?; + let folder = first.path().parent().unwrap_or(first.path()).to_path_buf(); + Some(OpenNotebooks { folder, handles }) + } + + fn counts(&self) -> NotebookCounts { + let (mut sections, mut pages) = (0usize, 0usize); + for handle in &self.handles { + let tree = handle.tree(); + sections += tree.sections.len(); + pages += tree.sections.iter().map(|section| section.pages.len()).sum::(); + } + let count = |number: usize| u32::try_from(number).unwrap_or(u32::MAX); + NotebookCounts { + notebooks: count(self.handles.len()), + sections: count(sections), + pages: count(pages), + } + } +} + +impl selfcheck::NotebookProbe for OpenNotebooks { + fn path(&self) -> &std::path::Path { + &self.folder + } + + fn verify(&self) -> Result { + let mut total = VerifyReport::default(); + for handle in &self.handles { + let report = handle.verify()?; + total.files = total.files.saturating_add(report.files); + total.problems.extend(report.problems); + } + Ok(total) + } + + fn has_unsaved(&self) -> bool { + self.handles.iter().any(NotebookHandle::has_unsaved) + } +} + +/// The open notebooks, and their names registered as private so no report or file repeats them. +fn notebook(hardening: &Hardening, bridge: &CoreBridge) -> Option { + let open = OpenNotebooks::new(bridge.notebooks())?; + for handle in &open.handles { + let title = handle.tree().title; + if !title.is_empty() { + hardening + .scrubber + .lock() + .unwrap_or_else(PoisonError::into_inner) + .add_private(&title); + opennote_crashreport::add_private(&title); + } + } + Some(open) +} + +fn run_self_check(hardening: &Hardening, paths: &Paths, notebook: Option<&OpenNotebooks>) -> SelfCheck { + let probe = notebook.map(|open| open as &dyn selfcheck::NotebookProbe); + selfcheck::run( + &Inputs { + now_unix: now_unix(), + app_data_dir: &paths.local, + updates_dir: Some(&paths.updates), + notebook: probe, + crash_store: Some(&hardening.store), + }, + &SystemDisk, + ) +} + +/// Runs the self-check. It changes nothing. +#[tauri::command] +pub async fn diagnostics_self_check(app: AppHandle) -> IpcResult { + // `verify` reads every file of the notebook, so it runs off the command thread. + blocking(move || { + let hardening = app.state::(); + let handle = notebook(&hardening, &app.state::()); + Ok(run_self_check(&hardening, &app.state::(), handle.as_ref())) + }) + .await +} + +/// The text the person reviews, and what saves exactly that text. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BuiltFeedback { + bundle: Bundle, + review: ReviewText, +} + +impl BuiltFeedback { + /// The text that was built, for tests. + #[cfg(test)] + pub(super) fn review_text(&self) -> &str { + &self.review.text + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ReviewText { + text: String, + digest: String, + suggested_file_name: String, +} + +/// Collects the feedback file and keeps it until it is saved or replaced. `facts` is what only the interface +/// knows, such as the theme; the versions, channel, and counts come from here. +#[tauri::command] +pub async fn diagnostics_build_feedback( + app: AppHandle, + options: BundleOptions, + facts: Option, +) -> IpcResult { + blocking(move || { + let hardening = app.state::(); + let handle = notebook(&hardening, &app.state::()); + let paths = app.state::(); + let check = run_self_check(&hardening, &paths, handle.as_ref()); + let webview2 = app.state::().webview2_version.clone(); + Ok(build_feedback( + &hardening, + &paths, + &check, + handle.as_ref(), + &options, + facts, + &webview2, + )) + }) + .await +} + +fn channel_name() -> String { + let channel = channel_for( + env!("CARGO_PKG_VERSION"), + cfg!(debug_assertions) || cfg!(feature = "test-endpoints"), + ); + serde_json::to_value(channel) + .ok() + .and_then(|value| value.as_str().map(str::to_owned)) + .unwrap_or_default() +} + +/// Builds the bundle and keeps its review. The versions, channel, and counts are the host's; the rest of `facts` +/// is the interface's. +pub fn build_feedback( + hardening: &Hardening, + paths: &Paths, + check: &SelfCheck, + notebook: Option<&OpenNotebooks>, + options: &BundleOptions, + facts: Option, + webview2: &str, +) -> BuiltFeedback { + let mut facts = facts.unwrap_or_default(); + facts.app_version = env!("CARGO_PKG_VERSION").to_owned(); + facts.channel = channel_name(); + facts.webview2_version = Some(webview2.to_owned()).filter(|version| !version.is_empty()); + facts.notebooks = notebook.map(OpenNotebooks::counts); + let scrubber = hardening + .scrubber + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + let bundle = Bundle::build(&BundleInputs { + facts: &facts, + self_check: Some(check), + logs_dir: Some(&paths.logs), + crash_store: Some(&hardening.store), + scrubber: &scrubber, + now_unix: now_unix(), + options, + }); + let review = bundle.review(); + let text = ReviewText { + text: review.text().to_owned(), + digest: review.digest().to_owned(), + suggested_file_name: review.suggested_file_name().to_owned(), + }; + *hardening.review.lock().unwrap_or_else(PoisonError::into_inner) = Some(review); + BuiltFeedback { bundle, review: text } +} + +#[derive(Serialize)] +pub struct SavedFile { + name: String, +} + +/// Saves the last built file in a folder the person picks. Only the file name goes back to the interface. +#[tauri::command] +pub async fn diagnostics_save_feedback( + window: WebviewWindow, + hardening: State<'_, Hardening>, + digest: String, +) -> IpcResult { + let review = hardening + .review + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + .ok_or_else(|| failure(errors::NOT_REVIEWED))?; + if review.digest() != digest { + return Err(failure(errors::NOT_REVIEWED)); + } + let owner = window.hwnd().map(|hwnd| hwnd.0 as isize).unwrap_or(0); + let folder = blocking(move || crate::install::pick_folder(owner, None)).await?; + let Some(folder) = folder else { + return Err(failure(errors::CANCELED)); + }; + let path = review + .save(&digest, std::path::Path::new(&folder)) + .map_err(|error| match error { + BundleError::NotReviewed => failure(errors::NOT_REVIEWED), + BundleError::Io(_) => failure(errors::IO), + })?; + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + Ok(SavedFile { name }) +} + +/// What start-up learned from the session record. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SessionStartup { + report: StartReport, + safe_mode: bool, + stats: SessionStats, +} + +#[tauri::command] +pub fn diagnostics_startup(hardening: State<'_, Hardening>) -> IpcResult { + Ok(SessionStartup { + report: hardening.start_report(), + safe_mode: super::safe_mode(), + stats: hardening.stats(), + }) +} + +/// Starts this session in safe mode. Features that safe mode turns off read [`super::safe_mode`]. +#[tauri::command] +pub fn diagnostics_enter_safe_mode(hardening: State<'_, Hardening>) -> IpcResult<()> { + hardening.enter_safe_mode(); + Ok(()) +} + +/// The privacy choices the panel shows. +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrivacyState { + pub work_offline: bool, + /// Where saved crash reports may be sent. Empty means nowhere. + pub report_endpoint: String, + pub report_sent_unix: Option, +} + +#[tauri::command] +pub fn privacy_get(hardening: State<'_, Hardening>) -> IpcResult { + let privacy = hardening.privacy(); + Ok(PrivacyState { + work_offline: privacy.work_offline, + report_endpoint: privacy.crash_reports.endpoint, + report_sent_unix: privacy.report_sent_unix, + }) +} + +#[tauri::command] +pub fn privacy_set_offline(hardening: State<'_, Hardening>, offline: bool) -> IpcResult<()> { + hardening + .update(|privacy| privacy.work_offline = offline) + .map(|_| ()) + .map_err(|_| failure(errors::IO)) +} + +/// Closes OpenNote and opens it again, normally, so safe mode ends. The exit goes through the usual handshake. +#[tauri::command] +pub fn diagnostics_restart(app: AppHandle) -> IpcResult<()> { + let exe = std::env::current_exe()?; + app.state::() + .plan_relaunch(Relaunch { exe, args: Vec::new() }); + request_exit(&app, ExitReason::Close); + Ok(()) +} diff --git a/app/src-tauri/src/hardening/mod.rs b/app/src-tauri/src/hardening/mod.rs new file mode 100644 index 00000000..e0084f77 --- /dev/null +++ b/app/src-tauri/src/hardening/mod.rs @@ -0,0 +1,164 @@ +//! Hardening and the beta (Phase 13): the host side of crash reports, the self-check, the feedback file, safe start, +//! and Work offline. The rules live in `crates/crashreport` and `crates/diagnostics`; this module gives them their +//! place on disk, their commands, and their network transport. See docs/HARDENING.md. +//! +//! - **Crash reports.** The panic hook and the exception handler are installed first thing in [`crate::run`], and +//! save nothing until the person said yes to the current wording. The decision is in `privacy.json` in the local +//! data folder, with the address a report may be sent to (empty by default, so nothing can be sent). +//! - **Safe start.** The session record says how the last sessions ended. After two crashes in a row the interface +//! offers safe mode, and [`safe_mode`] is the one value every feature that safe mode turns off should read. +//! - **Work offline.** [`offline`] is the one value every network use should read. Update checks and downloads, web +//! images on paste, and sending a crash report all read it. + +mod commands; +mod transport; + +pub use commands::*; + +use std::{ + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + Mutex, PoisonError, + }, + time::{SystemTime, UNIX_EPOCH}, +}; + +use opennote_crashreport::{Config, Consent, CrashStore, Scrubber, Settings as CrashSettings}; +use opennote_diagnostics::{Review, SessionLog, StartReport}; +use serde::{Deserialize, Serialize}; + +use crate::{paths::Paths, settings::file::write_atomic}; + +static OFFLINE: AtomicBool = AtomicBool::new(false); +static SAFE_MODE: AtomicBool = AtomicBool::new(false); + +/// Whether the person turned on Work offline. While it is on, the app makes no network request. +pub fn offline() -> bool { + OFFLINE.load(Ordering::Relaxed) +} + +/// Whether this session runs in safe mode: no background work, embeds, or on-device models. +pub fn safe_mode() -> bool { + SAFE_MODE.load(Ordering::Relaxed) +} + +/// The file name of the privacy choices, in the local data folder. +pub const PRIVACY_FILE: &str = "privacy.json"; + +/// What the person chose about crash reports and the network, and when a report last went out. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct PrivacyFile { + pub crash_reports: CrashSettings, + pub work_offline: bool, + /// When a crash report was last sent, in seconds since 1970. For the Privacy panel. + pub report_sent_unix: Option, +} + +impl PrivacyFile { + /// Reads the file. A missing or damaged file reads as the defaults: no consent and online. + pub fn read(path: &std::path::Path) -> PrivacyFile { + std::fs::read_to_string(path) + .ok() + .and_then(|text| serde_json::from_str(&text).ok()) + .unwrap_or_default() + } +} + +pub fn now_unix() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +/// The managed state behind the diagnostics and privacy commands. +pub struct Hardening { + file: PathBuf, + pub(crate) store: CrashStore, + pub(crate) scrubber: Mutex, + privacy: Mutex, + session: SessionLog, + start: StartReport, + /// The feedback file that was built last, kept until it is saved or replaced. + pub(crate) review: Mutex>, +} + +impl Hardening { + /// Installs the crash hooks, records that a session began, and reads the privacy choices. Call it once, as + /// early as the folders are known and this copy holds the instance lock. + pub fn start(paths: &Paths) -> Hardening { + let hardening = Hardening::open(paths); + opennote_crashreport::install(Config { + app_version: env!("CARGO_PKG_VERSION").to_owned(), + store: hardening.store.clone(), + settings: hardening.privacy().crash_reports, + }); + hardening.store.sweep(); + hardening + } + + /// Reads the privacy choices and records that a session began, without installing any hook. + fn open(paths: &Paths) -> Hardening { + let file = paths.local.join(PRIVACY_FILE); + let privacy = PrivacyFile::read(&file); + OFFLINE.store(privacy.work_offline, Ordering::Relaxed); + let session = SessionLog::new(&paths.local); + let start = session.begin(now_unix()); + Hardening { + file, + store: CrashStore::new(paths.local.join("crashes")), + scrubber: Mutex::new(Scrubber::detect()), + privacy: Mutex::new(privacy), + session, + start, + review: Mutex::new(None), + } + } + + pub(crate) fn privacy(&self) -> PrivacyFile { + self.privacy.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + /// Changes the privacy choices and saves them. The hooks and the offline switch follow at once. + pub(crate) fn update(&self, change: impl FnOnce(&mut PrivacyFile)) -> std::io::Result { + let mut privacy = self.privacy.lock().unwrap_or_else(PoisonError::into_inner); + change(&mut privacy); + // A yes counts only while the switch is on, and a no turns it off. + privacy.crash_reports.enabled = privacy.crash_reports.consent.saving_allowed(); + opennote_crashreport::apply(&privacy.crash_reports); + OFFLINE.store(privacy.work_offline, Ordering::Relaxed); + let json = serde_json::to_vec_pretty(&*privacy).map_err(std::io::Error::other)?; + write_atomic(&self.file, &json)?; + Ok(privacy.clone()) + } + + pub(crate) fn consent(&self) -> Consent { + self.privacy().crash_reports.consent + } + + pub(crate) fn start_report(&self) -> StartReport { + self.start + } + + pub(crate) fn enter_safe_mode(&self) { + SAFE_MODE.store(true, Ordering::Relaxed); + if let Err(error) = self.session.enter_safe_mode() { + ::log::warn!("Couldn't record safe mode: {error}"); + } + } + + pub(crate) fn stats(&self) -> opennote_diagnostics::SessionStats { + self.session.stats() + } + + /// Records that the person quit. Call it on the way out; calling it twice is harmless. + pub fn end_clean(&self) { + if let Err(error) = self.session.end_clean() { + ::log::warn!("Couldn't record the end of the session: {error}"); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/app/src-tauri/src/hardening/tests.rs b/app/src-tauri/src/hardening/tests.rs new file mode 100644 index 00000000..bcb66223 --- /dev/null +++ b/app/src-tauri/src/hardening/tests.rs @@ -0,0 +1,284 @@ +use std::sync::Mutex; + +use opennote_crashreport::{Consent, Report, SendError, Settings, Transport}; +use opennote_diagnostics::{selfcheck, BundleOptions, FixedDisk}; + +use super::*; +use crate::hardening::commands::{build_feedback, send_error, send_reviewed}; + +/// The offline and safe mode switches are global, so tests that read or change them take turns. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn turn() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn hardening() -> (tempfile::TempDir, Hardening, Paths) { + let dir = tempfile::tempdir().expect("a temp folder"); + let paths = Paths::under_profile(dir.path()); + std::fs::create_dir_all(&paths.local).expect("the local folder"); + (dir, Hardening::open(&paths), paths) +} + +/// A transport that records what it was given, and can be told to fail. +#[derive(Default)] +struct Recorder { + sent: Mutex>, + fail: bool, +} + +impl Transport for Recorder { + fn post(&self, endpoint: &str, body: &[u8]) -> Result<(), String> { + if self.fail { + return Err("unreachable".to_owned()); + } + self.sent + .lock() + .unwrap() + .push((endpoint.to_owned(), String::from_utf8_lossy(body).into_owned())); + Ok(()) + } +} + +fn opted_in(hardening: &Hardening, endpoint: &str) -> String { + hardening + .update(|privacy| privacy.crash_reports = Settings::opted_in(10, endpoint)) + .expect("the choices are saved"); + hardening + .store + .save(&Report::example("1.0.0", 100)) + .expect("the report is saved") +} + +fn digest_of(hardening: &Hardening, id: &str) -> String { + let scrubber = hardening.scrubber.lock().unwrap().clone(); + opennote_crashreport::prepare(&hardening.store, id, &hardening.privacy().crash_reports, &scrubber) + .expect("a report that can be prepared") + .digest() + .to_owned() +} + +#[test] +fn a_missing_or_damaged_choices_file_reads_as_no_consent_and_online() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(PRIVACY_FILE); + assert_eq!(PrivacyFile::read(&path), PrivacyFile::default()); + std::fs::write(&path, "{ not json").unwrap(); + let read = PrivacyFile::read(&path); + assert!(!read.crash_reports.saving_allowed()); + assert!(!read.work_offline); +} + +#[test] +fn a_yes_turns_saving_on_and_a_no_turns_it_off_and_both_are_kept() { + let _turn = turn(); + let (_dir, hardening, paths) = hardening(); + hardening + .update(|p| p.crash_reports.consent = Consent::accepted(5)) + .unwrap(); + assert!(hardening.privacy().crash_reports.saving_allowed()); + assert!(PrivacyFile::read(&paths.local.join(PRIVACY_FILE)) + .crash_reports + .saving_allowed()); + hardening + .update(|p| p.crash_reports.consent = Consent::declined(6)) + .unwrap(); + assert!(!hardening.privacy().crash_reports.enabled); + assert!(!PrivacyFile::read(&paths.local.join(PRIVACY_FILE)) + .crash_reports + .saving_allowed()); +} + +#[test] +fn work_offline_is_kept_and_read_back_at_the_next_start() { + let _turn = turn(); + let (_dir, hardening, paths) = hardening(); + hardening.update(|p| p.work_offline = true).unwrap(); + assert!(offline()); + OFFLINE.store(false, Ordering::Relaxed); + let again = Hardening::open(&paths); + assert!(offline(), "the next start reads the switch from the file"); + again.update(|p| p.work_offline = false).unwrap(); + assert!(!offline()); +} + +#[test] +fn nothing_is_sent_without_a_yes_an_address_and_the_reviewed_digest() { + let _turn = turn(); + let (_dir, hardening, _paths) = hardening(); + let transport = Recorder::default(); + let id = hardening.store.save(&Report::example("1.0.0", 100)).unwrap(); + // No yes yet. + let refused = send_reviewed(&hardening, &transport, &id, "any").unwrap_err(); + assert_eq!(refused.code, "notOptedIn"); + // A yes, but no address. + hardening + .update(|p| p.crash_reports = Settings::opted_in(10, "")) + .unwrap(); + assert_eq!( + send_reviewed(&hardening, &transport, &id, "any").unwrap_err().code, + "noAddress" + ); + // An address, but not the digest of the text that was shown. + hardening + .update(|p| p.crash_reports = Settings::opted_in(10, "https://collector.example/v1")) + .unwrap(); + assert_eq!( + send_reviewed(&hardening, &transport, &id, "not the digest") + .unwrap_err() + .code, + "notReviewed" + ); + assert!( + transport.sent.lock().unwrap().is_empty(), + "the transport was never reached" + ); +} + +#[test] +fn a_reviewed_report_is_sent_once_to_the_address_the_person_saw_and_the_time_is_kept() { + let _turn = turn(); + let (_dir, hardening, _paths) = hardening(); + let transport = Recorder::default(); + let id = opted_in(&hardening, "https://collector.example/v1"); + let digest = digest_of(&hardening, &id); + send_reviewed(&hardening, &transport, &id, &digest).unwrap(); + let sent = transport.sent.lock().unwrap(); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].0, "https://collector.example/v1"); + assert!(sent[0].1.contains("0xc0000005")); + assert!(hardening.privacy().report_sent_unix.is_some()); + assert_eq!( + hardening.store.list().len(), + 1, + "the report stays saved until the person deletes it" + ); +} + +#[test] +fn a_failed_send_says_io_and_keeps_the_report() { + let _turn = turn(); + let (_dir, hardening, _paths) = hardening(); + let id = opted_in(&hardening, "https://collector.example/v1"); + let digest = digest_of(&hardening, &id); + let broken = Recorder { + fail: true, + ..Recorder::default() + }; + assert_eq!(send_reviewed(&hardening, &broken, &id, &digest).unwrap_err().code, "io"); + assert!(hardening.privacy().report_sent_unix.is_none()); + assert_eq!(hardening.store.list().len(), 1); +} + +#[test] +fn work_offline_stops_a_send_before_anything_is_prepared() { + let _turn = turn(); + let (_dir, hardening, _paths) = hardening(); + let transport = Recorder::default(); + let id = opted_in(&hardening, "https://collector.example/v1"); + let digest = digest_of(&hardening, &id); + hardening.update(|p| p.work_offline = true).unwrap(); + assert_eq!( + send_reviewed(&hardening, &transport, &id, &digest).unwrap_err().code, + "offline" + ); + assert!(transport.sent.lock().unwrap().is_empty()); + hardening.update(|p| p.work_offline = false).unwrap(); +} + +#[test] +fn the_real_transport_refuses_while_working_offline() { + let _turn = turn(); + OFFLINE.store(true, Ordering::Relaxed); + let result = super::transport::HttpTransport.post("http://127.0.0.1:9/", b"{}"); + OFFLINE.store(false, Ordering::Relaxed); + assert!(result.is_err()); +} + +#[test] +fn an_error_from_the_transport_never_carries_report_text() { + let mapped = send_error(&SendError::Transport("secret text".to_owned())); + assert_eq!(mapped.code, "io"); + assert!(!mapped.message.contains("secret")); +} + +#[test] +fn safe_mode_is_recorded_and_the_record_survives_a_restart() { + let _turn = turn(); + let (_dir, first, paths) = hardening(); + assert!(!first.start_report().offer_safe_mode); + // Two sessions that never ended cleanly. + let second = Hardening::open(&paths); + assert_eq!(second.start_report().crashes_in_a_row, 1); + let third = Hardening::open(&paths); + assert!( + third.start_report().offer_safe_mode, + "two crashes in a row offer safe mode" + ); + third.enter_safe_mode(); + assert!(safe_mode()); + SAFE_MODE.store(false, Ordering::Relaxed); + third.end_clean(); + let fourth = Hardening::open(&paths); + assert!(!fourth.start_report().offer_safe_mode, "a clean end resets the count"); + assert_eq!(fourth.stats().clean, 1); +} + +#[test] +fn the_feedback_file_holds_what_was_asked_and_saves_only_the_text_that_was_shown() { + let _turn = turn(); + let (dir, hardening, paths) = hardening(); + let check = selfcheck::run( + &selfcheck::Inputs { + now_unix: 1_790_000_000, + app_data_dir: &paths.local, + updates_dir: Some(&paths.updates), + notebook: None, + crash_store: Some(&hardening.store), + }, + &FixedDisk(10 << 30), + ); + let options = BundleOptions { + description: "The page jumps when I scroll".to_owned(), + include_logs: false, + ..BundleOptions::default() + }; + let built = build_feedback(&hardening, &paths, &check, None, &options, None, "130.0"); + assert!(built.review_text().contains("The page jumps when I scroll")); + assert!(built.review_text().contains("- Recent log lines: not included")); + assert!(!built.review_text().contains("=== Recent log lines ===")); + let review = hardening.review.lock().unwrap().clone().expect("the review is kept"); + let target = dir.path().join("saved"); + std::fs::create_dir_all(&target).unwrap(); + assert!(review.save("not the digest", &target).is_err()); + let path = review.save(review.digest(), &target).unwrap(); + assert_eq!(std::fs::read_to_string(path).unwrap(), review.text()); +} + +#[test] +fn the_self_check_reads_every_notebook_of_the_notes_folder() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (local, notes) = (dir.path().join("local"), dir.path().join("Notes")); + let bridge = crate::core_bridge::CoreBridge::at(local); + bridge + .notes(Some(notes.clone()), |bridge| { + for title in ["Biology", "Cooking"] { + let input = serde_json::json!({ "kind": "notebook", "placement": { "parentId": null, "beforeId": null }, "title": title }); + bridge.dispatch("notes_create", &serde_json::json!({ "input": input }))?; + } + Ok(()) + }) + .expect("two notebooks"); + let open = crate::hardening::commands::OpenNotebooks::new(bridge.notebooks()).expect("open notebooks"); + let probe: &dyn selfcheck::NotebookProbe = &open; + assert_eq!( + probe.path(), + notes.as_path(), + "the notes folder is what the disk check looks at" + ); + let report = probe.verify().expect("the notebooks verify"); + assert!(report.is_clean(), "{:?}", report.problems); + assert!(report.files >= 2, "both notebooks were read: {}", report.files); + assert!(!probe.has_unsaved()); + bridge.shutdown(); +} diff --git a/app/src-tauri/src/hardening/transport.rs b/app/src-tauri/src/hardening/transport.rs new file mode 100644 index 00000000..be1343f3 --- /dev/null +++ b/app/src-tauri/src/hardening/transport.rs @@ -0,0 +1,53 @@ +//! Sending a crash report the person reviewed (docs/HARDENING.md, "How a report is sent"). The crashreport crate +//! has no network code; this is the only place a report can leave the computer. It runs only from +//! `crash_send`, after the person agreed to the exact text, and never while Work offline is on. + +use std::time::Duration; + +use opennote_crashreport::Transport; +use ureq::Agent; + +/// How long a send may take, in all. +const TIMEOUT: Duration = Duration::from_secs(20); + +/// Posts the report as JSON with `ureq` and Windows' TLS stack, as the updater does. No cookies, no `Referer`, +/// no proxy, and no redirects: a report goes to the address the person saw, and nowhere else. +pub struct HttpTransport; + +fn agent() -> Agent { + use ureq::tls::{RootCerts, TlsConfig, TlsProvider}; + let tls = TlsConfig::builder() + .provider(TlsProvider::NativeTls) + .root_certs(RootCerts::PlatformVerifier) + .build(); + Agent::config_builder() + .tls_config(tls) + .proxy(None) + .max_redirects(0) + .max_redirects_will_error(false) + .http_status_as_error(false) + .timeout_global(Some(TIMEOUT)) + .user_agent(concat!("OpenNote/", env!("CARGO_PKG_VERSION"))) + .build() + .into() +} + +impl Transport for HttpTransport { + fn post(&self, endpoint: &str, body: &[u8]) -> Result<(), String> { + if super::offline() { + return Err("Work offline is on.".to_owned()); + } + // The messages say what went wrong without repeating the address or any of the report. + let response = agent() + .post(endpoint) + .header("Content-Type", "application/json") + .send(body) + .map_err(|_| "The collector could not be reached.".to_owned())?; + let status = response.status().as_u16(); + if (200..300).contains(&status) { + Ok(()) + } else { + Err(format!("The collector answered with status {status}.")) + } + } +} diff --git a/app/src-tauri/src/images/header.rs b/app/src-tauri/src/images/header.rs index 2ec8d627..4b35d75a 100644 --- a/app/src-tauri/src/images/header.rs +++ b/app/src-tauri/src/images/header.rs @@ -31,6 +31,8 @@ pub fn sniff(bytes: &[u8]) -> ImageFormat { format } else if looks_like_svg(bytes) { ImageFormat::Svg + } else if opennote_metafile::is_metafile(bytes) { + ImageFormat::Metafile } else { ImageFormat::Other } @@ -85,7 +87,7 @@ pub fn read(format: ImageFormat, bytes: &[u8]) -> Option { ImageFormat::Webp => webp(bytes), ImageFormat::Tiff => tiff(bytes, true), ImageFormat::Avif | ImageFormat::Heic => isobmff(bytes), - ImageFormat::Svg | ImageFormat::Other => None, + ImageFormat::Svg | ImageFormat::Metafile | ImageFormat::Other => None, }?; (size.width > 0 && size.height > 0).then_some(size) } diff --git a/app/src-tauri/src/images/import.rs b/app/src-tauri/src/images/import.rs index f1908ccd..a8ca9833 100644 --- a/app/src-tauri/src/images/import.rs +++ b/app/src-tauri/src/images/import.rs @@ -85,6 +85,20 @@ pub fn check(bytes: Vec, name: &str, declared: &str, convert: bool) -> Resul if !generic && ImageFormat::from_mime(declared) != Some(probed.format) { return Err(unsupported()); } + if probed.format == ImageFormat::Metafile { + // WebView2 cannot show a metafile, so the page keeps the drawing as SVG. + let svg = opennote_metafile::parse(&bytes) + .ok() + .and_then(|picture| opennote_metafile::to_svg(&picture)) + .ok_or_else(unsupported)?; + return Ok(Checked { + name: name_for(name, ImageFormat::Svg), + mime: ImageFormat::Svg.mime(), + width: probed.width, + height: probed.height, + bytes: svg.into_bytes(), + }); + } if probed.format.displayable() { return Ok(Checked { name: name_for(name, probed.format), @@ -342,4 +356,53 @@ mod tests { assert_eq!(percent_decode("Fot%C3%B3%20%25.png"), "Fotó %.png"); assert_eq!(percent_decode("100%"), "100%"); } + + /// A small EMF: a 400 by 200 device-unit frame with one black rectangle. + fn emf() -> Vec { + let ints = |values: &[i32]| values.iter().flat_map(|v| v.to_le_bytes()).collect::>(); + let record = |kind: u32, payload: &[u8]| { + let mut out = kind.to_le_bytes().to_vec(); + out.extend(((8 + payload.len()) as u32).to_le_bytes()); + out.extend(payload); + out + }; + let mut body = record(37, &0x8000_0004u32.to_le_bytes()); + body.extend(record(43, &ints(&[40, 20, 360, 180]))); + body.extend(record(14, &[0; 12])); + let mut header = vec![0u8; 88]; + header[0..4].copy_from_slice(&1u32.to_le_bytes()); + header[4..8].copy_from_slice(&88u32.to_le_bytes()); + header[8..24].copy_from_slice(&ints(&[0, 0, 399, 199])); + header[24..40].copy_from_slice(&ints(&[0, 0, 4000, 2000])); + header[40..44].copy_from_slice(&0x464D_4520u32.to_le_bytes()); + header[44..48].copy_from_slice(&0x0001_0000u32.to_le_bytes()); + header[48..52].copy_from_slice(&((88 + body.len()) as u32).to_le_bytes()); + header[52..56].copy_from_slice(&3u32.to_le_bytes()); + header[56..58].copy_from_slice(&1u16.to_le_bytes()); + header[72..76].copy_from_slice(&400i32.to_le_bytes()); + header[76..80].copy_from_slice(&200i32.to_le_bytes()); + header[80..84].copy_from_slice(&40i32.to_le_bytes()); + header[84..88].copy_from_slice(&20i32.to_le_bytes()); + header.extend(body); + header + } + + #[test] + fn a_metafile_becomes_an_svg_and_a_damaged_or_hostile_one_is_refused() { + let checked = check(emf(), "Chart.emf", "image/x-emf", false).unwrap(); + assert_eq!((checked.mime, checked.name.as_str()), ("image/svg+xml", "Chart.svg")); + assert!(checked.width > 0 && checked.height > 0); + assert!(String::from_utf8(checked.bytes).unwrap().starts_with(" "image/svg+xml", ImageFormat::Heic => "image/heic", ImageFormat::Tiff => "image/tiff", + ImageFormat::Metafile => "image/emf", ImageFormat::Other => "application/octet-stream", } } @@ -48,6 +51,7 @@ impl ImageFormat { ImageFormat::Svg => "svg", ImageFormat::Heic => "heic", ImageFormat::Tiff => "tif", + ImageFormat::Metafile => "emf", ImageFormat::Other => "bin", } } @@ -65,6 +69,9 @@ impl ImageFormat { "image/svg+xml" => ImageFormat::Svg, "image/heic" | "image/heif" | "image/heic-sequence" | "image/heif-sequence" => ImageFormat::Heic, "image/tiff" | "image/tif" => ImageFormat::Tiff, + "image/emf" | "image/x-emf" | "image/wmf" | "image/x-wmf" | "application/x-msmetafile" => { + ImageFormat::Metafile + } _ => return None, }) } @@ -119,6 +126,15 @@ pub fn probe(bytes: &[u8]) -> Result { orientation: 1, }) } + ImageFormat::Metafile => { + let (width, height) = metafile_size(bytes).ok_or(ProbeError::Malformed(format))?; + Ok(Probe { + format, + width, + height, + orientation: 1, + }) + } _ => { let raw = header::read(format, bytes); let checked = confirm(bytes, raw.map(|raw| (raw.width, raw.height))); @@ -135,6 +151,16 @@ pub fn probe(bytes: &[u8]) -> Result { } } +/// The size of a metafile's drawing in CSS pixels (its points at 96 per inch), as the SVG made from it states. +pub fn metafile_size(bytes: &[u8]) -> Option<(u32, u32)> { + let picture = opennote_metafile::parse(bytes).ok()?; + let px = |points: f32| { + let px = (points * 96.0 / 72.0).round(); + (px.is_finite() && (1.0..=100_000.0).contains(&px)).then_some(px as u32) + }; + Some((px(picture.width)?, px(picture.height)?)) +} + /// The stored size as WIC reads it, when a codec is installed for the format, else the header's. #[cfg(windows)] fn confirm(bytes: &[u8], header: Option<(u32, u32)>) -> Option<(u32, u32)> { @@ -228,7 +254,8 @@ mod tests { ImageFormat::from_mime("image/svg+xml; charset=utf-8"), Some(ImageFormat::Svg) ); - assert_eq!(ImageFormat::from_mime("image/x-emf"), None); + assert_eq!(ImageFormat::from_mime("image/x-emf"), Some(ImageFormat::Metafile)); + assert!(!ImageFormat::Metafile.displayable()); assert!(ImageFormat::Avif.displayable()); assert!(!ImageFormat::Heic.displayable() && !ImageFormat::Tiff.displayable()); } diff --git a/app/src-tauri/src/images/protocol.rs b/app/src-tauri/src/images/protocol.rs index e819fe92..b6335a6a 100644 --- a/app/src-tauri/src/images/protocol.rs +++ b/app/src-tauri/src/images/protocol.rs @@ -74,6 +74,8 @@ fn answer(app: &AppHandle, request: &Request>) -> Response> { .header(header::CACHE_CONTROL, "private, max-age=31536000, immutable") .header(header::ACCEPT_RANGES, "bytes") .header("X-Content-Type-Options", "nosniff") + // The page reads an image's pixels for text recognition (Phase 12), which is a cross-origin fetch. + .header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") .header( header::CONTENT_SECURITY_POLICY, "default-src 'none'; style-src 'unsafe-inline'; sandbox", diff --git a/app/src-tauri/src/images/web.rs b/app/src-tauri/src/images/web.rs index 4758e880..d48372df 100644 --- a/app/src-tauri/src/images/web.rs +++ b/app/src-tauri/src/images/web.rs @@ -107,11 +107,35 @@ impl Resolver for GuardedResolver { } } +/// The system resolver that keeps only public addresses, for any other request that must stay off the local +/// network, such as a webhook delivery. +pub(crate) fn public_resolver() -> impl Resolver { + GuardedResolver { + inner: DefaultResolver::default(), + policy: Policy { allow_loopback: false }, + } +} + +/// The certificate a test's local HTTPS server uses, which downloads in tests trust instead of the Windows store. +#[cfg(test)] +static TEST_ROOT: std::sync::OnceLock> = std::sync::OnceLock::new(); + +fn roots() -> ureq::tls::RootCerts { + #[cfg(test)] + if let Some(der) = TEST_ROOT.get() { + let cert = ureq::tls::Certificate::from_der(der).to_owned(); + return ureq::tls::RootCerts::Specific(std::sync::Arc::new(vec![cert])); + } + ureq::tls::RootCerts::PlatformVerifier +} + fn agent(policy: Policy) -> Agent { - use ureq::tls::{RootCerts, TlsConfig, TlsProvider}; + use ureq::tls::{TlsConfig, TlsProvider}; + // The provider's own feature must be on: ureq panics on an https address otherwise, and a release build aborts on + // a panic (T1-1). The https tests below fail the same way if it is ever turned off. let tls = TlsConfig::builder() .provider(TlsProvider::NativeTls) - .root_certs(RootCerts::PlatformVerifier) + .root_certs(roots()) .build(); let config = Agent::config_builder() .tls_config(tls) @@ -250,6 +274,55 @@ fn download(url: &str, policy: Policy) -> Result { Err(failed("The image's address redirects too many times.")) } +/// The most of a page that is read to find its title. +const HEAD_LIMIT: u64 = 256 * 1024; + +/// The first 256 KB of a web page, for its title (Link titles on paste). It follows the same rules as an image +/// download: the system's TLS, at most 3 redirects on one scheme, no cookies, no `Referer`, no proxy, and no +/// address on the local network. Only HTML answers count. +pub(crate) fn fetch_html_head(url: &str, allow_loopback: bool) -> Result, IpcError> { + let agent = agent(Policy { allow_loopback }); + let mut current = parse_url(url)?; + for _ in 0..=MAX_REDIRECTS { + let response = agent + .get(&url_text(¤t)) + .header("Accept", "text/html,application/xhtml+xml") + .call() + .map_err(|error| failed(format!("The page couldn't be reached: {error}")))?; + let status = response.status().as_u16(); + if (300..400).contains(&status) { + let location = response + .headers() + .get("location") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| failed("The page's redirect has no address."))?; + current = redirect(¤t, location)?; + continue; + } + if status != 200 { + return Err(failed(format!("The page's server answered {status}."))); + } + let content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_ascii_lowercase(); + if !(content_type.contains("text/html") || content_type.contains("application/xhtml")) { + return Err(failed("The address isn't a web page.")); + } + let mut bytes = Vec::new(); + response + .into_body() + .into_reader() + .take(HEAD_LIMIT) + .read_to_end(&mut bytes) + .map_err(|error| failed(format!("The page stopped loading: {error}")))?; + return Ok(bytes); + } + Err(failed("The page redirects too many times.")) +} + /// The last part of the address's path, as the asset's original name. fn name_from(uri: &Uri) -> String { let last = uri.path().rsplit('/').next().unwrap_or(""); @@ -274,6 +347,10 @@ fn fetch_checked(url: &str, policy: Policy, convert: bool) -> Result IpcResult { + // Work offline blocks every network use, including this one (docs/FEATURES.md, Privacy panel). + if crate::hardening::offline() { + return Err(failed("Work offline is on.")); + } let convert = convert_enabled(&app); on_blocking(move || { let handle = open_page(&app, &page)?; @@ -352,6 +429,86 @@ mod tests { std::fs::read(format!("{FIXTURES}/png.png")).unwrap() } + /// A local HTTPS server that answers each request path with a canned response, like [`serve`]. Its certificate, + /// for 127.0.0.1, is made once per run, and the downloads trust it. + fn serve_tls(routes: Vec<(&'static str, Vec)>) -> String { + static KEY: std::sync::OnceLock<(Vec, Vec)> = std::sync::OnceLock::new(); + let (cert, key) = KEY.get_or_init(|| { + let made = rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_owned()]).expect("a certificate"); + (made.cert.der().to_vec(), made.key_pair.serialize_der()) + }); + TEST_ROOT.get_or_init(|| cert.clone()); + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![rustls::pki_types::CertificateDer::from(cert.clone())], + rustls::pki_types::PrivatePkcs8KeyDer::from(key.clone()).into(), + ) + .expect("a server configuration"); + let config = Arc::new(config); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("https://{}", listener.local_addr().unwrap()); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { break }; + let Ok(connection) = rustls::ServerConnection::new(config.clone()) else { + break; + }; + let mut tls = rustls::StreamOwned::new(connection, stream); + let mut reader = BufReader::new(&mut tls); + let mut head = String::new(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 || line == "\r\n" { + break; + } + head.push_str(&line); + } + let path = head.split_whitespace().nth(1).unwrap_or("/").to_owned(); + let response = routes + .iter() + .find(|(route, _)| *route == path) + .map(|(_, response)| response.clone()) + .unwrap_or_else(|| b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n".to_vec()); + let _ = tls.write_all(&response); + tls.conn.send_close_notify(); + let _ = tls.flush(); + } + }); + base + } + + #[test] + fn an_image_downloads_over_https() { + // T1-1: the TLS stack the app ships, end to end. ureq panicked on every https address before. + let base = serve_tls(vec![ + ("/pic.png", response("image/png", &png())), + ( + "/page", + response("text/html", b"T"), + ), + ]); + let downloaded = download(&format!("{base}/pic.png"), TEST).expect("downloads"); + assert_eq!(downloaded.bytes, png()); + let head = fetch_html_head(&format!("{base}/page"), true).expect("fetches"); + assert!(String::from_utf8_lossy(&head).contains("T")); + } + + #[test] + fn an_https_address_fails_cleanly_rather_than_panicking() { + // The pasted picture of T1-1: ureq panicked on every https address because its TLS provider's feature + // wasn't on, and a release build aborts on a panic. A server that doesn't speak TLS must give an error. + let server = serve(vec![("/pic.png", response("image/png", &png()))]); + let https = server.base.replacen("http://", "https://", 1); + let error = match download(&format!("{https}/pic.png"), TEST) { + Ok(_) => panic!("a TLS handshake with a plain server can't succeed"), + Err(error) => error, + }; + assert_eq!(error.code, errors::DOWNLOAD_FAILED, "{}", error.message); + let error = fetch_html_head(&format!("{https}/page"), true).expect_err("refused"); + assert_eq!(error.code, errors::DOWNLOAD_FAILED, "{}", error.message); + } + #[test] fn an_image_downloads_and_passes_the_probe() { let server = serve(vec![ diff --git a/app/src-tauri/src/ink_bridge.rs b/app/src-tauri/src/ink_bridge.rs new file mode 100644 index 00000000..dff26d7f --- /dev/null +++ b/app/src-tauri/src/ink_bridge.rs @@ -0,0 +1,108 @@ +//! Phase 5's ink commands over the core bridge. New strokes arrive as binary records with the edits that go before +//! them in the same transaction (`page_add_strokes`), and a page whose envelope held only the strokes near the +//! viewport reads the rest at once (`page_read_strokes`). +//! +//! Both commands work on the client's open session of a page, which `page_open` made from the page's folder in its +//! notebook. So the strokes go to that folder's ink segments, with the page: they survive a restart, travel with a +//! copied notes folder, and go to Trash and back with their page. +//! +//! A `page_add_strokes` body is a `u32` length, little-endian, then that many bytes of JSON (`StrokesHeader`), and +//! then the stroke records in the segment record format. The JSON rides in the body rather than a header, so a +//! partial erase that removes many strokes never runs into a header size limit. + +use opennote_core::{ + ops::{ + resolve::{Edit, StrokeTxnMeta}, + CoalesceKey, + }, + session::page::{PageHandle, TxnAck}, +}; +use serde::Deserialize; +use serde_json::Value; +use tauri::{ + ipc::{InvokeBody, Response}, + State, +}; + +use crate::{ + core_bridge::CoreBridge, + ipc::{IpcError, IpcResult}, +}; + +/// The JSON part of a `page_add_strokes` body: the interface's page, its client, and the transaction's metadata. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StrokesHeader { + page: String, + client: String, + client_seq: u64, + #[serde(default)] + coalesce: Option, + #[serde(default)] + ui: Option, + #[serde(default)] + edits: Vec, +} + +/// Splits a body into its header and its records. +pub fn split_body(body: &[u8]) -> IpcResult<(StrokesHeader, &[u8])> { + let bad = |message: &str| IpcError::invalid("body", message); + let length = body + .get(..4) + .map(|bytes| u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as usize) + .ok_or_else(|| bad("The strokes body is too short."))?; + let json = body + .get(4..4 + length) + .ok_or_else(|| bad("The strokes header runs past the body."))?; + let header: StrokesHeader = + serde_json::from_slice(json).map_err(|error| bad(&format!("The strokes header isn't valid: {error}")))?; + Ok((header, &body[4 + length..])) +} + +/// Adds the records to the page as one transaction, after the header's edits. +pub fn add_strokes(handle: &PageHandle, header: StrokesHeader, records: &[u8]) -> IpcResult { + let meta = StrokeTxnMeta { + page: handle.id(), + client: handle.client().clone(), + client_seq: header.client_seq, + coalesce: header.coalesce, + ui: header.ui, + edits: header.edits, + }; + handle + .add_strokes(meta, records) + .map_err(|error| IpcError::new(error.code(), error.to_string())) +} + +/// New strokes and the edits before them, as one transaction (see the module comment for the body). +#[tauri::command] +pub async fn page_add_strokes(bridge: State<'_, CoreBridge>, request: tauri::ipc::Request<'_>) -> IpcResult { + let InvokeBody::Raw(body) = request.body() else { + return Err(IpcError::invalid("body", "The strokes must come as raw bytes.")); + }; + let (header, records) = split_body(body)?; + let records = records.to_vec(); + bridge + .run(move |bridge| { + let handle = bridge.with_named("page_add_strokes", |bridge| { + bridge.open_handle(&header.page, &header.client) + })?; + add_strokes(&handle, header, &records) + }) + .await +} + +/// Every live stroke of the page as ink records, in drawing order. +#[tauri::command] +pub async fn page_read_strokes(bridge: State<'_, CoreBridge>, page: String, client: String) -> IpcResult { + bridge + .run(move |bridge| { + let handle = bridge.with_named("page_read_strokes", |bridge| bridge.open_handle(&page, &client))?; + Ok(Response::new(handle.read_strokes(None, None).records)) + }) + .await +} + +#[cfg(test)] +#[path = "ink_bridge_tests.rs"] +mod tests; diff --git a/app/src-tauri/src/ink_bridge_tests.rs b/app/src-tauri/src/ink_bridge_tests.rs new file mode 100644 index 00000000..a23d5aeb --- /dev/null +++ b/app/src-tauri/src/ink_bridge_tests.rs @@ -0,0 +1,282 @@ +//! Tests of the ink commands over pages the notes commands made: strokes live in the page's folder in its notebook. + +use std::{fs, path::Path, sync::Arc}; + +use opennote_core::{ + format::{points::encode_points, segment::encode_records}, + model::{Channels, InkRecord, Point, Stroke, StrokeStyle}, + BlockId, StrokeId, Timestamp, +}; +use serde_json::{json, Value}; + +use super::*; +use crate::core_bridge::Bridge; + +const INK: &str = "01k6f00000000000000000k001"; + +fn stroke(id: &str) -> Vec { + let points = [ + Point { + x: 640, + y: 1_280, + pressure: 32_768, + ..Point::default() + }, + Point { + x: 720, + y: 1_440, + pressure: 36_044, + ..Point::default() + }, + ]; + let channels = Channels(Channels::PRESSURE); + let mut encoded = Vec::new(); + let bbox = encode_points(&points, channels, &mut encoded).expect("points encode"); + let stroke = Stroke { + id: StrokeId::parse(id).expect("a stroke ID"), + block: BlockId::parse(INK).expect("a block ID"), + start: Timestamp::from_unix_ms(1_790_777_258_345), + start_unknown: false, + style: StrokeStyle { + tool: 0, + palette: 1, + color: [0x2b, 0x25, 0x21, 0xff], + width: 2.0, + }, + transform: None, + origin: None, + bbox, + channels, + point_count: 2, + points: Arc::from(encoded), + }; + encode_records(&[InkRecord::Stroke(Arc::new(stroke))]) +} + +fn body(header: &Value, records: &[u8]) -> Vec { + let json = serde_json::to_vec(header).expect("JSON"); + let mut out = u32::try_from(json.len()).expect("short").to_le_bytes().to_vec(); + out.extend_from_slice(&json); + out.extend_from_slice(records); + out +} + +fn header(page: &str, seq: u64, edits: Value) -> Value { + json!({ "page": page, "client": "main-1", "clientSeq": seq, "edits": edits }) +} + +fn new_layer() -> Value { + json!([{ "edit": "insertBlock", "block": { + "id": INK, "type": "ink", "frame": { "x": 0, "y": 0 }, "data": { "role": "layer" } + } }]) +} + +/// A notebook with a section and a page in `notes`; returns the page's ID. +fn a_page(bridge: &CoreBridge, notes: &Path) -> String { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + let create = |bridge: &mut Bridge, kind: &str, parent: Option| { + let input = json!({ "kind": kind, "placement": { "parentId": parent, "beforeId": null } }); + let node = bridge.dispatch("notes_create", &json!({ "input": input }))?; + Ok::<_, IpcError>(node["id"].as_str().unwrap_or_default().to_owned()) + }; + let notebook = create(bridge, "notebook", None)?; + let section = create(bridge, "section", Some(notebook))?; + create(bridge, "page", Some(section)) + }) + .expect("a page") +} + +/// Opens the page for `main-1` and adds one stroke with its ink layer, as the page view's first stroke does. +fn draw(bridge: &CoreBridge, notes: &Path, page: &str, records: &[u8]) -> TxnAck { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + let handle = bridge.handle(page, "main-1")?; + let sent = body(&header(page, 1, new_layer()), records); + let (parsed, rest) = split_body(&sent)?; + add_strokes(&handle, parsed, rest) + }) + .expect("draws") +} + +/// The page's live strokes as `main-1` reads them after opening it. +fn strokes(bridge: &CoreBridge, notes: &Path, page: &str) -> Vec { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + Ok(bridge.handle(page, "main-1")?.read_strokes(None, None).records) + }) + .expect("reads") +} + +fn copy_dir(from: &Path, to: &Path) { + fs::create_dir_all(to).expect("a folder"); + for entry in fs::read_dir(from).expect("a listing") { + let entry = entry.expect("an entry"); + let target = to.join(entry.file_name()); + if entry.file_type().expect("a type").is_dir() { + copy_dir(&entry.path(), &target); + } else { + fs::copy(entry.path(), &target).expect("a copy"); + } + } +} + +#[test] +fn a_body_splits_into_its_header_and_records() { + let records = stroke("01k6f00000000000000000s001"); + let sent = body(&header("01k6f00000000000000000p001", 3, json!([])), &records); + let (parsed, rest) = split_body(&sent).expect("splits"); + assert_eq!(parsed.client_seq, 3); + assert_eq!(parsed.page, "01k6f00000000000000000p001"); + assert_eq!(rest, records.as_slice()); +} + +#[test] +fn a_short_or_damaged_body_is_invalid() { + assert_eq!(split_body(&[1, 0]).err().map(|e| e.code), Some("invalid".to_owned())); + assert_eq!( + split_body(&[200, 0, 0, 0, b'{']).err().map(|e| e.code), + Some("invalid".to_owned()) + ); + let bad = body(&json!({ "page": 1 }), &[]); + assert_eq!(split_body(&bad).err().map(|e| e.code), Some("invalid".to_owned())); +} + +#[test] +fn strokes_and_their_layer_land_in_one_step_and_undo_together() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let page = a_page(&bridge, ¬es); + let records = stroke("01k6f00000000000000000s001"); + assert!(draw(&bridge, ¬es, &page, &records).can_undo); + assert_eq!(strokes(&bridge, ¬es, &page), records); + let undone = bridge + .with(|bridge| { + let handle = bridge.open_handle(&page, "main-1")?; + let frame = handle + .undo(handle.client()) + .map_err(|error| IpcError::new(error.code(), error.to_string()))?; + Ok((frame.is_some(), handle.read_strokes(None, None).strokes)) + }) + .expect("undoes"); + bridge.shutdown(); + assert_eq!(undone, (true, 0)); +} + +#[test] +fn a_stroke_without_its_ink_block_is_refused() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let page = a_page(&bridge, ¬es); + let refused = bridge + .notes(Some(notes), |bridge| { + let handle = bridge.handle(&page, "main-1")?; + let sent = body(&header(&page, 1, json!([])), &stroke("01k6f00000000000000000s002")); + let (parsed, rest) = split_body(&sent)?; + Ok(add_strokes(&handle, parsed, rest).expect_err("no ink block")) + }) + .expect("the bridge"); + bridge.shutdown(); + assert_ne!(refused.code, "internal"); +} + +#[test] +fn strokes_need_an_open_session_of_a_page_in_a_notebook() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let page = a_page(&bridge, ¬es); + let (unopened, unknown) = bridge + .notes(Some(notes), |bridge| { + let unopened = bridge.open_handle(&page, "main-1").map(|_| ()).unwrap_err(); + let unknown = bridge + .handle("01k6f00000000000000000p009", "main-1") + .map(|_| ()) + .unwrap_err(); + Ok((unopened.code, unknown.code)) + }) + .expect("the bridge"); + bridge.shutdown(); + assert_eq!((unopened.as_str(), unknown.as_str()), ("notFound", "notFound")); +} + +#[test] +fn strokes_survive_a_restart_and_travel_with_a_copied_notes_folder() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let page = a_page(&bridge, ¬es); + let records = stroke("01k6f00000000000000000s003"); + draw(&bridge, ¬es, &page, &records); + bridge.shutdown(); + + // In one process, the first core keeps the notebook's device lock, so this one reads it read-only, from disk. + let again = CoreBridge::at(dir.path().join("local")); + assert_eq!(strokes(&again, ¬es, &page), records, "after a restart"); + again.shutdown(); + + // A second device: a copy of the notes folder and none of this device's files. + let copy = dir.path().join("Copy"); + copy_dir(¬es, ©); + let other = CoreBridge::at(dir.path().join("other")); + assert_eq!(strokes(&other, ©, &page), records, "in a copied notes folder"); + other.shutdown(); +} + +#[test] +fn strokes_go_to_trash_and_back_with_their_page() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let bridge = CoreBridge::at(dir.path().join("local")); + let page = a_page(&bridge, ¬es); + let records = stroke("01k6f00000000000000000s004"); + draw(&bridge, ¬es, &page, &records); + + // Undo of the delete: the receipt puts the page back. + let receipt = bridge + .notes(Some(notes.clone()), |bridge| { + bridge.dispatch("notes_trash", &json!({ "ids": [page] })) + }) + .expect("trashes"); + bridge + .notes(Some(notes.clone()), |bridge| { + bridge.dispatch("notes_restore", &json!({ "receiptId": receipt["id"] })) + }) + .expect("restores"); + assert_eq!(strokes(&bridge, ¬es, &page), records, "after Undo"); + + // Trash, then Restore from the Trash view. + let trash = |bridge: &CoreBridge, notes: &Path| { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + bridge.dispatch("notes_trash", &json!({ "ids": [page] })) + }) + .expect("trashes"); + }; + let restore = |bridge: &CoreBridge, notes: &Path| { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + bridge.dispatch("notes_restore_from_trash", &json!({ "ids": [page] })) + }) + .expect("restores from Trash"); + }; + trash(&bridge, ¬es); + restore(&bridge, ¬es); + assert_eq!(strokes(&bridge, ¬es, &page), records, "after Restore"); + + // A page in Trash keeps its strokes in the notes folder: a copy of the folder restores it with them. + trash(&bridge, ¬es); + bridge.shutdown(); + let copy = dir.path().join("Copy"); + copy_dir(¬es, ©); + let other = CoreBridge::at(dir.path().join("other")); + restore(&other, ©); + assert_eq!( + strokes(&other, ©, &page), + records, + "restored in a copied notes folder" + ); + other.shutdown(); +} diff --git a/app/src-tauri/src/install.rs b/app/src-tauri/src/install.rs index 769cdf72..336dcdd3 100644 --- a/app/src-tauri/src/install.rs +++ b/app/src-tauri/src/install.rs @@ -12,13 +12,16 @@ use std::{ time::{Duration, Instant}, }; +pub mod cli; +pub mod path_entry; mod shortcut; +pub mod uninstall; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use tauri::{AppHandle, Manager, State, WebviewWindow}; -pub use shortcut::create_shortcut; +pub use shortcut::{create_link, create_shortcut, set_jump_list}; use crate::{ ipc::{IpcError, IpcResult}, @@ -87,29 +90,23 @@ pub fn shortcut_path(paths: &Paths) -> PathBuf { /// Whether two paths name the same folder. Case and trailing separators don't matter, as on Windows. pub fn same_folder(a: &Path, b: &Path) -> bool { - let norm = |path: &Path| { - path.to_string_lossy() - .trim_start_matches(r"\\?\") - .trim_end_matches(['\\', '/']) - .to_lowercase() - }; - norm(a) == norm(b) + normalized(a) == normalized(b) +} + +/// A path as Windows compares it: no `\\?\` prefix, backslashes only (a profile folder given as `C:/x/profile` +/// yields `C:/x/profile\Programs`), no trailing separator, any case. +fn normalized(path: &Path) -> String { + path.to_string_lossy() + .trim_start_matches(r"\\?\") + .replace('/', "\\") + .trim_end_matches('\\') + .to_lowercase() } /// Whether `path` is `folder` or lies inside it. pub fn is_inside(path: &Path, folder: &Path) -> bool { - let (path, folder) = ( - path.to_string_lossy().trim_start_matches(r"\\?\").to_lowercase(), - folder - .to_string_lossy() - .trim_start_matches(r"\\?\") - .trim_end_matches(['\\', '/']) - .to_lowercase(), - ); - path == folder - || path - .strip_prefix(&folder) - .is_some_and(|rest| rest.starts_with(['\\', '/'])) + let (path, folder) = (normalized(path), normalized(folder)); + path == folder || path.strip_prefix(&folder).is_some_and(|rest| rest.starts_with('\\')) } /// Whether the app can create a file in `dir`, checked by creating and deleting a probe file. @@ -150,7 +147,7 @@ pub async fn install_pick_folder(window: WebviewWindow, initial: Option) } #[cfg(windows)] -fn pick_folder(owner: isize, initial: Option) -> IpcResult> { +pub(crate) fn pick_folder(owner: isize, initial: Option) -> IpcResult> { use windows::{ core::HSTRING, Win32::{ @@ -199,7 +196,7 @@ fn pick_folder(owner: isize, initial: Option) -> IpcResult) -> IpcResult> { +pub(crate) fn pick_folder(_owner: isize, _initial: Option) -> IpcResult> { Err(IpcError::not_implemented("install_pick_folder")) } @@ -248,8 +245,19 @@ pub fn check_folder_with(path: &Path, app_folders: &[&Path]) -> FolderCheck { pub fn install_check_folder(paths: State<'_, Paths>, path: String) -> IpcResult { let exe = std::env::current_exe().unwrap_or_default(); let exe_folder = exe.parent().unwrap_or(Path::new("")); - let programs = programs_dir(&paths); - Ok(check_folder_with(Path::new(&path), &[exe_folder, &programs])) + Ok(check_notes_folder(Path::new(&path), exe_folder, &programs_dir(&paths))) +} + +/// Checks a proposed notes folder against where the app lives. Anything inside the Programs folder is refused, +/// since updating or uninstalling OpenNote replaces that folder. Where a download sits is only the exe's own folder: +/// an update swaps the exe file, never the folder around it, and setup usually moves the exe away. So only +/// that folder itself is refused, and a folder beside or below the exe (`Downloads\OpenNote notes`, or a test +/// profile next to the exe) is fine. +pub fn check_notes_folder(path: &Path, exe_folder: &Path, programs: &Path) -> FolderCheck { + if is_local_path(path) && !exe_folder.as_os_str().is_empty() && same_folder(path, exe_folder) { + return FolderCheck::InsideAppFolder; + } + check_folder_with(path, &[programs]) } /// The SHA-256 of a file. @@ -312,6 +320,7 @@ pub fn prepare_move(paths: &Paths) -> IpcResult<(PathBuf, PathBuf)> { )); } let target = copy_exe(¤t, &dir)?; + cli::install(&dir); if let Err(error) = create_shortcut(&shortcut_path(paths), &target) { // The move still works without the shortcut, and Settings offers it again. log::warn!("Couldn't create the Start menu shortcut: {error}"); @@ -324,9 +333,22 @@ pub fn prepare_move(paths: &Paths) -> IpcResult<(PathBuf, PathBuf)> { #[tauri::command] pub async fn install_move_to_user_programs(app: AppHandle) -> IpcResult<()> { let paths = app.state::().inner().clone(); - let (current, target) = tauri::async_runtime::spawn_blocking(move || prepare_move(&paths)) - .await - .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string()))??; + let entry = crate::platform_flags::is_on_for(&app, crate::platform_flags::UNINSTALL_ENTRY); + let (current, target) = tauri::async_runtime::spawn_blocking(move || { + let moved = prepare_move(&paths)?; + // The Installed apps entry (install.uninstallEntry). The new copy refreshes it at each start too. + if entry { + let dir = programs_dir(&paths); + if let Err(error) = + uninstall::write_entry(uninstall::user_registry().as_ref(), &dir, env!("CARGO_PKG_VERSION")) + { + log::warn!("Couldn't write the Installed apps entry: {error}"); + } + } + Ok::<_, IpcError>(moved) + }) + .await + .map_err(|error| IpcError::new(crate::ipc::codes::INTERNAL, error.to_string()))??; app.state::().plan_relaunch(Relaunch { exe: target, args: vec!["--moved-from".into(), current.display().to_string()], diff --git a/app/src-tauri/src/install/cli.rs b/app/src-tauri/src/install/cli.rs new file mode 100644 index 00000000..f5fa23b0 --- /dev/null +++ b/app/src-tauri/src/install/cli.rs @@ -0,0 +1,81 @@ +//! The `opennote` command-line tool beside the app (docs/help/command-line.md). A release exe carries the tool +//! (build.rs, `OPENNOTE_CLI_EXE`) and writes it to `bin\opennote.exe` in its folder: when setup moves OpenNote into +//! the Programs folder, and at each start from there, so an update brings the matching tool. Adding the `bin` folder +//! to the user's PATH is the person's choice, in App permissions (path_entry.rs). + +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +use sha2::{Digest, Sha256}; + +use super::path_entry::{CLI_DIR, CLI_EXE}; + +/// The tool, as the release build put it in. Empty in development builds. +pub const EMBEDDED: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/opennote-cli.bin")); + +/// Where the tool goes for the app in `app_dir`. +pub fn tool_path(app_dir: &Path) -> PathBuf { + app_dir.join(CLI_DIR).join(CLI_EXE) +} + +/// Writes `bytes` to `app_dir\bin\opennote.exe` unless the same bytes are there. Returns whether it wrote. The file +/// is written beside, then moved into place, so a half-written tool never runs. +pub fn install_bytes(app_dir: &Path, bytes: &[u8]) -> io::Result { + if bytes.is_empty() { + return Ok(false); + } + let target = tool_path(app_dir); + if fs::read(&target).is_ok_and(|current| Sha256::digest(¤t) == Sha256::digest(bytes)) { + return Ok(false); + } + let folder = app_dir.join(CLI_DIR); + fs::create_dir_all(&folder)?; + let temp = folder.join("opennote.exe.new"); + fs::write(&temp, bytes)?; + // A running tool can't be replaced; it gets the new one next start. + match fs::rename(&temp, &target) { + Ok(()) => Ok(true), + Err(error) => { + let _ = fs::remove_file(&temp); + Err(error) + } + } +} + +/// Installs the tool this exe carries beside the app in `app_dir`. +pub fn install(app_dir: &Path) { + if let Err(error) = install_bytes(app_dir, EMBEDDED) { + log::warn!("Couldn't install the opennote command: {error}"); + } +} + +/// Brings the tool up to date when this exe runs from the user's Programs folder. Runs at each start. +pub fn refresh(paths: &crate::paths::Paths, settings_flags: &std::collections::BTreeMap) { + if EMBEDDED.is_empty() || !crate::platform_flags::is_on_now(crate::platform_flags::API_LOCAL, settings_flags) { + return; + } + let exe = std::env::current_exe().unwrap_or_default(); + let dir = super::programs_dir(paths); + if exe.parent().is_some_and(|folder| super::same_folder(folder, &dir)) { + install(&dir); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn writes_the_tool_once_and_replaces_it_when_it_changes() { + let dir = tempfile::tempdir().expect("a folder"); + assert!(!install_bytes(dir.path(), b"").expect("nothing to do")); + assert!(!tool_path(dir.path()).exists()); + assert!(install_bytes(dir.path(), b"MZ one").expect("written")); + assert!(!install_bytes(dir.path(), b"MZ one").expect("the same")); + assert!(install_bytes(dir.path(), b"MZ two").expect("replaced")); + assert_eq!(fs::read(tool_path(dir.path())).expect("read"), b"MZ two"); + assert!(!dir.path().join(CLI_DIR).join("opennote.exe.new").exists()); + } +} diff --git a/app/src-tauri/src/install/path_entry.rs b/app/src-tauri/src/install/path_entry.rs new file mode 100644 index 00000000..78d45947 --- /dev/null +++ b/app/src-tauri/src/install/path_entry.rs @@ -0,0 +1,190 @@ +//! The command-line tool's folder in the user's `PATH` (`HKCU\Environment`, value `Path`). Setup adds it only +//! when the person ticks "Add the opennote command to PATH", and removing the app takes it out. The folder is +//! `bin` inside the app's folder, because Windows names are case-insensitive: `opennote.exe` beside +//! `OpenNote.exe` would be the same file. + +use std::path::Path; + +/// The tool's folder inside the app's folder. +pub const CLI_DIR: &str = "bin"; + +/// The tool's file name. +pub const CLI_EXE: &str = "opennote.exe"; + +fn same_entry(entry: &str, dir: &str) -> bool { + entry + .trim() + .trim_end_matches(['\\', '/']) + .eq_ignore_ascii_case(dir.trim_end_matches(['\\', '/'])) +} + +/// `path` with `dir` added at the end, unless it is already there. +pub fn with_entry(path: &str, dir: &str) -> String { + if path.split(';').any(|entry| same_entry(entry, dir)) { + return path.to_owned(); + } + let trimmed = path.trim_end_matches(';'); + if trimmed.is_empty() { + dir.to_owned() + } else { + format!("{trimmed};{dir}") + } +} + +/// `path` without `dir`, keeping every other entry as it was. +pub fn without_entry(path: &str, dir: &str) -> String { + path.split(';') + .filter(|entry| !same_entry(entry, dir)) + .collect::>() + .join(";") +} + +/// Adds `dir` to the user's `PATH`. Returns whether it changed anything. +pub fn add(dir: &Path) -> bool { + let dir = dir.display().to_string(); + let current = system::read().unwrap_or_default(); + let next = with_entry(¤t, &dir); + next != current && system::write(&next) +} + +/// Takes `dir` out of the user's `PATH`. Returns whether it changed anything. +pub fn remove(dir: &Path) -> bool { + let dir = dir.display().to_string(); + let Some(current) = system::read() else { + return false; + }; + let next = without_entry(¤t, &dir); + next != current && system::write(&next) +} + +/// Whether `dir` is in the user's `PATH`. +pub fn contains(dir: &Path) -> bool { + let dir = dir.display().to_string(); + system::read().is_some_and(|path| path.split(';').any(|entry| same_entry(entry, &dir))) +} + +#[cfg(windows)] +mod system { + use windows::{ + core::{w, HSTRING}, + Win32::{ + Foundation::{LPARAM, WPARAM}, + System::Registry::{ + RegCloseKey, RegGetValueW, RegOpenKeyExW, RegSetValueExW, HKEY, HKEY_CURRENT_USER, KEY_SET_VALUE, + REG_EXPAND_SZ, RRF_NOEXPAND, RRF_RT_REG_EXPAND_SZ, RRF_RT_REG_SZ, + }, + UI::WindowsAndMessaging::{SendMessageTimeoutW, HWND_BROADCAST, SMTO_ABORTIFHUNG, WM_SETTINGCHANGE}, + }, + }; + + /// The user's own `Path` value, unexpanded. + pub fn read() -> Option { + let mut size = 0u32; + // SAFETY: the first call only asks for the size. + let sized = unsafe { + RegGetValueW( + HKEY_CURRENT_USER, + w!("Environment"), + w!("Path"), + RRF_RT_REG_SZ | RRF_RT_REG_EXPAND_SZ | RRF_NOEXPAND, + None, + None, + Some(&mut size), + ) + }; + if sized.is_err() { + return None; + } + let mut buffer = vec![0u16; size as usize / 2 + 1]; + let mut size = u32::try_from(buffer.len() * 2).ok()?; + // SAFETY: the buffer holds `size` bytes. + let read = unsafe { + RegGetValueW( + HKEY_CURRENT_USER, + w!("Environment"), + w!("Path"), + RRF_RT_REG_SZ | RRF_RT_REG_EXPAND_SZ | RRF_NOEXPAND, + None, + Some(buffer.as_mut_ptr().cast()), + Some(&mut size), + ) + }; + if read.is_err() { + return None; + } + let chars = (size as usize / 2).saturating_sub(1).min(buffer.len()); + Some(String::from_utf16_lossy(&buffer[..chars])) + } + + /// Writes the user's `Path` and tells running programs that the environment changed. + pub fn write(value: &str) -> bool { + let mut key = HKEY::default(); + // SAFETY: `key` receives an open key that is closed below. + if unsafe { RegOpenKeyExW(HKEY_CURRENT_USER, w!("Environment"), None, KEY_SET_VALUE, &mut key) }.is_err() { + return false; + } + let wide: Vec = value.encode_utf16().chain(Some(0)).collect(); + // SAFETY: the bytes are the NUL-terminated UTF-16 text, which Windows copies. + let bytes = unsafe { std::slice::from_raw_parts(wide.as_ptr().cast::(), wide.len() * 2) }; + let written = unsafe { RegSetValueExW(key, w!("Path"), None, REG_EXPAND_SZ, Some(bytes)) }.is_ok(); + // SAFETY: `key` was opened above and is closed once. + let _ = unsafe { RegCloseKey(key) }; + if written { + let environment = HSTRING::from("Environment"); + // SAFETY: the string outlives the call, which waits at most a second for each window. + unsafe { + let _ = SendMessageTimeoutW( + HWND_BROADCAST, + WM_SETTINGCHANGE, + WPARAM(0), + LPARAM(environment.as_ptr() as isize), + SMTO_ABORTIFHUNG, + 1000, + None, + ); + } + } + written + } +} + +#[cfg(not(windows))] +mod system { + pub fn read() -> Option { + None + } + + pub fn write(_value: &str) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn adds_once_at_the_end() { + let dir = r"C:\Users\Ada\AppData\Local\Programs\OpenNote\bin"; + assert_eq!(with_entry("", dir), dir); + assert_eq!(with_entry(r"C:\Tools;", dir), format!(r"C:\Tools;{dir}")); + let once = with_entry(r"C:\Tools", dir); + assert_eq!(with_entry(&once, dir), once); + assert_eq!( + with_entry(&once.to_uppercase(), dir), + once.to_uppercase(), + "case doesn't matter" + ); + } + + #[test] + fn removes_only_its_own_folder() { + let dir = r"C:\Programs\OpenNote\bin"; + let path = format!(r"%USERPROFILE%\bin;{dir}\;C:\Tools"); + assert_eq!(without_entry(&path, dir), r"%USERPROFILE%\bin;C:\Tools"); + assert_eq!( + without_entry(r"C:\Programs\OpenNote\binary", dir), + r"C:\Programs\OpenNote\binary" + ); + } +} diff --git a/app/src-tauri/src/install/shortcut.rs b/app/src-tauri/src/install/shortcut.rs index c13704d7..dd218856 100644 --- a/app/src-tauri/src/install/shortcut.rs +++ b/app/src-tauri/src/install/shortcut.rs @@ -8,6 +8,18 @@ use super::APP_USER_MODEL_ID; /// Creates the Start menu shortcut to `target`, with the app user model ID set. #[cfg(windows)] pub fn create_shortcut(shortcut: &Path, target: &Path) -> windows::core::Result<()> { + create_link(shortcut, target, None, "OpenNote") +} + +/// Creates a shortcut to `target` that passes `arguments`, such as an `opennote://` link, with the app user model +/// ID set so a pinned copy groups with the running window on the taskbar. +#[cfg(windows)] +pub fn create_link( + shortcut: &Path, + target: &Path, + arguments: Option<&str>, + description: &str, +) -> windows::core::Result<()> { use windows::Win32::System::Com::{CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED}; if let Some(parent) = shortcut.parent() { @@ -16,7 +28,7 @@ pub fn create_shortcut(shortcut: &Path, target: &Path) -> windows::core::Result< // SAFETY: COM is initialized for this call and released after it, and every object lives within that span. unsafe { let apartment = CoInitializeEx(None, COINIT_APARTMENTTHREADED); - let result = write_link(shortcut, target); + let result = write_link(shortcut, target, arguments, description); if apartment.is_ok() { CoUninitialize(); } @@ -29,7 +41,12 @@ pub fn create_shortcut(shortcut: &Path, target: &Path) -> windows::core::Result< /// # Safety /// COM must be initialized on the calling thread. #[cfg(windows)] -unsafe fn write_link(shortcut: &Path, target: &Path) -> windows::core::Result<()> { +unsafe fn write_link( + shortcut: &Path, + target: &Path, + arguments: Option<&str>, + description: &str, +) -> windows::core::Result<()> { use windows::{ core::{Interface, HSTRING}, Win32::{ @@ -43,7 +60,11 @@ unsafe fn write_link(shortcut: &Path, target: &Path) -> windows::core::Result<() if let Some(dir) = target.parent() { link.SetWorkingDirectory(&HSTRING::from(dir.as_os_str()))?; } - link.SetDescription(&HSTRING::from("OpenNote"))?; + if let Some(arguments) = arguments { + link.SetArguments(&HSTRING::from(arguments))?; + } + link.SetIconLocation(&HSTRING::from(target.as_os_str()), 0)?; + link.SetDescription(&HSTRING::from(description))?; set_app_user_model_id(&link.cast::()?)?; link.cast::()? .Save(&HSTRING::from(shortcut.as_os_str()), true) @@ -56,13 +77,33 @@ unsafe fn write_link(shortcut: &Path, target: &Path) -> windows::core::Result<() #[cfg(windows)] unsafe fn set_app_user_model_id( store: &windows::Win32::UI::Shell::PropertiesSystem::IPropertyStore, +) -> windows::core::Result<()> { + use windows::{core::GUID, Win32::Foundation::PROPERTYKEY}; + + // PKEY_AppUserModel_ID: {9F4C2855-9F79-4B39-A8D0-E1D42DE1D5F3}, 5. + const KEY: PROPERTYKEY = PROPERTYKEY { + fmtid: GUID::from_u128(0x9F4C2855_9F79_4B39_A8D0_E1D42DE1D5F3), + pid: 5, + }; + set_string_property(store, &KEY, APP_USER_MODEL_ID) +} + +/// Stores a text value in a property store, as a `VT_LPWSTR` value. +/// +/// # Safety +/// COM must be initialized on the calling thread. +#[cfg(windows)] +unsafe fn set_string_property( + store: &windows::Win32::UI::Shell::PropertiesSystem::IPropertyStore, + key: &windows::Win32::Foundation::PROPERTYKEY, + value_text: &str, ) -> windows::core::Result<()> { use std::mem::ManuallyDrop; use windows::{ - core::{Error, GUID, PWSTR}, + core::{Error, PWSTR}, Win32::{ - Foundation::{E_OUTOFMEMORY, PROPERTYKEY}, + Foundation::E_OUTOFMEMORY, System::{ Com::{ CoTaskMemAlloc, @@ -75,13 +116,8 @@ unsafe fn set_app_user_model_id( }, }; - // PKEY_AppUserModel_ID: {9F4C2855-9F79-4B39-A8D0-E1D42DE1D5F3}, 5. - const KEY: PROPERTYKEY = PROPERTYKEY { - fmtid: GUID::from_u128(0x9F4C2855_9F79_4B39_A8D0_E1D42DE1D5F3), - pid: 5, - }; - // The identifier in memory that PropVariantClear frees. - let wide: Vec = APP_USER_MODEL_ID.encode_utf16().chain(Some(0)).collect(); + // The text in memory that PropVariantClear frees. + let wide: Vec = value_text.encode_utf16().chain(Some(0)).collect(); let text = CoTaskMemAlloc(wide.len() * 2).cast::(); if text.is_null() { return Err(Error::from(E_OUTOFMEMORY)); @@ -98,11 +134,108 @@ unsafe fn set_app_user_model_id( }), }, }; - let stored = store.SetValue(&KEY, &value).and_then(|()| store.Commit()); + let stored = store.SetValue(key, &value).and_then(|()| store.Commit()); let _ = PropVariantClear(&mut value); stored } +/// Sets the taskbar jump list: "New quick note" as a task, and the recent pages (title and `opennote://` link) as +/// a category. +#[cfg(windows)] +pub fn set_jump_list(exe: &Path, recent: &[(String, String)]) -> windows::core::Result<()> { + use windows::Win32::System::Com::{CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED}; + + // SAFETY: COM is initialized for this call and released after it, and every object lives within that span. + unsafe { + let apartment = CoInitializeEx(None, COINIT_APARTMENTTHREADED); + let result = write_jump_list(exe, recent); + if apartment.is_ok() { + CoUninitialize(); + } + result + } +} + +/// A link for a jump list entry: the program with arguments, and a title. +/// +/// # Safety +/// COM must be initialized on the calling thread. +#[cfg(windows)] +unsafe fn entry_link( + exe: &Path, + arguments: &str, + title: &str, +) -> windows::core::Result { + use windows::{ + core::{Interface, GUID, HSTRING}, + Win32::{ + Foundation::PROPERTYKEY, + System::Com::{CoCreateInstance, CLSCTX_INPROC_SERVER}, + UI::Shell::{IShellLinkW, PropertiesSystem::IPropertyStore, ShellLink}, + }, + }; + + // PKEY_Title: {F29F85E0-4FF9-1068-AB91-08002B27B3D9}, 2. + const TITLE: PROPERTYKEY = PROPERTYKEY { + fmtid: GUID::from_u128(0xF29F85E0_4FF9_1068_AB91_08002B27B3D9), + pid: 2, + }; + let link: IShellLinkW = CoCreateInstance(&ShellLink, None, CLSCTX_INPROC_SERVER)?; + link.SetPath(&HSTRING::from(exe.as_os_str()))?; + link.SetArguments(&HSTRING::from(arguments))?; + link.SetIconLocation(&HSTRING::from(exe.as_os_str()), 0)?; + link.SetDescription(&HSTRING::from(title))?; + set_string_property(&link.cast::()?, &TITLE, title)?; + Ok(link) +} + +/// # Safety +/// COM must be initialized on the calling thread. +#[cfg(windows)] +unsafe fn write_jump_list(exe: &Path, recent: &[(String, String)]) -> windows::core::Result<()> { + use windows::{ + core::{Interface, HSTRING}, + Win32::{ + System::Com::{CoCreateInstance, CLSCTX_INPROC_SERVER}, + UI::Shell::{ + Common::{IObjectArray, IObjectCollection}, + DestinationList, EnumerableObjectCollection, ICustomDestinationList, + }, + }, + }; + + let list: ICustomDestinationList = CoCreateInstance(&DestinationList, None, CLSCTX_INPROC_SERVER)?; + list.SetAppID(&HSTRING::from(APP_USER_MODEL_ID))?; + let mut slots = 0u32; + let _removed: IObjectArray = list.BeginList(&mut slots)?; + if !recent.is_empty() { + let pages: IObjectCollection = CoCreateInstance(&EnumerableObjectCollection, None, CLSCTX_INPROC_SERVER)?; + for (title, link) in recent.iter().take(slots.max(1) as usize) { + pages.AddObject(&entry_link(exe, link, title)?)?; + } + list.AppendCategory(&HSTRING::from("Recent pages"), &pages.cast::()?)?; + } + let tasks: IObjectCollection = CoCreateInstance(&EnumerableObjectCollection, None, CLSCTX_INPROC_SERVER)?; + tasks.AddObject(&entry_link(exe, "--quick-note", "New quick note")?)?; + list.AddUserTasks(&tasks.cast::()?)?; + list.CommitList() +} + +#[cfg(not(windows))] +pub fn create_link( + _shortcut: &Path, + _target: &Path, + _arguments: Option<&str>, + _description: &str, +) -> Result<(), std::io::Error> { + Err(std::io::Error::other("Shortcuts need Windows.")) +} + +#[cfg(not(windows))] +pub fn set_jump_list(_exe: &Path, _recent: &[(String, String)]) -> Result<(), std::io::Error> { + Err(std::io::Error::other("Jump lists need Windows.")) +} + #[cfg(not(windows))] pub fn create_shortcut(_shortcut: &Path, _target: &Path) -> Result<(), std::io::Error> { Err(std::io::Error::other("Shortcuts need Windows.")) diff --git a/app/src-tauri/src/install/tests.rs b/app/src-tauri/src/install/tests.rs index ff072553..c61b24ed 100644 --- a/app/src-tauri/src/install/tests.rs +++ b/app/src-tauri/src/install/tests.rs @@ -240,3 +240,41 @@ fn removes_the_old_copy_after_a_move() { } assert!(!old.exists()); } + +#[test] +fn notes_may_sit_below_the_exe_but_not_in_its_folder_or_in_programs() { + let dir = tempfile::tempdir().expect("a temp folder"); + let exe_folder = dir.path(); + let programs = exe_folder.join("profile").join("Programs").join("OpenNote"); + let notes = exe_folder.join("profile").join("Documents").join("OpenNote"); + assert_eq!( + check_notes_folder(¬es, exe_folder, &programs), + FolderCheck::WillCreate + ); + assert_eq!( + check_notes_folder(&exe_folder.join("notes"), exe_folder, &programs), + FolderCheck::WillCreate + ); + assert_eq!( + check_notes_folder(exe_folder, exe_folder, &programs), + FolderCheck::InsideAppFolder + ); + assert_eq!( + check_notes_folder(&programs.join("notes"), exe_folder, &programs), + FolderCheck::InsideAppFolder + ); +} + +#[test] +fn folders_match_whichever_way_the_slashes_lean() { + // A profile folder given as `C:/x/profile` makes Programs `C:/x/profile\Programs\OpenNote`. + assert!(same_folder( + Path::new(r"C:/x/profile\Programs\OpenNote"), + Path::new(r"C:\x\profile\Programs\OpenNote\") + )); + assert!(is_inside( + Path::new(r"C:\x\profile\Documents"), + Path::new("C:/x/profile/") + )); + assert!(!is_inside(Path::new(r"C:\x\profiles"), Path::new("C:/x/profile"))); +} diff --git a/app/src-tauri/src/install/uninstall.rs b/app/src-tauri/src/install/uninstall.rs new file mode 100644 index 00000000..08667add --- /dev/null +++ b/app/src-tauri/src/install/uninstall.rs @@ -0,0 +1,563 @@ +//! The Installed apps entry and `OpenNote.exe --uninstall` (behind `install.uninstallEntry`). +//! +//! Setup's move to `%LOCALAPPDATA%\Programs\OpenNote` writes a per-user entry under +//! `HKCU\Software\Microsoft\Windows\CurrentVersion\Uninstall\OpenNote`, and each start from that folder refreshes +//! it, so an update shows its new version there. Nothing needs administrator rights. +//! +//! Uninstalling asks first, and only runs while no other OpenNote process holds the profile. It removes the +//! Start menu shortcut, the command-line tool's folder from the user's `PATH`, the Installed apps entry, and the +//! app's own folder. It never touches the notes folder, even when the notes folder sits inside the app's folder, +//! and it keeps the settings, so a later install starts where the person left off. A running exe can't delete +//! itself, so the last step runs from a copy in the temporary folder (`--uninstall-finish`), which waits for this +//! process to exit. The finishing copy works out the folder again from the profile, so the argument can't point +//! it anywhere else. + +use std::{ + collections::BTreeMap, + fs, io, + path::{Path, PathBuf}, + sync::Mutex, +}; + +use super::{is_inside, programs_dir, same_folder, shortcut_path, INSTALLED_EXE_NAME}; +use crate::paths::Paths; + +/// Where per-user uninstall entries live, under `HKEY_CURRENT_USER`. +pub const UNINSTALL_ROOT: &str = r"Software\Microsoft\Windows\CurrentVersion\Uninstall"; + +/// The entry's own key under [`UNINSTALL_ROOT`]. +pub const ENTRY_KEY: &str = "OpenNote"; + +/// The argument that starts the uninstall. +pub const UNINSTALL_ARG: &str = "--uninstall"; + +/// The argument of the copy that removes the app's folder once the first process has exited. +pub const FINISH_ARG: &str = "--uninstall-finish"; + +/// The publisher shown in Installed apps. +pub const PUBLISHER: &str = "OpenNote contributors"; + +/// A registry value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RegValue { + Text(String), + Number(u32), +} + +/// The part of the registry the entry needs: one key's values, written, and removed as a whole. +pub trait Registry { + /// Creates `key` if needed and sets each value. + fn write(&self, key: &str, values: &[(&str, RegValue)]) -> io::Result<()>; + /// Removes `key` with everything under it. A key that isn't there is not an error. + fn delete(&self, key: &str) -> io::Result<()>; + /// One value of `key`. + fn read(&self, key: &str, name: &str) -> Option; +} + +/// A registry in memory: the fake root the tests use. +#[derive(Default)] +pub struct MemoryRegistry { + keys: Mutex>>, +} + +impl MemoryRegistry { + pub fn has_key(&self, key: &str) -> bool { + self.keys + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .contains_key(key) + } +} + +impl Registry for MemoryRegistry { + fn write(&self, key: &str, values: &[(&str, RegValue)]) -> io::Result<()> { + let mut keys = self.keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let entry = keys.entry(key.to_owned()).or_default(); + for (name, value) in values { + entry.insert((*name).to_owned(), value.clone()); + } + Ok(()) + } + + fn delete(&self, key: &str) -> io::Result<()> { + let mut keys = self.keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let prefix = format!("{key}\\"); + keys.retain(|name, _| name != key && !name.starts_with(&prefix)); + Ok(()) + } + + fn read(&self, key: &str, name: &str) -> Option { + let keys = self.keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + keys.get(key).and_then(|values| values.get(name)).cloned() + } +} + +/// The entry's full key under `HKEY_CURRENT_USER`. +pub fn entry_key() -> String { + format!(r"{UNINSTALL_ROOT}\{ENTRY_KEY}") +} + +/// The values of the entry for an install in `dir`. +pub fn entry_values(dir: &Path, version: &str, size_kb: u32, date: &str) -> Vec<(&'static str, RegValue)> { + let exe = dir.join(INSTALLED_EXE_NAME); + let text = |value: String| RegValue::Text(value); + vec![ + ("DisplayName", text("OpenNote".into())), + ("DisplayVersion", text(version.into())), + ("Publisher", text(PUBLISHER.into())), + ("InstallLocation", text(dir.display().to_string())), + ("DisplayIcon", text(format!("\"{}\",0", exe.display()))), + ( + "UninstallString", + text(format!("\"{}\" {UNINSTALL_ARG}", exe.display())), + ), + ("URLInfoAbout", text("https://github.com/XrxcGH/OpenNote".into())), + ("InstallDate", text(date.into())), + ("EstimatedSize", RegValue::Number(size_kb)), + ("NoModify", RegValue::Number(1)), + ("NoRepair", RegValue::Number(1)), + ] +} + +/// Writes or refreshes the entry for the install in `dir`. +pub fn write_entry(registry: &dyn Registry, dir: &Path, version: &str) -> io::Result<()> { + let size = fs::metadata(dir.join(INSTALLED_EXE_NAME)) + .map(|meta| u32::try_from(meta.len() / 1024).unwrap_or(u32::MAX)) + .unwrap_or(0); + // Keep the first install date across refreshes. + let date = match registry.read(&entry_key(), "InstallDate") { + Some(RegValue::Text(date)) if date.len() == 8 => date, + _ => today(), + }; + registry.write(&entry_key(), &entry_values(dir, version, size, &date)) +} + +/// Removes the entry. +pub fn remove_entry(registry: &dyn Registry) -> io::Result<()> { + registry.delete(&entry_key()) +} + +/// Today as `YYYYMMDD`, in UTC. +fn today() -> String { + let days = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs() / 86_400) + .unwrap_or(0); + let (year, month, day) = civil_from_days(i64::try_from(days).unwrap_or(0)); + format!("{year:04}{month:02}{day:02}") +} + +/// The calendar date of a day count since 1970-01-01 (Howard Hinnant's `civil_from_days`). +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = u32::try_from(doy - (153 * mp + 2) / 5 + 1).unwrap_or(1); + let month = u32::try_from(if mp < 10 { mp + 3 } else { mp - 9 }).unwrap_or(1); + let year = yoe + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +/// What uninstalling removes, worked out from the profile alone. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Plan { + /// The app's folder, `%LOCALAPPDATA%\Programs\OpenNote`. + pub app_dir: PathBuf, + /// The Start menu shortcut. + pub shortcut: PathBuf, + /// The notes folder from settings, which is never removed. + pub notes_folder: Option, +} + +/// The plan for this profile. The notes folder comes from the settings file, read without changing it. +pub fn plan(paths: &Paths) -> Plan { + let notes_folder = fs::read_to_string(&paths.settings_file) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .and_then(|json| json["storage"]["notesFolder"].as_str().map(PathBuf::from)) + .filter(|folder| !folder.as_os_str().is_empty()); + Plan { + app_dir: programs_dir(paths), + shortcut: shortcut_path(paths), + notes_folder, + } +} + +/// Removes the app's folder, leaving the notes folder (and everything above it) alone when it sits inside. A +/// folder that isn't the app's folder, or that holds no `OpenNote.exe`, is left alone. Returns how many files +/// were removed. +pub fn remove_app_dir(plan: &Plan) -> io::Result { + let dir = &plan.app_dir; + if !dir.join(INSTALLED_EXE_NAME).is_file() { + return Ok(0); + } + let keep = plan.notes_folder.as_deref(); + let mut removed = 0; + remove_tree(dir, keep, &mut removed)?; + Ok(removed) +} + +/// Removes `dir`'s contents and then `dir`, skipping the folder `keep` and the folders that lead to it. +fn remove_tree(dir: &Path, keep: Option<&Path>, removed: &mut usize) -> io::Result<()> { + if keep.is_some_and(|keep| same_folder(dir, keep)) { + return Ok(()); + } + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + // A link is removed as a link, never followed. + let kind = entry.file_type()?; + if kind.is_dir() && !kind.is_symlink() { + remove_tree(&path, keep, removed)?; + } else { + match fs::remove_file(&path) { + Ok(()) => *removed += 1, + Err(error) if kind.is_symlink() => fs::remove_dir(&path).map_err(|_| error)?, + Err(error) => return Err(error), + } + } + } + let holds_notes = keep.is_some_and(|keep| is_inside(keep, dir)); + if !holds_notes { + fs::remove_dir(dir)?; + } + Ok(()) +} + +/// The start-menu and registry part of uninstalling, which the first process does before it hands over. +pub fn remove_links(plan: &Plan, registry: &dyn Registry) -> io::Result<()> { + match fs::remove_file(&plan.shortcut) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + crate::install::path_entry::remove(&plan.app_dir.join(crate::install::path_entry::CLI_DIR)); + remove_entry(registry) +} + +/// The registry of this user, `HKEY_CURRENT_USER`. +#[cfg(windows)] +pub struct UserRegistry; + +#[cfg(windows)] +impl Registry for UserRegistry { + fn write(&self, key: &str, values: &[(&str, RegValue)]) -> io::Result<()> { + use windows::{ + core::HSTRING, + Win32::System::Registry::{ + RegCloseKey, RegCreateKeyExW, RegSetValueExW, HKEY, HKEY_CURRENT_USER, KEY_WRITE, REG_DWORD, + REG_OPTION_NON_VOLATILE, REG_SZ, + }, + }; + + let mut handle = HKEY::default(); + // SAFETY: the key name is NUL-terminated, and `handle` receives an open key that is closed below. + let created = unsafe { + RegCreateKeyExW( + HKEY_CURRENT_USER, + &HSTRING::from(key), + None, + None, + REG_OPTION_NON_VOLATILE, + KEY_WRITE, + None, + &mut handle, + None, + ) + }; + if created.is_err() { + return Err(io::Error::from_raw_os_error(created.0 as i32)); + } + let mut result = Ok(()); + for (name, value) in values { + let name = HSTRING::from(*name); + let status = match value { + RegValue::Text(text) => { + let wide: Vec = text.encode_utf16().chain(Some(0)).collect(); + // SAFETY: the bytes are the NUL-terminated UTF-16 text, which Windows copies. + let bytes = unsafe { std::slice::from_raw_parts(wide.as_ptr().cast::(), wide.len() * 2) }; + unsafe { RegSetValueExW(handle, &name, None, REG_SZ, Some(bytes)) } + } + RegValue::Number(number) => { + let bytes = number.to_le_bytes(); + // SAFETY: four bytes of a DWORD, which Windows copies. + unsafe { RegSetValueExW(handle, &name, None, REG_DWORD, Some(&bytes)) } + } + }; + if status.is_err() { + result = Err(io::Error::from_raw_os_error(status.0 as i32)); + break; + } + } + // SAFETY: `handle` was opened above and is closed once. + let _ = unsafe { RegCloseKey(handle) }; + result + } + + fn delete(&self, key: &str) -> io::Result<()> { + use windows::{ + core::HSTRING, + Win32::{ + Foundation::ERROR_FILE_NOT_FOUND, + System::Registry::{RegDeleteTreeW, HKEY_CURRENT_USER}, + }, + }; + + // SAFETY: the key name is NUL-terminated. + let status = unsafe { RegDeleteTreeW(HKEY_CURRENT_USER, &HSTRING::from(key)) }; + if status.is_ok() || status == ERROR_FILE_NOT_FOUND { + return Ok(()); + } + Err(io::Error::from_raw_os_error(status.0 as i32)) + } + + fn read(&self, key: &str, name: &str) -> Option { + use windows::{ + core::HSTRING, + Win32::System::Registry::{RegGetValueW, HKEY_CURRENT_USER, RRF_RT_REG_DWORD, RRF_RT_REG_SZ}, + }; + + let (key, name) = (HSTRING::from(key), HSTRING::from(name)); + let mut number = 0u32; + let mut size = 4u32; + // SAFETY: the buffer is four bytes, as `size` says. + let read = unsafe { + RegGetValueW( + HKEY_CURRENT_USER, + &key, + &name, + RRF_RT_REG_DWORD, + None, + Some(std::ptr::from_mut(&mut number).cast()), + Some(&mut size), + ) + }; + if read.is_ok() { + return Some(RegValue::Number(number)); + } + let mut buffer = vec![0u16; 1024]; + let mut size = u32::try_from(buffer.len() * 2).unwrap_or(u32::MAX); + // SAFETY: the buffer holds `size` bytes. + let read = unsafe { + RegGetValueW( + HKEY_CURRENT_USER, + &key, + &name, + RRF_RT_REG_SZ, + None, + Some(buffer.as_mut_ptr().cast()), + Some(&mut size), + ) + }; + if read.is_err() { + return None; + } + let chars = (size as usize / 2).saturating_sub(1).min(buffer.len()); + Some(RegValue::Text(String::from_utf16_lossy(&buffer[..chars]))) + } +} + +/// The registry this system has. Other systems keep nothing. +pub fn user_registry() -> Box { + #[cfg(windows)] + { + Box::new(UserRegistry) + } + #[cfg(not(windows))] + { + Box::new(MemoryRegistry::default()) + } +} + +/// Writes or refreshes the entry when this exe runs from the user's Programs folder and the flag is on. Called +/// after setup's move and at each start. +pub fn refresh_if_installed(paths: &Paths, settings_flags: &BTreeMap) { + if cfg!(debug_assertions) + || !crate::platform_flags::is_on_now(crate::platform_flags::UNINSTALL_ENTRY, settings_flags) + { + return; + } + let exe = std::env::current_exe().unwrap_or_default(); + let dir = programs_dir(paths); + if !exe.parent().is_some_and(|folder| same_folder(folder, &dir)) { + return; + } + if let Err(error) = write_entry(user_registry().as_ref(), &dir, env!("CARGO_PKG_VERSION")) { + log::warn!("Couldn't write the Installed apps entry: {error}"); + } +} + +/// What the process should do with its arguments: nothing, start the uninstall, or finish one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Request { + None, + Uninstall, + /// Finish after the process with this ID has exited. + Finish { + wait_pid: Option, + }, +} + +/// Reads the uninstall arguments, which come before anything else on the command line. +pub fn request_from(args: &[String]) -> Request { + let wait_pid = args + .iter() + .position(|arg| arg == "--wait-pid") + .and_then(|at| args.get(at + 1)) + .and_then(|pid| pid.parse().ok()); + if args.iter().any(|arg| arg == FINISH_ARG) { + Request::Finish { wait_pid } + } else if args.iter().any(|arg| arg == UNINSTALL_ARG) { + Request::Uninstall + } else { + Request::None + } +} + +/// Runs the uninstall request from the command line, and returns the exit code, or `None` to start normally. +pub fn run_from_args() -> Option { + let args: Vec = std::env::args().skip(1).collect(); + match request_from(&args) { + Request::None => None, + Request::Uninstall => Some(start_uninstall()), + Request::Finish { wait_pid } => Some(finish_uninstall(wait_pid)), + } +} + +fn profile_paths() -> io::Result { + let program_dir = std::env::current_exe() + .ok() + .and_then(|exe| exe.parent().map(Path::to_path_buf)); + Paths::resolve(crate::paths::profile_override( + std::env::var_os(crate::paths::PROFILE_DIR_VAR).map(Into::into), + program_dir.as_deref(), + )) +} + +/// Asks, checks that OpenNote isn't running, removes the shortcut, the `PATH` entry, and the Installed apps entry, +/// then hands the app's folder to a copy in the temporary folder. +fn start_uninstall() -> i32 { + let Ok(paths) = profile_paths() else { + return 1; + }; + let plan = plan(&paths); + let notes = plan.notes_folder.as_ref().map_or_else( + || "your Documents folder".to_owned(), + |folder| folder.display().to_string(), + ); + let question = format!( + "Remove OpenNote from this PC?\n\nYour notes stay where they are, in {notes}. Your settings stay too, so \ + installing OpenNote again picks up where you left off." + ); + if !dialog::confirm(&question) { + return 0; + } + // Another process holding the profile is told to come forward, and the person is asked to close it. + let instance = match crate::instance::acquire(&paths, &crate::args::Args::default()) { + crate::instance::InstanceOutcome::Owner(guard) => guard, + _ => { + dialog::inform("OpenNote is open. Close it, then remove it again from Installed apps."); + return 1; + } + }; + if let Err(error) = remove_links(&plan, user_registry().as_ref()) { + log::warn!("Couldn't remove the shortcut or the Installed apps entry: {error}"); + } + let running = std::env::current_exe().unwrap_or_default(); + let runs_inside = running.parent().is_some_and(|folder| is_inside(folder, &plan.app_dir)); + if !runs_inside { + drop(instance); + return finish_now(&plan); + } + match hand_over(&running) { + Ok(()) => 0, + Err(error) => { + log::warn!("Couldn't start the last step of the uninstall: {error}"); + dialog::inform( + "OpenNote couldn't remove its folder. Delete it by hand if you like; your notes are not in it.", + ); + 1 + } + } +} + +/// Copies this exe to the temporary folder and starts it with [`FINISH_ARG`], waiting for this process. +fn hand_over(running: &Path) -> io::Result<()> { + let copy = std::env::temp_dir().join(format!("OpenNote-uninstall-{}.exe", std::process::id())); + fs::copy(running, ©)?; + std::process::Command::new(©) + .args([FINISH_ARG, "--wait-pid", &std::process::id().to_string()]) + .spawn()?; + Ok(()) +} + +fn finish_uninstall(wait_pid: Option) -> i32 { + if let Some(pid) = wait_pid { + let _ = crate::args::wait_for_exit(pid, crate::args::WAIT_PID_TIMEOUT); + } + let Ok(paths) = profile_paths() else { + return 1; + }; + finish_now(&plan(&paths)) +} + +fn finish_now(plan: &Plan) -> i32 { + match remove_app_dir(plan) { + Ok(_) => { + dialog::inform("OpenNote was removed. Your notes are still in their folder."); + 0 + } + Err(error) => { + log::warn!("Couldn't remove the app's folder: {error}"); + dialog::inform( + "OpenNote was mostly removed, but some of its files were in use. Delete its folder later if you like.", + ); + 1 + } + } +} + +/// The two message boxes the uninstall shows, before any window exists. +mod dialog { + #[cfg(windows)] + pub fn confirm(text: &str) -> bool { + use windows::{ + core::{w, HSTRING}, + Win32::UI::WindowsAndMessaging::{MessageBoxW, IDYES, MB_ICONQUESTION, MB_YESNO}, + }; + // SAFETY: both strings are NUL-terminated and outlive the call. + unsafe { + MessageBoxW( + None, + &HSTRING::from(text), + w!("Remove OpenNote"), + MB_YESNO | MB_ICONQUESTION, + ) == IDYES + } + } + + #[cfg(windows)] + pub fn inform(text: &str) { + use windows::{ + core::{w, HSTRING}, + Win32::UI::WindowsAndMessaging::{MessageBoxW, MB_ICONINFORMATION, MB_OK}, + }; + // SAFETY: both strings are NUL-terminated and outlive the call. + let _ = unsafe { MessageBoxW(None, &HSTRING::from(text), w!("OpenNote"), MB_OK | MB_ICONINFORMATION) }; + } + + #[cfg(not(windows))] + pub fn confirm(_text: &str) -> bool { + false + } + + #[cfg(not(windows))] + pub fn inform(_text: &str) {} +} + +#[cfg(test)] +#[path = "uninstall_tests.rs"] +mod tests; diff --git a/app/src-tauri/src/install/uninstall_tests.rs b/app/src-tauri/src/install/uninstall_tests.rs new file mode 100644 index 00000000..4e9de071 --- /dev/null +++ b/app/src-tauri/src/install/uninstall_tests.rs @@ -0,0 +1,154 @@ +//! The Installed apps entry against a fake registry root, and the removal of the app's folder. + +use super::*; + +fn entry(registry: &MemoryRegistry, name: &str) -> Option { + registry.read(&entry_key(), name) +} + +#[test] +fn writes_every_value_installed_apps_shows() { + let dir = tempfile::tempdir().expect("a temp folder"); + fs::write(dir.path().join(INSTALLED_EXE_NAME), vec![0u8; 4096]).expect("an exe"); + let registry = MemoryRegistry::default(); + write_entry(®istry, dir.path(), "0.1.0-beta.5").expect("written"); + + let text = |value: &str| Some(RegValue::Text(value.to_owned())); + let exe = dir.path().join(INSTALLED_EXE_NAME); + assert_eq!(entry(®istry, "DisplayName"), text("OpenNote")); + assert_eq!(entry(®istry, "DisplayVersion"), text("0.1.0-beta.5")); + assert_eq!(entry(®istry, "Publisher"), text(PUBLISHER)); + assert_eq!( + entry(®istry, "InstallLocation"), + text(&dir.path().display().to_string()) + ); + assert_eq!( + entry(®istry, "UninstallString"), + text(&format!("\"{}\" --uninstall", exe.display())) + ); + assert_eq!(entry(®istry, "EstimatedSize"), Some(RegValue::Number(4))); + assert_eq!(entry(®istry, "NoModify"), Some(RegValue::Number(1))); + assert_eq!( + entry_key(), + r"Software\Microsoft\Windows\CurrentVersion\Uninstall\OpenNote" + ); +} + +#[test] +fn a_refresh_after_an_update_keeps_the_install_date_and_changes_the_version() { + let dir = tempfile::tempdir().expect("a temp folder"); + let registry = MemoryRegistry::default(); + registry + .write(&entry_key(), &[("InstallDate", RegValue::Text("20260102".into()))]) + .expect("an old entry"); + write_entry(®istry, dir.path(), "0.1.0-beta.6").expect("written"); + assert_eq!(entry(®istry, "InstallDate"), Some(RegValue::Text("20260102".into()))); + assert_eq!( + entry(®istry, "DisplayVersion"), + Some(RegValue::Text("0.1.0-beta.6".into())) + ); +} + +#[test] +fn removing_the_entry_removes_the_key_and_nothing_beside_it() { + let registry = MemoryRegistry::default(); + let other = format!(r"{UNINSTALL_ROOT}\SomethingElse"); + registry + .write(&other, &[("DisplayName", RegValue::Text("Else".into()))]) + .expect("other"); + write_entry(®istry, Path::new(r"C:\nowhere"), "1.0.0").expect("written"); + remove_entry(®istry).expect("removed"); + assert!(!registry.has_key(&entry_key())); + assert!(registry.has_key(&other)); + remove_entry(®istry).expect("removing twice is fine"); +} + +#[test] +fn reads_the_uninstall_arguments() { + let args = |items: &[&str]| items.iter().map(|item| (*item).to_owned()).collect::>(); + assert_eq!(request_from(&args(&[])), Request::None); + assert_eq!(request_from(&args(&["opennote://page/x"])), Request::None); + assert_eq!(request_from(&args(&["--uninstall"])), Request::Uninstall); + assert_eq!( + request_from(&args(&["--uninstall-finish", "--wait-pid", "42"])), + Request::Finish { wait_pid: Some(42) } + ); +} + +fn installed(root: &Path) -> Plan { + let app_dir = root.join("Programs").join("OpenNote"); + fs::create_dir_all(app_dir.join("bin")).expect("folders"); + fs::write(app_dir.join(INSTALLED_EXE_NAME), b"exe").expect("exe"); + fs::write(app_dir.join("bin").join("opennote.exe"), b"cli").expect("cli"); + Plan { + app_dir, + shortcut: root.join("OpenNote.lnk"), + notes_folder: Some(root.join("Documents").join("OpenNote")), + } +} + +#[test] +fn removes_the_app_folder_and_never_the_notes_folder() { + let root = tempfile::tempdir().expect("a temp folder"); + let plan = installed(root.path()); + let notes = plan.notes_folder.clone().expect("notes"); + fs::create_dir_all(¬es).expect("notes folder"); + fs::write(notes.join("notebook.json"), b"{}").expect("a notebook"); + fs::write(&plan.shortcut, b"lnk").expect("shortcut"); + let registry = MemoryRegistry::default(); + write_entry(®istry, &plan.app_dir, "1.0.0").expect("entry"); + + remove_links(&plan, ®istry).expect("links"); + assert_eq!(remove_app_dir(&plan).expect("removed"), 2); + assert!(!plan.app_dir.exists()); + assert!(!plan.shortcut.exists()); + assert!(!registry.has_key(&entry_key())); + assert!(notes.join("notebook.json").is_file()); +} + +#[test] +fn a_notes_folder_inside_the_app_folder_survives() { + let root = tempfile::tempdir().expect("a temp folder"); + let mut plan = installed(root.path()); + let notes = plan.app_dir.join("My notes"); + fs::create_dir_all(notes.join("Biology")).expect("notes"); + fs::write(notes.join("Biology").join("notebook.json"), b"{}").expect("a notebook"); + plan.notes_folder = Some(notes.clone()); + + remove_app_dir(&plan).expect("removed"); + assert!(!plan.app_dir.join(INSTALLED_EXE_NAME).exists()); + assert!(notes.join("Biology").join("notebook.json").is_file()); +} + +#[test] +fn a_folder_without_the_app_is_left_alone() { + let root = tempfile::tempdir().expect("a temp folder"); + fs::write(root.path().join("keep.txt"), b"x").expect("a file"); + let plan = Plan { + app_dir: root.path().to_path_buf(), + shortcut: root.path().join("none.lnk"), + notes_folder: None, + }; + assert_eq!(remove_app_dir(&plan).expect("nothing to do"), 0); + assert!(root.path().join("keep.txt").is_file()); +} + +#[test] +fn the_plan_reads_the_notes_folder_from_settings_without_changing_them() { + let root = tempfile::tempdir().expect("a temp folder"); + let paths = Paths::under_profile(root.path()); + fs::create_dir_all(paths.settings_file.parent().expect("a folder")).expect("folders"); + let settings = r#"{ "storage": { "notesFolder": "D:\\Notes" } }"#; + fs::write(&paths.settings_file, settings).expect("settings"); + let plan = plan(&paths); + assert_eq!(plan.notes_folder, Some(PathBuf::from(r"D:\Notes"))); + assert_eq!(plan.app_dir, programs_dir(&paths)); + assert_eq!(fs::read_to_string(&paths.settings_file).expect("settings"), settings); +} + +#[test] +fn dates_are_calendar_dates() { + assert_eq!(civil_from_days(0), (1970, 1, 1)); + assert_eq!(civil_from_days(20_454), (2026, 1, 1)); + assert_eq!(today().len(), 8); +} diff --git a/app/src-tauri/src/intel/caption_style.rs b/app/src-tauri/src/intel/caption_style.rs new file mode 100644 index 00000000..6067076b --- /dev/null +++ b/app/src-tauri/src/intel/caption_style.rs @@ -0,0 +1,102 @@ +//! How the person set up closed captions in Windows (A2-25): the text size, font style, and colors. The live caption +//! strip follows them, so captions look the way the person has already chosen for everything else. Read-only: this +//! changes no system setting. + +use serde::Serialize; + +use crate::ipc::IpcResult; + +/// The look of captions. Colors are `#rrggbb`, opacities run from 0 to 1, and `scale` is the size against normal. +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CaptionStyle { + pub font_color: String, + pub font_opacity: f32, + pub scale: f32, + /// `serif`, `sans`, `mono`, `casual`, `cursive`, or `small-caps`. + pub family: &'static str, + pub background_color: String, + pub background_opacity: f32, + pub region_color: String, + pub region_opacity: f32, +} + +fn opacity(value: i32, default: f32) -> f32 { + match value { + 1 => 1.0, + 2 => 0.75, + 3 => 0.25, + 4 => 0.0, + _ => default, + } +} + +fn scale(value: i32) -> f32 { + match value { + 1 => 0.5, + 3 => 1.5, + 4 => 2.0, + _ => 1.0, + } +} + +fn family(value: i32) -> &'static str { + match value { + 1 | 2 => "serif", + 3 => "mono", + 4 => "sans", + 5 => "casual", + 6 => "cursive", + 7 => "small-caps", + _ => "sans", + } +} + +fn hex(r: u8, g: u8, b: u8) -> String { + format!("#{r:02x}{g:02x}{b:02x}") +} + +#[cfg(windows)] +fn read() -> Option { + use windows::Media::ClosedCaptioning::ClosedCaptionProperties as Props; + let font = Props::ComputedFontColor().ok()?; + let background = Props::ComputedBackgroundColor().ok()?; + let region = Props::ComputedRegionColor().ok()?; + Some(CaptionStyle { + font_color: hex(font.R, font.G, font.B), + font_opacity: opacity(Props::FontOpacity().ok()?.0, 1.0), + scale: scale(Props::FontSize().ok()?.0), + family: family(Props::FontStyle().ok()?.0), + background_color: hex(background.R, background.G, background.B), + background_opacity: opacity(Props::BackgroundOpacity().ok()?.0, 0.75), + region_color: hex(region.R, region.G, region.B), + region_opacity: opacity(Props::RegionOpacity().ok()?.0, 0.0), + }) +} + +#[cfg(not(windows))] +fn read() -> Option { + None +} + +/// The caption look chosen in Windows, or none where Windows has no such setting. +#[tauri::command] +pub async fn intel_caption_style() -> IpcResult> { + Ok(read()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn windows_choices_become_css_numbers() { + assert_eq!(opacity(1, 0.5), 1.0); + assert_eq!(opacity(4, 0.5), 0.0); + assert_eq!(opacity(0, 0.5), 0.5); + assert_eq!(scale(4), 2.0); + assert_eq!(scale(0), 1.0); + assert_eq!(family(3), "mono"); + assert_eq!(hex(255, 0, 16), "#ff0010"); + } +} diff --git a/app/src-tauri/src/intel/catalog.rs b/app/src-tauri/src/intel/catalog.rs new file mode 100644 index 00000000..d95ada28 --- /dev/null +++ b/app/src-tauri/src/intel/catalog.rs @@ -0,0 +1,1087 @@ +//! The catalog of the downloads beyond speech models: the language models, the meaning model, and the translation +//! packs. Generated from the Hugging Face file listings at the revisions named in each address, so every address is +//! pinned to one commit and every file has the size and checksum published for it. A file Hugging Face stores +//! outside its large-file storage has only git's own hash published for it, so those carry that instead. + +use super::models::{Digest, ModelSpec, Part}; + +pub const LANGUAGE: &[ModelSpec] = &[ + ModelSpec { + id: "language-small", + kind: "language", + name: "Writing and Ask, small", + detail: "Fast and light. Good for short answers and simple rewrites.", + parts: &[ + Part { + file: "qwen2.5-0.5b-instruct-q4_k_m.gguf", + size: 491400032, + digest: Digest::Sha256("74a4da8c9fdbcd15bd1f6d01d621410d31c6fc00986f5eb687824e7b93d7a9db"), + url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/9217f5db79a29953eb74d5343926648285ec7e67/qwen2.5-0.5b-instruct-q4_k_m.gguf", + }, + ], + }, + ModelSpec { + id: "language-balanced", + kind: "language", + name: "Writing and Ask, balanced", + detail: "Clearer writing and better answers, and slower. Recommended if your computer has 8 GB of memory or more.", + parts: &[ + Part { + file: "qwen2.5-1.5b-instruct-q4_k_m.gguf", + size: 1117320736, + digest: Digest::Sha256("6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e"), + url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/91cad51170dc346986eccefdc2dd33a9da36ead9/qwen2.5-1.5b-instruct-q4_k_m.gguf", + }, + ], + }, +]; + +pub const MEANING: &[ModelSpec] = &[ModelSpec { + id: "meaning-minilm", + kind: "meaning", + name: "Find by meaning, better", + detail: "A small model that reads what your pages mean, so a search can find them by idea and not only by word.", + parts: &[ + Part { + file: "meaning-minilm/model.onnx", + size: 22972370, + digest: Digest::Sha256("afdb6f1a0e45b715d0bb9b11772f032c399babd23bfc31fed1c170afc848bdb1"), + url: "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/751bff37182d3f1213fa05d7196b954e230abad9/onnx/model_quantized.onnx", + }, + Part { + file: "meaning-minilm/tokenizer.json", + size: 711661, + digest: Digest::GitBlob("c17ed520ed8438736732a54957a69306b8822215"), + url: "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/751bff37182d3f1213fa05d7196b954e230abad9/tokenizer.json", + }, + ], +}]; + +pub const TRANSLATE: &[ModelSpec] = &[ + ModelSpec { + id: "translate-en-es", + kind: "translate", + name: "English to Spanish", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-es/encoder.onnx", + size: 52899742, + digest: Digest::Sha256("03434edd1147e6c49e22066e9dc5888235e99cc35a6711d1c4ea5b06455d5da9"), + url: "https://huggingface.co/Xenova/opus-mt-en-es/resolve/4b002a4c7edd54a7ced58877258b87f7efd3f892/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-es/decoder.onnx", + size: 59842102, + digest: Digest::Sha256("2b45074cc819f1d901e40510d7bc0781d99327d16c7eca04518c4ee0c3d6ba10"), + url: "https://huggingface.co/Xenova/opus-mt-en-es/resolve/4b002a4c7edd54a7ced58877258b87f7efd3f892/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-es/tokenizer.json", + size: 6262682, + digest: Digest::GitBlob("30e25a315e2e6ed4f9fa5ddba3182995f7f4d9f7"), + url: "https://huggingface.co/Xenova/opus-mt-en-es/resolve/4b002a4c7edd54a7ced58877258b87f7efd3f892/tokenizer.json", + }, + Part { + file: "translate-en-es/config.json", + size: 1468, + digest: Digest::GitBlob("1a12ace130d96e926d1d3ca34ba11c83e51cbcde"), + url: "https://huggingface.co/Xenova/opus-mt-en-es/resolve/4b002a4c7edd54a7ced58877258b87f7efd3f892/config.json", + }, + Part { + file: "translate-en-es/generation_config.json", + size: 293, + digest: Digest::GitBlob("015c3e06e8eac3e32e2078ba3140f246e754ae7b"), + url: "https://huggingface.co/Xenova/opus-mt-en-es/resolve/4b002a4c7edd54a7ced58877258b87f7efd3f892/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-es-en", + kind: "translate", + name: "Spanish to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-es-en/encoder.onnx", + size: 52899742, + digest: Digest::Sha256("c01e70f8455efc350831aa2af7ed187b16829241f0018ba07d1ba643a391bc18"), + url: "https://huggingface.co/Xenova/opus-mt-es-en/resolve/eadfd7c658a9d8929ac3b8e996b68a68e2c7d480/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-es-en/decoder.onnx", + size: 59842102, + digest: Digest::Sha256("d518f1f0041b2831f0aeb5a087cf899742cd9d4b574f6b98ad9f816b8c06925b"), + url: "https://huggingface.co/Xenova/opus-mt-es-en/resolve/eadfd7c658a9d8929ac3b8e996b68a68e2c7d480/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-es-en/tokenizer.json", + size: 6262682, + digest: Digest::GitBlob("30e25a315e2e6ed4f9fa5ddba3182995f7f4d9f7"), + url: "https://huggingface.co/Xenova/opus-mt-es-en/resolve/eadfd7c658a9d8929ac3b8e996b68a68e2c7d480/tokenizer.json", + }, + Part { + file: "translate-es-en/config.json", + size: 1433, + digest: Digest::GitBlob("0761a7bbe6cc7f76dd7a52599bb2c9722ee80aa4"), + url: "https://huggingface.co/Xenova/opus-mt-es-en/resolve/eadfd7c658a9d8929ac3b8e996b68a68e2c7d480/config.json", + }, + Part { + file: "translate-es-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("015c3e06e8eac3e32e2078ba3140f246e754ae7b"), + url: "https://huggingface.co/Xenova/opus-mt-es-en/resolve/eadfd7c658a9d8929ac3b8e996b68a68e2c7d480/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-fr", + kind: "translate", + name: "English to French", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-fr/encoder.onnx", + size: 50090398, + digest: Digest::Sha256("0a81bdba62f53223740a8c6c6f58e716eba8ea1f92dfa569caf27ed5e3b6a0f7"), + url: "https://huggingface.co/Xenova/opus-mt-en-fr/resolve/28726206f80896b90035bd99cccd5cc1e151f916/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-fr/decoder.onnx", + size: 57010810, + digest: Digest::Sha256("bdacc5f38e4d2b805a2fc9f38cf43844cdea56666e735137dbacf2a54a785257"), + url: "https://huggingface.co/Xenova/opus-mt-en-fr/resolve/28726206f80896b90035bd99cccd5cc1e151f916/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-fr/tokenizer.json", + size: 5637839, + digest: Digest::GitBlob("75a7ff0cd7f07b02bf1bada7aaebb78a3df6ab0c"), + url: "https://huggingface.co/Xenova/opus-mt-en-fr/resolve/28726206f80896b90035bd99cccd5cc1e151f916/tokenizer.json", + }, + Part { + file: "translate-en-fr/config.json", + size: 1411, + digest: Digest::GitBlob("0384b144d7dd05ba622f17ffb3e77a76f0dba743"), + url: "https://huggingface.co/Xenova/opus-mt-en-fr/resolve/28726206f80896b90035bd99cccd5cc1e151f916/config.json", + }, + Part { + file: "translate-en-fr/generation_config.json", + size: 293, + digest: Digest::GitBlob("22387016535f6300a9f8c5b68055b5c0ffe5cc34"), + url: "https://huggingface.co/Xenova/opus-mt-en-fr/resolve/28726206f80896b90035bd99cccd5cc1e151f916/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-fr-en", + kind: "translate", + name: "French to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-fr-en/encoder.onnx", + size: 50090398, + digest: Digest::Sha256("e727cb26ac6bf816394c49671af69c9ae5798868fed1e40849872415dffc1772"), + url: "https://huggingface.co/Xenova/opus-mt-fr-en/resolve/6b166a182780e118c997879d0ad5be4b53671644/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-fr-en/decoder.onnx", + size: 57010810, + digest: Digest::Sha256("8c5a0c5d113a1c4c5ae46e014aacf770f3bff2109fe8b74ead6aae5c5f408ffc"), + url: "https://huggingface.co/Xenova/opus-mt-fr-en/resolve/6b166a182780e118c997879d0ad5be4b53671644/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-fr-en/tokenizer.json", + size: 5637839, + digest: Digest::GitBlob("75a7ff0cd7f07b02bf1bada7aaebb78a3df6ab0c"), + url: "https://huggingface.co/Xenova/opus-mt-fr-en/resolve/6b166a182780e118c997879d0ad5be4b53671644/tokenizer.json", + }, + Part { + file: "translate-fr-en/config.json", + size: 1411, + digest: Digest::GitBlob("c313d87894f90a99ec40f30eccd85c1210d84ae9"), + url: "https://huggingface.co/Xenova/opus-mt-fr-en/resolve/6b166a182780e118c997879d0ad5be4b53671644/config.json", + }, + Part { + file: "translate-fr-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("22387016535f6300a9f8c5b68055b5c0ffe5cc34"), + url: "https://huggingface.co/Xenova/opus-mt-fr-en/resolve/6b166a182780e118c997879d0ad5be4b53671644/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-de", + kind: "translate", + name: "English to German", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-de/encoder.onnx", + size: 49366942, + digest: Digest::Sha256("15834b45fabd2dfb8c6c029b3ca3e7289aeefd90ece798ce42bcf548d1bd3b8d"), + url: "https://huggingface.co/Xenova/opus-mt-en-de/resolve/1ca130c44c4c5441ef16d48aae521a424ab644f7/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-de/decoder.onnx", + size: 56281702, + digest: Digest::Sha256("75ef79aa9bde9e3dce9ca584c29507be5f464973f6c600c89ff419bc8de29ebc"), + url: "https://huggingface.co/Xenova/opus-mt-en-de/resolve/1ca130c44c4c5441ef16d48aae521a424ab644f7/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-de/tokenizer.json", + size: 5498450, + digest: Digest::GitBlob("e9f5162c6f280e93955b4cadaddb0b8645fa94c7"), + url: "https://huggingface.co/Xenova/opus-mt-en-de/resolve/1ca130c44c4c5441ef16d48aae521a424ab644f7/tokenizer.json", + }, + Part { + file: "translate-en-de/config.json", + size: 1411, + digest: Digest::GitBlob("188a79d8e9befa9bce0a1e4648a5a21db0ac34f3"), + url: "https://huggingface.co/Xenova/opus-mt-en-de/resolve/1ca130c44c4c5441ef16d48aae521a424ab644f7/config.json", + }, + Part { + file: "translate-en-de/generation_config.json", + size: 293, + digest: Digest::GitBlob("a99b643eda6cb47bdb49c9cf5408c689461e5419"), + url: "https://huggingface.co/Xenova/opus-mt-en-de/resolve/1ca130c44c4c5441ef16d48aae521a424ab644f7/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-de-en", + kind: "translate", + name: "German to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-de-en/encoder.onnx", + size: 49366942, + digest: Digest::Sha256("4cedda8f8c89b72a42b3c6cd1e7a27f2de24457093e3bf80cb3e46829641fcd8"), + url: "https://huggingface.co/Xenova/opus-mt-de-en/resolve/399dfd68706739fffd503f876093e455ae268a06/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-de-en/decoder.onnx", + size: 56281702, + digest: Digest::Sha256("e44c1c4b50e8f51e49d4d5e54a9af1550dc74763d4023644a38af62611e6efc9"), + url: "https://huggingface.co/Xenova/opus-mt-de-en/resolve/399dfd68706739fffd503f876093e455ae268a06/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-de-en/tokenizer.json", + size: 5498450, + digest: Digest::GitBlob("e9f5162c6f280e93955b4cadaddb0b8645fa94c7"), + url: "https://huggingface.co/Xenova/opus-mt-de-en/resolve/399dfd68706739fffd503f876093e455ae268a06/tokenizer.json", + }, + Part { + file: "translate-de-en/config.json", + size: 1376, + digest: Digest::GitBlob("27f00c653f028b409f10f59b9aa22b855bd463a4"), + url: "https://huggingface.co/Xenova/opus-mt-de-en/resolve/399dfd68706739fffd503f876093e455ae268a06/config.json", + }, + Part { + file: "translate-de-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("a99b643eda6cb47bdb49c9cf5408c689461e5419"), + url: "https://huggingface.co/Xenova/opus-mt-de-en/resolve/399dfd68706739fffd503f876093e455ae268a06/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-it", + kind: "translate", + name: "English to Italian", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-it/encoder.onnx", + size: 60597150, + digest: Digest::Sha256("b51c5110c7ba356ef01ee400d427afe162e657de1d7a2e9a40deedd811b49e6f"), + url: "https://huggingface.co/Xenova/opus-mt-en-it/resolve/075406e3c8c2c30634d4a1bd8f00c21d9e162011/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-it/decoder.onnx", + size: 67599646, + digest: Digest::Sha256("d5c77fd7f60eebdf81d3f6f6fecd8e6804529cbcbecc2bb742fd60f225e2cf3e"), + url: "https://huggingface.co/Xenova/opus-mt-en-it/resolve/075406e3c8c2c30634d4a1bd8f00c21d9e162011/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-it/tokenizer.json", + size: 7904597, + digest: Digest::GitBlob("e02c1e0dcedb9b317a4ddb032fa9dadde10910b4"), + url: "https://huggingface.co/Xenova/opus-mt-en-it/resolve/075406e3c8c2c30634d4a1bd8f00c21d9e162011/tokenizer.json", + }, + Part { + file: "translate-en-it/config.json", + size: 1376, + digest: Digest::GitBlob("493415a1cf0731e06c5fddda4dfe00012d03e9ab"), + url: "https://huggingface.co/Xenova/opus-mt-en-it/resolve/075406e3c8c2c30634d4a1bd8f00c21d9e162011/config.json", + }, + Part { + file: "translate-en-it/generation_config.json", + size: 293, + digest: Digest::GitBlob("f8c1049bf577e8178c6acf0d558f9a8cd07aff9d"), + url: "https://huggingface.co/Xenova/opus-mt-en-it/resolve/075406e3c8c2c30634d4a1bd8f00c21d9e162011/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-it-en", + kind: "translate", + name: "Italian to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-it-en/encoder.onnx", + size: 60773278, + digest: Digest::Sha256("a025d2d85c90da23880116b67175e43e71921d8d2bf446b9f1eacdc86015aa4c"), + url: "https://huggingface.co/Xenova/opus-mt-it-en/resolve/fd0b89b9c052adc1f2f64152f555aa17353728be/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-it-en/decoder.onnx", + size: 67777150, + digest: Digest::Sha256("203bac7b98e438e65e1ddfcbec254ffb6811e6a8f4f71e93517ca9550323bf6e"), + url: "https://huggingface.co/Xenova/opus-mt-it-en/resolve/fd0b89b9c052adc1f2f64152f555aa17353728be/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-it-en/tokenizer.json", + size: 7942314, + digest: Digest::GitBlob("98045dc27b25cf71138b36ddbe3bda60e6daa9fc"), + url: "https://huggingface.co/Xenova/opus-mt-it-en/resolve/fd0b89b9c052adc1f2f64152f555aa17353728be/tokenizer.json", + }, + Part { + file: "translate-it-en/config.json", + size: 1376, + digest: Digest::GitBlob("d775dbf362457fac5cfa3ec2e7a7c423bb9844fc"), + url: "https://huggingface.co/Xenova/opus-mt-it-en/resolve/fd0b89b9c052adc1f2f64152f555aa17353728be/config.json", + }, + Part { + file: "translate-it-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("9224e175f4c1eeb51acca946ae5817cd35a363f1"), + url: "https://huggingface.co/Xenova/opus-mt-it-en/resolve/fd0b89b9c052adc1f2f64152f555aa17353728be/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-nl", + kind: "translate", + name: "English to Dutch", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-nl/encoder.onnx", + size: 53937565, + digest: Digest::Sha256("0febb990f8ca8ed9fd3e0017dfcb55ec5293999f20ead81d2bb4370e9e3120ff"), + url: "https://huggingface.co/Xenova/opus-mt-en-nl/resolve/c3a0541231a7329b4492809fc2bd7ab7d2ceb0bc/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-nl/decoder.onnx", + size: 60888033, + digest: Digest::Sha256("e9d1474106c1bdf8aaf236eb7ace8a29edf6fee655c583132f521e5b63ccbf97"), + url: "https://huggingface.co/Xenova/opus-mt-en-nl/resolve/c3a0541231a7329b4492809fc2bd7ab7d2ceb0bc/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-nl/tokenizer.json", + size: 6433167, + digest: Digest::GitBlob("9ba4b5114d0f45310812481536998c609f999091"), + url: "https://huggingface.co/Xenova/opus-mt-en-nl/resolve/c3a0541231a7329b4492809fc2bd7ab7d2ceb0bc/tokenizer.json", + }, + Part { + file: "translate-en-nl/config.json", + size: 1376, + digest: Digest::GitBlob("3eeb93a3605e2e166c3d4d47865cb4b744685bbf"), + url: "https://huggingface.co/Xenova/opus-mt-en-nl/resolve/c3a0541231a7329b4492809fc2bd7ab7d2ceb0bc/config.json", + }, + Part { + file: "translate-en-nl/generation_config.json", + size: 293, + digest: Digest::GitBlob("4ab9d470a0e79d814a268cdaea4d6462b21528cc"), + url: "https://huggingface.co/Xenova/opus-mt-en-nl/resolve/c3a0541231a7329b4492809fc2bd7ab7d2ceb0bc/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-nl-en", + kind: "translate", + name: "Dutch to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-nl-en/encoder.onnx", + size: 53937566, + digest: Digest::Sha256("7b8b1e9071bed8e2394e3bd2f65b952700b1dd1293d087141134c8bac018653b"), + url: "https://huggingface.co/Xenova/opus-mt-nl-en/resolve/82c42d95d1508037cbe7172d85fd4e940a2f3584/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-nl-en/decoder.onnx", + size: 60888034, + digest: Digest::Sha256("d70da1130c5a0746f8f18081854511b5a0671e563a56b4dd7870b2eb922944cc"), + url: "https://huggingface.co/Xenova/opus-mt-nl-en/resolve/82c42d95d1508037cbe7172d85fd4e940a2f3584/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-nl-en/tokenizer.json", + size: 6433167, + digest: Digest::GitBlob("9ba4b5114d0f45310812481536998c609f999091"), + url: "https://huggingface.co/Xenova/opus-mt-nl-en/resolve/82c42d95d1508037cbe7172d85fd4e940a2f3584/tokenizer.json", + }, + Part { + file: "translate-nl-en/config.json", + size: 1376, + digest: Digest::GitBlob("85a94e8eea72df569aab43f9487731d573e8bb73"), + url: "https://huggingface.co/Xenova/opus-mt-nl-en/resolve/82c42d95d1508037cbe7172d85fd4e940a2f3584/config.json", + }, + Part { + file: "translate-nl-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("51217edaed11e5353ef646ff1c33146621e3a3f1"), + url: "https://huggingface.co/Xenova/opus-mt-nl-en/resolve/82c42d95d1508037cbe7172d85fd4e940a2f3584/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-ru", + kind: "translate", + name: "English to Russian", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-ru/encoder.onnx", + size: 51628446, + digest: Digest::Sha256("97f28d8295d231b7da721ded06fa8c034787df2122cd8f85b904abdc4d15bd53"), + url: "https://huggingface.co/Xenova/opus-mt-en-ru/resolve/e050376e960175e8ddc5cff85025dc8436cddd68/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-ru/decoder.onnx", + size: 58560874, + digest: Digest::Sha256("7e4e602c31130176b086cd4bf31d6d08d44a6a367617e910870e0ea3e5408574"), + url: "https://huggingface.co/Xenova/opus-mt-en-ru/resolve/e050376e960175e8ddc5cff85025dc8436cddd68/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-ru/tokenizer.json", + size: 7205388, + digest: Digest::GitBlob("3b14111caaaf9af454726e1677c64b26339ad521"), + url: "https://huggingface.co/Xenova/opus-mt-en-ru/resolve/e050376e960175e8ddc5cff85025dc8436cddd68/tokenizer.json", + }, + Part { + file: "translate-en-ru/config.json", + size: 1376, + digest: Digest::GitBlob("cbee9bc43dcbf54f4fbfa95133cb6337f74ccafc"), + url: "https://huggingface.co/Xenova/opus-mt-en-ru/resolve/e050376e960175e8ddc5cff85025dc8436cddd68/config.json", + }, + Part { + file: "translate-en-ru/generation_config.json", + size: 293, + digest: Digest::GitBlob("40ef46c574097d9a2dd14ba2e63ae537674d864d"), + url: "https://huggingface.co/Xenova/opus-mt-en-ru/resolve/e050376e960175e8ddc5cff85025dc8436cddd68/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-ru-en", + kind: "translate", + name: "Russian to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-ru-en/encoder.onnx", + size: 51628446, + digest: Digest::Sha256("c10df0466540534bb70c4402d539c349b8bfa9968184cad89f51408cf13e7eb7"), + url: "https://huggingface.co/Xenova/opus-mt-ru-en/resolve/afe8c6c738ec81b6d033fd8f44f9678a639a7c67/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-ru-en/decoder.onnx", + size: 58560874, + digest: Digest::Sha256("d880abf58612013815554ef55c98d0f28b68190b2cb6f65c39d6e834f51dda8c"), + url: "https://huggingface.co/Xenova/opus-mt-ru-en/resolve/afe8c6c738ec81b6d033fd8f44f9678a639a7c67/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-ru-en/tokenizer.json", + size: 7205388, + digest: Digest::GitBlob("3b14111caaaf9af454726e1677c64b26339ad521"), + url: "https://huggingface.co/Xenova/opus-mt-ru-en/resolve/afe8c6c738ec81b6d033fd8f44f9678a639a7c67/tokenizer.json", + }, + Part { + file: "translate-ru-en/config.json", + size: 1376, + digest: Digest::GitBlob("efe5b5bcfc175072c1b4b60435a2a7d0fcfb5b10"), + url: "https://huggingface.co/Xenova/opus-mt-ru-en/resolve/afe8c6c738ec81b6d033fd8f44f9678a639a7c67/config.json", + }, + Part { + file: "translate-ru-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("c3f19d9ef5965b76099efe81189c0d1d5518b694"), + url: "https://huggingface.co/Xenova/opus-mt-ru-en/resolve/afe8c6c738ec81b6d033fd8f44f9678a639a7c67/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-zh", + kind: "translate", + name: "English to Chinese", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-zh/encoder.onnx", + size: 52899742, + digest: Digest::Sha256("d3b7912bf6a9bd27e4c074c2df91d4ff3d5b4bc5f7f6c8d7cc9c805c98fbafee"), + url: "https://huggingface.co/Xenova/opus-mt-en-zh/resolve/046f55aec303cdee3e0318604406d4df20f1e8ea/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-zh/decoder.onnx", + size: 59842102, + digest: Digest::Sha256("2c66a3981099b40edbb3a0d65e015d05964d42fecb9780f8422776eff5939112"), + url: "https://huggingface.co/Xenova/opus-mt-en-zh/resolve/046f55aec303cdee3e0318604406d4df20f1e8ea/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-zh/tokenizer.json", + size: 6380952, + digest: Digest::GitBlob("7106ca5b95c748f039dd9adcc13c7542b997762d"), + url: "https://huggingface.co/Xenova/opus-mt-en-zh/resolve/046f55aec303cdee3e0318604406d4df20f1e8ea/tokenizer.json", + }, + Part { + file: "translate-en-zh/config.json", + size: 1503, + digest: Digest::GitBlob("787a84798a695a75720fa01ad2881c8d193c45f2"), + url: "https://huggingface.co/Xenova/opus-mt-en-zh/resolve/046f55aec303cdee3e0318604406d4df20f1e8ea/config.json", + }, + Part { + file: "translate-en-zh/generation_config.json", + size: 293, + digest: Digest::GitBlob("015c3e06e8eac3e32e2078ba3140f246e754ae7b"), + url: "https://huggingface.co/Xenova/opus-mt-en-zh/resolve/046f55aec303cdee3e0318604406d4df20f1e8ea/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-zh-en", + kind: "translate", + name: "Chinese to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-zh-en/encoder.onnx", + size: 52899742, + digest: Digest::Sha256("84d5e171b626bc8b6b220d022ac58696e9528c25deeacca62b5cbf4364547a99"), + url: "https://huggingface.co/Xenova/opus-mt-zh-en/resolve/39d480d52a9ea3065a1f117adfe4dbc55de10e6f/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-zh-en/decoder.onnx", + size: 59842102, + digest: Digest::Sha256("debcc3054b9ff9aaed972bd6e251a459abe1e2f1402da1efa63163dc2f46ef73"), + url: "https://huggingface.co/Xenova/opus-mt-zh-en/resolve/39d480d52a9ea3065a1f117adfe4dbc55de10e6f/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-zh-en/tokenizer.json", + size: 6381339, + digest: Digest::GitBlob("545ab64f6d0ae9bbbb0c8899f04e1ae4f805fc87"), + url: "https://huggingface.co/Xenova/opus-mt-zh-en/resolve/39d480d52a9ea3065a1f117adfe4dbc55de10e6f/tokenizer.json", + }, + Part { + file: "translate-zh-en/config.json", + size: 1389, + digest: Digest::GitBlob("dcd3adfa622976cc4a51c68d115dc9d443d3cbd0"), + url: "https://huggingface.co/Xenova/opus-mt-zh-en/resolve/39d480d52a9ea3065a1f117adfe4dbc55de10e6f/config.json", + }, + Part { + file: "translate-zh-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("11cf47bfd6c2e512bf7ab5d58d49ab109732647a"), + url: "https://huggingface.co/Xenova/opus-mt-zh-en/resolve/39d480d52a9ea3065a1f117adfe4dbc55de10e6f/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-ar", + kind: "translate", + name: "English to Arabic", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-ar/encoder.onnx", + size: 51773854, + digest: Digest::Sha256("53c354d87ced44b39085fe8a46a25f09ea4578d442dbce9663f3977228f8a04e"), + url: "https://huggingface.co/Xenova/opus-mt-en-ar/resolve/034a684356c19021a187cda4d7f823298b913921/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-ar/decoder.onnx", + size: 58707418, + digest: Digest::Sha256("94e6bdf90866151787d014b7b77be0adb0f036418ee04d235ec01238aa8426b9"), + url: "https://huggingface.co/Xenova/opus-mt-en-ar/resolve/034a684356c19021a187cda4d7f823298b913921/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-ar/tokenizer.json", + size: 6748848, + digest: Digest::GitBlob("074a6b5ead40759fdbbe61ce5ccaf194e073e35d"), + url: "https://huggingface.co/Xenova/opus-mt-en-ar/resolve/034a684356c19021a187cda4d7f823298b913921/tokenizer.json", + }, + Part { + file: "translate-en-ar/config.json", + size: 1389, + digest: Digest::GitBlob("c4756a24ae0f5ceae268ac6c4d43ee8d4ffdd259"), + url: "https://huggingface.co/Xenova/opus-mt-en-ar/resolve/034a684356c19021a187cda4d7f823298b913921/config.json", + }, + Part { + file: "translate-en-ar/generation_config.json", + size: 293, + digest: Digest::GitBlob("9c739b102ea65f277241288c29dd73f659df93d2"), + url: "https://huggingface.co/Xenova/opus-mt-en-ar/resolve/034a684356c19021a187cda4d7f823298b913921/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-ar-en", + kind: "translate", + name: "Arabic to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-ar-en/encoder.onnx", + size: 51790238, + digest: Digest::Sha256("d9b0d612acff86cf87634b49bda566d1ef39b312837de549786ea18fecefc708"), + url: "https://huggingface.co/Xenova/opus-mt-ar-en/resolve/09c7556866400dcc53562ba8d7035119e7d9a2c1/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-ar-en/decoder.onnx", + size: 58723930, + digest: Digest::Sha256("66f2be0af1ad3d875315f7e8ad2de4f8248fba75dc7173b1a93bf48023041e80"), + url: "https://huggingface.co/Xenova/opus-mt-ar-en/resolve/09c7556866400dcc53562ba8d7035119e7d9a2c1/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-ar-en/tokenizer.json", + size: 6734647, + digest: Digest::GitBlob("3efa97e355ff9bf8abc2bdb4e948affba012389e"), + url: "https://huggingface.co/Xenova/opus-mt-ar-en/resolve/09c7556866400dcc53562ba8d7035119e7d9a2c1/tokenizer.json", + }, + Part { + file: "translate-ar-en/config.json", + size: 1376, + digest: Digest::GitBlob("82cd89d073952adcb4e56c229e4c1006e3fe432a"), + url: "https://huggingface.co/Xenova/opus-mt-ar-en/resolve/09c7556866400dcc53562ba8d7035119e7d9a2c1/config.json", + }, + Part { + file: "translate-ar-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("7f73f68a58fae3741aadf300bd2d9db60cdd5704"), + url: "https://huggingface.co/Xenova/opus-mt-ar-en/resolve/09c7556866400dcc53562ba8d7035119e7d9a2c1/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-hi", + kind: "translate", + name: "English to Hindi", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-hi/encoder.onnx", + size: 51337630, + digest: Digest::Sha256("3045cd6eaf06db96cff853204d6f0ffeb0c5af238a4f5a87a19fcaa94f7493c4"), + url: "https://huggingface.co/Xenova/opus-mt-en-hi/resolve/7aad72006a1588a9c3485ed6203aed99eebbc872/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-hi/decoder.onnx", + size: 58267786, + digest: Digest::Sha256("f563f658721046b17aa3f3ec33d12efdb1497c962876ff7d7a5ca513f6b49f4e"), + url: "https://huggingface.co/Xenova/opus-mt-en-hi/resolve/7aad72006a1588a9c3485ed6203aed99eebbc872/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-hi/tokenizer.json", + size: 6679560, + digest: Digest::GitBlob("c482b5926001aaa959c2ad5dc8839d4e9de055f1"), + url: "https://huggingface.co/Xenova/opus-mt-en-hi/resolve/7aad72006a1588a9c3485ed6203aed99eebbc872/tokenizer.json", + }, + Part { + file: "translate-en-hi/config.json", + size: 1389, + digest: Digest::GitBlob("47a20aef8837b30a1cf9e653bf4c0af0bea8f0d1"), + url: "https://huggingface.co/Xenova/opus-mt-en-hi/resolve/7aad72006a1588a9c3485ed6203aed99eebbc872/config.json", + }, + Part { + file: "translate-en-hi/generation_config.json", + size: 293, + digest: Digest::GitBlob("b0df89396a01c5b74d3798f21414b1a7885e2794"), + url: "https://huggingface.co/Xenova/opus-mt-en-hi/resolve/7aad72006a1588a9c3485ed6203aed99eebbc872/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-sv", + kind: "translate", + name: "English to Swedish", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-sv/encoder.onnx", + size: 48513438, + digest: Digest::Sha256("2477c172f2001ee486f2d21e452c4d74838af973dfb58d00cf53632c2e3f3dba"), + url: "https://huggingface.co/Xenova/opus-mt-en-sv/resolve/33d522d806c041cdebecd549c558d22a2b9b4ea4/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-sv/decoder.onnx", + size: 55421530, + digest: Digest::Sha256("953946225fb335da55b2d0568ae55cc30bacaf598920055c3cacf199e785cb6c"), + url: "https://huggingface.co/Xenova/opus-mt-en-sv/resolve/33d522d806c041cdebecd549c558d22a2b9b4ea4/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-sv/tokenizer.json", + size: 5450794, + digest: Digest::GitBlob("ecf8096e48a03536eed0801e1d687d00d62d73db"), + url: "https://huggingface.co/Xenova/opus-mt-en-sv/resolve/33d522d806c041cdebecd549c558d22a2b9b4ea4/tokenizer.json", + }, + Part { + file: "translate-en-sv/config.json", + size: 1376, + digest: Digest::GitBlob("4cbf1f4b2c5ad5f1deceea69d28e277bfc3df645"), + url: "https://huggingface.co/Xenova/opus-mt-en-sv/resolve/33d522d806c041cdebecd549c558d22a2b9b4ea4/config.json", + }, + Part { + file: "translate-en-sv/generation_config.json", + size: 293, + digest: Digest::GitBlob("5d61803a64b70079a46db42b120eda4a8bf9ea3b"), + url: "https://huggingface.co/Xenova/opus-mt-en-sv/resolve/33d522d806c041cdebecd549c558d22a2b9b4ea4/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-sv-en", + kind: "translate", + name: "Swedish to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-sv-en/encoder.onnx", + size: 48513438, + digest: Digest::Sha256("07ae6446ef6520117da869986e396ee55b05c471124ff851399701253fb306cc"), + url: "https://huggingface.co/Xenova/opus-mt-sv-en/resolve/63a40256869b6a2b29a84e47aa7f03524fa5f3a2/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-sv-en/decoder.onnx", + size: 55421530, + digest: Digest::Sha256("31965a3c359076de33f9676ec0ec88312b34c64a35d648e99f81bf79ca883ad4"), + url: "https://huggingface.co/Xenova/opus-mt-sv-en/resolve/63a40256869b6a2b29a84e47aa7f03524fa5f3a2/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-sv-en/tokenizer.json", + size: 5450794, + digest: Digest::GitBlob("ecf8096e48a03536eed0801e1d687d00d62d73db"), + url: "https://huggingface.co/Xenova/opus-mt-sv-en/resolve/63a40256869b6a2b29a84e47aa7f03524fa5f3a2/tokenizer.json", + }, + Part { + file: "translate-sv-en/config.json", + size: 1376, + digest: Digest::GitBlob("d5dc0a50088100c322a6868c8cee4d5ecd8c0a13"), + url: "https://huggingface.co/Xenova/opus-mt-sv-en/resolve/63a40256869b6a2b29a84e47aa7f03524fa5f3a2/config.json", + }, + Part { + file: "translate-sv-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("a89e5c1fc7bf249838fcd8bcedf9e70f1993af6d"), + url: "https://huggingface.co/Xenova/opus-mt-sv-en/resolve/63a40256869b6a2b29a84e47aa7f03524fa5f3a2/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-uk", + kind: "translate", + name: "English to Ukrainian", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-uk/encoder.onnx", + size: 51151774, + digest: Digest::Sha256("234c3738c258b3b7c913452a4ab6203b1a24f459355af8ef563dd4d94792d087"), + url: "https://huggingface.co/Xenova/opus-mt-en-uk/resolve/e6ca587292984b343fd829b6430ed55858496525/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-uk/decoder.onnx", + size: 58080478, + digest: Digest::Sha256("60881867e00ee1c5ef5fb79cde3a52e5d76437199731cd6896b1cd948bb3ea29"), + url: "https://huggingface.co/Xenova/opus-mt-en-uk/resolve/e6ca587292984b343fd829b6430ed55858496525/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-uk/tokenizer.json", + size: 6922026, + digest: Digest::GitBlob("cf8ebce00f976fa17ac7ab4218f5d79080525070"), + url: "https://huggingface.co/Xenova/opus-mt-en-uk/resolve/e6ca587292984b343fd829b6430ed55858496525/tokenizer.json", + }, + Part { + file: "translate-en-uk/config.json", + size: 1376, + digest: Digest::GitBlob("970d8085d84d648ab36ede6f79a703db002664f2"), + url: "https://huggingface.co/Xenova/opus-mt-en-uk/resolve/e6ca587292984b343fd829b6430ed55858496525/config.json", + }, + Part { + file: "translate-en-uk/generation_config.json", + size: 293, + digest: Digest::GitBlob("e57259a789f47157546973caea71ab45d5f779fc"), + url: "https://huggingface.co/Xenova/opus-mt-en-uk/resolve/e6ca587292984b343fd829b6430ed55858496525/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-uk-en", + kind: "translate", + name: "Ukrainian to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-uk-en/encoder.onnx", + size: 51151774, + digest: Digest::Sha256("5e8c4c7e88742efa0a8e013f3897f17e29357d54b8c86a13472669546d6ed561"), + url: "https://huggingface.co/Xenova/opus-mt-uk-en/resolve/b2ad6afa424aaeff8913ff350860b7a7b7c0a607/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-uk-en/decoder.onnx", + size: 58080478, + digest: Digest::Sha256("83b44c8e29a1667c010f9ce6a261c060f03d55f25b57715be7e148a090d8674e"), + url: "https://huggingface.co/Xenova/opus-mt-uk-en/resolve/b2ad6afa424aaeff8913ff350860b7a7b7c0a607/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-uk-en/tokenizer.json", + size: 6922026, + digest: Digest::GitBlob("cf8ebce00f976fa17ac7ab4218f5d79080525070"), + url: "https://huggingface.co/Xenova/opus-mt-uk-en/resolve/b2ad6afa424aaeff8913ff350860b7a7b7c0a607/tokenizer.json", + }, + Part { + file: "translate-uk-en/config.json", + size: 1376, + digest: Digest::GitBlob("486f0a7254a3615ff88d9de99daa9cb5ed90e531"), + url: "https://huggingface.co/Xenova/opus-mt-uk-en/resolve/b2ad6afa424aaeff8913ff350860b7a7b7c0a607/config.json", + }, + Part { + file: "translate-uk-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("690289af0a057bc2174b54794066ebb47b471f33"), + url: "https://huggingface.co/Xenova/opus-mt-uk-en/resolve/b2ad6afa424aaeff8913ff350860b7a7b7c0a607/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-tr-en", + kind: "translate", + name: "Turkish to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-tr-en/encoder.onnx", + size: 51562398, + digest: Digest::Sha256("b9b73903b99de7303f3c465b335a9cb695ba6c9c93ef527064f5fe2cf7a3235f"), + url: "https://huggingface.co/Xenova/opus-mt-tr-en/resolve/78247f41f354af036d24e6f0f8a3d02c6806f96e/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-tr-en/decoder.onnx", + size: 58494310, + digest: Digest::Sha256("70a8a968a4fe8cbf75733fb287957e53c204f17513ff7445e5a68e66874f1ec4"), + url: "https://huggingface.co/Xenova/opus-mt-tr-en/resolve/78247f41f354af036d24e6f0f8a3d02c6806f96e/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-tr-en/tokenizer.json", + size: 6123578, + digest: Digest::GitBlob("696fce2a16d0088546fca598c5cd63884ac608f0"), + url: "https://huggingface.co/Xenova/opus-mt-tr-en/resolve/78247f41f354af036d24e6f0f8a3d02c6806f96e/tokenizer.json", + }, + Part { + file: "translate-tr-en/config.json", + size: 1376, + digest: Digest::GitBlob("8a7b9c08b555b003c75ad03d5d68b690d55fa333"), + url: "https://huggingface.co/Xenova/opus-mt-tr-en/resolve/78247f41f354af036d24e6f0f8a3d02c6806f96e/config.json", + }, + Part { + file: "translate-tr-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("96630f8439fde84a9b04bb846e73843d843093f6"), + url: "https://huggingface.co/Xenova/opus-mt-tr-en/resolve/78247f41f354af036d24e6f0f8a3d02c6806f96e/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-pl-en", + kind: "translate", + name: "Polish to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-pl-en/encoder.onnx", + size: 52095390, + digest: Digest::Sha256("2416eb3c6db52a2d116539bf673b596e6a3d1453153b17778e85c9f55249c832"), + url: "https://huggingface.co/Xenova/opus-mt-pl-en/resolve/aeaf0e003b045248c28a559a5ce6027b54ebdaba/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-pl-en/decoder.onnx", + size: 59031466, + digest: Digest::Sha256("a459f0f27fcb96752c6b2c7cc14e576fe2049247c5dc310283fb783cce6908ca"), + url: "https://huggingface.co/Xenova/opus-mt-pl-en/resolve/aeaf0e003b045248c28a559a5ce6027b54ebdaba/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-pl-en/tokenizer.json", + size: 6118576, + digest: Digest::GitBlob("961e7f4e5d6d7ba1172d2e2616a0356a5f56b638"), + url: "https://huggingface.co/Xenova/opus-mt-pl-en/resolve/aeaf0e003b045248c28a559a5ce6027b54ebdaba/tokenizer.json", + }, + Part { + file: "translate-pl-en/config.json", + size: 1376, + digest: Digest::GitBlob("ea860d8185aa356c797d9840acc232327744c0af"), + url: "https://huggingface.co/Xenova/opus-mt-pl-en/resolve/aeaf0e003b045248c28a559a5ce6027b54ebdaba/config.json", + }, + Part { + file: "translate-pl-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("00c85f50777e53b9eebd81c957f2ff9f21895ae8"), + url: "https://huggingface.co/Xenova/opus-mt-pl-en/resolve/aeaf0e003b045248c28a559a5ce6027b54ebdaba/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-ja-en", + kind: "translate", + name: "Japanese to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-ja-en/encoder.onnx", + size: 50705822, + digest: Digest::Sha256("345262b16bcdda1468b0f3380c112b7ce79f731176b4b1d21f6edd5b2ae0d25c"), + url: "https://huggingface.co/Xenova/opus-mt-ja-en/resolve/1a906cfaaf7c8f4193f67f5885c082aa6dbd9d16/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-ja-en/decoder.onnx", + size: 57631042, + digest: Digest::Sha256("ec639efdbb8dd9a5c458ab3aeeaee34402b21200eb9c24a95a32a16f0bb22d66"), + url: "https://huggingface.co/Xenova/opus-mt-ja-en/resolve/1a906cfaaf7c8f4193f67f5885c082aa6dbd9d16/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-ja-en/tokenizer.json", + size: 5991485, + digest: Digest::GitBlob("2183cfa1bb47d494426f4903a86c19097dfd4e0e"), + url: "https://huggingface.co/Xenova/opus-mt-ja-en/resolve/1a906cfaaf7c8f4193f67f5885c082aa6dbd9d16/tokenizer.json", + }, + Part { + file: "translate-ja-en/config.json", + size: 1376, + digest: Digest::GitBlob("c64ee5662fb65d93fcca8837b957650a6e97e3fd"), + url: "https://huggingface.co/Xenova/opus-mt-ja-en/resolve/1a906cfaaf7c8f4193f67f5885c082aa6dbd9d16/config.json", + }, + Part { + file: "translate-ja-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("3f32cba8c6a674550f8e5490a739f63fb129e5cb"), + url: "https://huggingface.co/Xenova/opus-mt-ja-en/resolve/1a906cfaaf7c8f4193f67f5885c082aa6dbd9d16/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-ko-en", + kind: "translate", + name: "Korean to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-ko-en/encoder.onnx", + size: 52899741, + digest: Digest::Sha256("210e1d2cb5b457cdeeffe288099cc9d1df9f13c35dc3d39fde4ba252af877380"), + url: "https://huggingface.co/Xenova/opus-mt-ko-en/resolve/d9fa1ac6008242100fecb4fff9b0a5917a1be81f/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-ko-en/decoder.onnx", + size: 59842101, + digest: Digest::Sha256("0c961ea33a62ea84e73c9ec94c0737e65d06b5f73a7d0398b018fbbb7262d929"), + url: "https://huggingface.co/Xenova/opus-mt-ko-en/resolve/d9fa1ac6008242100fecb4fff9b0a5917a1be81f/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-ko-en/tokenizer.json", + size: 6487044, + digest: Digest::GitBlob("e767434793eb9b067d110140d7a11065dd36b7f8"), + url: "https://huggingface.co/Xenova/opus-mt-ko-en/resolve/d9fa1ac6008242100fecb4fff9b0a5917a1be81f/tokenizer.json", + }, + Part { + file: "translate-ko-en/config.json", + size: 1389, + digest: Digest::GitBlob("eab42d9fb5e5ceca5b18c91ed36b8f88d32689f4"), + url: "https://huggingface.co/Xenova/opus-mt-ko-en/resolve/d9fa1ac6008242100fecb4fff9b0a5917a1be81f/config.json", + }, + Part { + file: "translate-ko-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("11cf47bfd6c2e512bf7ab5d58d49ab109732647a"), + url: "https://huggingface.co/Xenova/opus-mt-ko-en/resolve/d9fa1ac6008242100fecb4fff9b0a5917a1be81f/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-en-vi", + kind: "translate", + name: "English to Vietnamese", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-en-vi/encoder.onnx", + size: 47105950, + digest: Digest::Sha256("dfcd7a32be3fdd21cbcf5d354520d626f1768fd6e205ca62a6a11a4641d31f36"), + url: "https://huggingface.co/Xenova/opus-mt-en-vi/resolve/30bcd461c0d7338ad106b36314d3eedc2d4e998c/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-en-vi/decoder.onnx", + size: 54003046, + digest: Digest::Sha256("cde3e02aaca72cfaf2267f6c31c4dcc1b323d769bf90c273f2a1c1d56198ddba"), + url: "https://huggingface.co/Xenova/opus-mt-en-vi/resolve/30bcd461c0d7338ad106b36314d3eedc2d4e998c/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-en-vi/tokenizer.json", + size: 5163544, + digest: Digest::GitBlob("8d97f5657db110e6e70fe43a52bbdf119df40671"), + url: "https://huggingface.co/Xenova/opus-mt-en-vi/resolve/30bcd461c0d7338ad106b36314d3eedc2d4e998c/tokenizer.json", + }, + Part { + file: "translate-en-vi/config.json", + size: 1389, + digest: Digest::GitBlob("86ba342d2fcd025ca82afec6169c2e96dd1f761c"), + url: "https://huggingface.co/Xenova/opus-mt-en-vi/resolve/30bcd461c0d7338ad106b36314d3eedc2d4e998c/config.json", + }, + Part { + file: "translate-en-vi/generation_config.json", + size: 293, + digest: Digest::GitBlob("20b9442c4b19ec2d6b172078b9bf0f23f177fae2"), + url: "https://huggingface.co/Xenova/opus-mt-en-vi/resolve/30bcd461c0d7338ad106b36314d3eedc2d4e998c/generation_config.json", + }, + ], + }, + ModelSpec { + id: "translate-vi-en", + kind: "translate", + name: "Vietnamese to English", + detail: "Translates on this computer. Nothing is sent anywhere.", + parts: &[ + Part { + file: "translate-vi-en/encoder.onnx", + size: 47133597, + digest: Digest::Sha256("fa7bd78e8bed6bdc03728f62f1592c4bb52d9e12795496604c1e990e102abdb2"), + url: "https://huggingface.co/Xenova/opus-mt-vi-en/resolve/541dfd72ec07e8f1563795e0224408ad8d729e80/onnx/encoder_model_quantized.onnx", + }, + Part { + file: "translate-vi-en/decoder.onnx", + size: 54030909, + digest: Digest::Sha256("7ba31d206ad237dbd35dbf46b9fe4d0368c082fd9e2d0e9603bd709facf4119d"), + url: "https://huggingface.co/Xenova/opus-mt-vi-en/resolve/541dfd72ec07e8f1563795e0224408ad8d729e80/onnx/decoder_model_quantized.onnx", + }, + Part { + file: "translate-vi-en/tokenizer.json", + size: 5169708, + digest: Digest::GitBlob("62f0f2e549ffc30605b8552a4f15d52a95f04677"), + url: "https://huggingface.co/Xenova/opus-mt-vi-en/resolve/541dfd72ec07e8f1563795e0224408ad8d729e80/tokenizer.json", + }, + Part { + file: "translate-vi-en/config.json", + size: 1389, + digest: Digest::GitBlob("5dc1ab6fcd695cf3b9c6686c061b2635dfad71d4"), + url: "https://huggingface.co/Xenova/opus-mt-vi-en/resolve/541dfd72ec07e8f1563795e0224408ad8d729e80/config.json", + }, + Part { + file: "translate-vi-en/generation_config.json", + size: 293, + digest: Digest::GitBlob("946ab291886018c2e63115669243914d0c4264d1"), + url: "https://huggingface.co/Xenova/opus-mt-vi-en/resolve/541dfd72ec07e8f1563795e0224408ad8d729e80/generation_config.json", + }, + ], + }, +]; diff --git a/app/src-tauri/src/intel/cloud.rs b/app/src-tauri/src/intel/cloud.rs new file mode 100644 index 00000000..35b76b03 --- /dev/null +++ b/app/src-tauri/src/intel/cloud.rs @@ -0,0 +1,615 @@ +//! A cloud service with the person's own key, chosen feature by feature (A1-33). Every feature runs on this device +//! unless the person picks "With my own key" for it, types a key, and confirms that the feature's input then goes to +//! the service. The key goes straight to Windows Credential Manager (`OpenNote/intel-cloud/`) and is never +//! handed back to the interface, written to a file or a log, or sent anywhere but the service's own host. A request +//! goes over HTTPS to the one pinned host only, and never while Work offline or safe mode is on. When it last ran +//! is kept for the Privacy panel. + +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, + sync::{Arc, Mutex, PoisonError}, + time::{SystemTime, UNIX_EPOCH}, +}; + +use opennote_intel::wire::TranscriptLine; +use serde::Serialize; +use serde_json::{json, Value}; + +use crate::{ + connectors::{Body, HostPolicy, Http, HttpError, HttpRequest, Method, Secret, SecretStore}, + settings::file::write_atomic, +}; + +/// The service a key is for. One provider for now: its API shape is what the requests below speak. +pub const PROVIDER: &str = "openai"; +/// The only host a key or a feature's input may go to. +pub const HOST: &str = "api.openai.com"; +const BASE: &str = "https://api.openai.com/v1"; + +/// The longest answer read back. +const MAX_ANSWER_BYTES: usize = 4 * 1024 * 1024; +/// Audio goes up in pieces of this many seconds, so each request stays small and progress can be told. +const PIECE_SECONDS: usize = 120; +/// The most text a summary request carries. +const MAX_SUMMARY_CHARS: usize = 48_000; +/// What the service is told to do with the text. +const SUMMARY_PROMPT: &str = "Summarize these notes in three to five short, plain sentences. \ + Keep names, numbers, and terms exactly as written. Answer with the summary only."; +const LAST_USED_FILE: &str = "cloud-used.json"; + +/// The features that can use a cloud key. The others always run on this device. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum CloudFeature { + Transcription, + Summaries, +} + +impl CloudFeature { + pub const ALL: [CloudFeature; 2] = [CloudFeature::Transcription, CloudFeature::Summaries]; + + pub fn id(self) -> &'static str { + match self { + CloudFeature::Transcription => "transcription", + CloudFeature::Summaries => "summaries", + } + } + + pub fn parse(text: &str) -> Option { + CloudFeature::ALL.into_iter().find(|feature| feature.id() == text) + } + + fn target(self) -> String { + format!("OpenNote/intel-cloud/{}", self.id()) + } +} + +/// Why a cloud call didn't happen or failed. None of the messages holds the key or the input. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CloudError { + Offline, + SafeMode, + NoKey, + /// The key isn't one a service could have issued: wrong length or characters. + BadKey, + /// The service refused the key. + Refused, + /// Too many requests, or the account is out of credit. + Limited, + Network, + /// Windows Credential Manager could not be read or written. + Store, + /// The answer was not in the shape expected. + Answer, + Canceled, +} + +impl CloudError { + /// The code the interface reads, from the list `services/intel/ext.ts` knows. + pub fn code(&self) -> &'static str { + match self { + CloudError::Offline => "offline", + CloudError::SafeMode => "safeMode", + CloudError::Network | CloudError::Limited | CloudError::Answer => "network", + CloudError::Store => "io", + CloudError::NoKey | CloudError::BadKey | CloudError::Refused => "invalid", + CloudError::Canceled => "canceled", + } + } + + pub fn message(&self) -> &'static str { + match self { + CloudError::Offline => "Work offline is on, so nothing is sent to the cloud service.", + CloudError::SafeMode => "Safe mode is on, so nothing is sent to the cloud service.", + CloudError::NoKey => "No key is saved for this feature.", + CloudError::BadKey => "That doesn't look like a key. Paste the whole key, with no spaces.", + CloudError::Refused => "The service didn't accept the key.", + CloudError::Limited => "The service is busy or the account has no credit left. Try again later.", + CloudError::Network => "The service couldn't be reached.", + CloudError::Store => "Windows Credential Manager couldn't be used.", + CloudError::Answer => "The service sent an answer OpenNote couldn't read.", + CloudError::Canceled => "Stopped.", + } + } +} + +/// Whether `key` could be a key: 20 to 400 printable ASCII characters with no spaces. +pub fn plausible_key(key: &str) -> bool { + (20..=400).contains(&key.len()) && key.bytes().all(|b| b.is_ascii_graphic()) +} + +/// A feature's line in Settings and the Privacy panel. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct CloudStatus { + pub feature: &'static str, + pub provider: &'static str, + pub host: &'static str, + pub has_key: bool, + pub last_used_unix: Option, +} + +/// What the cloud calls need, so tests can swap the store and the network. +#[derive(Clone)] +pub struct Cloud { + device: PathBuf, + store: Arc, + http: Arc, + lock: Arc>, +} + +fn now_unix() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_secs()) +} + +impl Cloud { + pub fn new(device: &Path, store: Arc, http: Arc) -> Cloud { + Cloud { + device: device.to_path_buf(), + store, + http, + lock: Arc::default(), + } + } + + fn policy() -> HostPolicy { + HostPolicy::new([HOST]) + } + + fn last_used(&self) -> BTreeMap { + fs::read_to_string(self.device.join(LAST_USED_FILE)) + .ok() + .and_then(|text| serde_json::from_str(&text).ok()) + .unwrap_or_default() + } + + fn mark_used(&self, feature: CloudFeature) { + let _held = self.lock.lock().unwrap_or_else(PoisonError::into_inner); + let mut used = self.last_used(); + used.insert(feature.id().to_owned(), now_unix()); + if let Ok(text) = serde_json::to_vec(&used) { + let _ = fs::create_dir_all(&self.device); + let _ = write_atomic(&self.device.join(LAST_USED_FILE), &text); + } + } + + pub fn status(&self) -> Vec { + let used = self.last_used(); + CloudFeature::ALL + .into_iter() + .map(|feature| CloudStatus { + feature: feature.id(), + provider: PROVIDER, + host: HOST, + has_key: matches!(self.store.get(&feature.target()), Ok(Some(secret)) if !secret.is_empty()), + last_used_unix: used.get(feature.id()).copied(), + }) + .collect() + } + + /// Saves the key for a feature, replacing any key it had. + pub fn set_key(&self, feature: CloudFeature, key: &str) -> Result<(), CloudError> { + let key = key.trim(); + if !plausible_key(key) { + return Err(CloudError::BadKey); + } + self.store + .put(&feature.target(), &Secret::new(key)) + .map_err(|_| CloudError::Store) + } + + pub fn forget(&self, feature: CloudFeature) -> Result<(), CloudError> { + self.store.delete(&feature.target()).map_err(|_| CloudError::Store) + } + + fn key(&self, feature: CloudFeature) -> Result { + match self.store.get(&feature.target()) { + Ok(Some(secret)) if !secret.is_empty() => Ok(secret), + Ok(_) => Err(CloudError::NoKey), + Err(_) => Err(CloudError::Store), + } + } + + fn send(&self, feature: CloudFeature, request: HttpRequest) -> Result { + let response = self + .http + .send(&Cloud::policy(), &request, MAX_ANSWER_BYTES) + .map_err(|error| match error { + HttpError::ForeignHost | HttpError::Network | HttpError::TooLarge => CloudError::Network, + })?; + self.mark_used(feature); + match response.status { + 200..=299 => response.json().ok_or(CloudError::Answer), + 401 | 403 => Err(CloudError::Refused), + 429 | 402 => Err(CloudError::Limited), + _ => Err(CloudError::Network), + } + } + + fn check_allowed(offline: bool, safe_mode: bool) -> Result<(), CloudError> { + if safe_mode { + Err(CloudError::SafeMode) + } else if offline { + Err(CloudError::Offline) + } else { + Ok(()) + } + } + + /// A short summary of `text` from the service. + pub fn summarize(&self, text: &str, offline: bool, safe_mode: bool) -> Result { + Cloud::check_allowed(offline, safe_mode)?; + let key = self.key(CloudFeature::Summaries)?; + let text: String = text.chars().take(MAX_SUMMARY_CHARS).collect(); + let body = json!({ + "model": "gpt-4o-mini", + "temperature": 0.2, + "messages": [ + { "role": "system", "content": SUMMARY_PROMPT }, + { "role": "user", "content": text }, + ], + }); + let request = HttpRequest::new(Method::Post, format!("{BASE}/chat/completions")) + .header("Authorization", format!("Bearer {}", key.expose())); + let request = HttpRequest { + follow_redirects: true, + body: Some(Body::Bytes { + content_type: "application/json".to_owned(), + data: body.to_string().into_bytes(), + }), + ..request + }; + let answer = self.send(CloudFeature::Summaries, request)?; + answer["choices"][0]["message"]["content"] + .as_str() + .map(|summary| summary.trim().to_owned()) + .filter(|summary| !summary.is_empty()) + .ok_or(CloudError::Answer) + } + + /// Transcribes 16 kHz mono audio read from `read`, piece by piece. `progress` hears the fraction done after each + /// piece, and `canceled`, `offline` and `safe_mode` are asked before each one, so turning either mode on stops + /// the pieces still to go. + #[allow(clippy::too_many_arguments)] + pub fn transcribe( + &self, + read: &mut dyn FnMut(&mut [f32]) -> Result, + total_samples: Option, + language: Option<&str>, + prompt: &str, + offline: &dyn Fn() -> bool, + safe_mode: &dyn Fn() -> bool, + progress: &dyn Fn(f32), + canceled: &dyn Fn() -> bool, + ) -> Result, CloudError> { + Cloud::check_allowed(offline(), safe_mode())?; + let key = self.key(CloudFeature::Transcription)?; + let piece_len = PIECE_SECONDS * 16_000; + let mut piece = vec![0.0_f32; piece_len]; + let mut lines = Vec::new(); + let mut offset_ms: u64 = 0; + let mut done: u64 = 0; + loop { + if canceled() { + return Err(CloudError::Canceled); + } + Cloud::check_allowed(offline(), safe_mode())?; + let mut filled = 0; + while filled < piece_len { + let count = read(&mut piece[filled..]).map_err(|_| CloudError::Answer)?; + if count == 0 { + break; + } + filled += count; + } + if filled == 0 { + break; + } + let request = transcription_request(&key, &piece[..filled], language, prompt); + let answer = self.send(CloudFeature::Transcription, request)?; + lines.extend(lines_of(&answer, offset_ms)?); + offset_ms += filled as u64 * 1000 / 16_000; + done += filled as u64; + if let Some(total) = total_samples.filter(|&t| t > 0) { + progress((done as f32 / total as f32).min(1.0)); + } + if filled < piece_len { + break; + } + } + Ok(lines) + } +} + +/// 16-bit PCM WAV at 16 kHz, mono. +pub fn wav_16k(samples: &[f32]) -> Vec { + let data_len = (samples.len() * 2) as u32; + let mut out = Vec::with_capacity(44 + samples.len() * 2); + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&(36 + data_len).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16_u32.to_le_bytes()); + out.extend_from_slice(&1_u16.to_le_bytes()); + out.extend_from_slice(&1_u16.to_le_bytes()); + out.extend_from_slice(&16_000_u32.to_le_bytes()); + out.extend_from_slice(&32_000_u32.to_le_bytes()); + out.extend_from_slice(&2_u16.to_le_bytes()); + out.extend_from_slice(&16_u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&data_len.to_le_bytes()); + for sample in samples { + let value = (sample.clamp(-1.0, 1.0) * 32_767.0).round() as i16; + out.extend_from_slice(&value.to_le_bytes()); + } + out +} + +fn transcription_request(key: &Secret, samples: &[f32], language: Option<&str>, prompt: &str) -> HttpRequest { + let boundary = format!("opennote-{:x}", now_unix() ^ samples.len() as u64); + let mut data = Vec::new(); + let mut field = |name: &str, value: &str| { + data.extend_from_slice( + format!("--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n").as_bytes(), + ); + }; + field("model", "whisper-1"); + field("response_format", "verbose_json"); + if let Some(language) = language { + // The service takes the language part of the tag only. + field("language", language.split('-').next().unwrap_or(language)); + } + if !prompt.is_empty() { + field("prompt", prompt); + } + data.extend_from_slice( + format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\n" + ) + .as_bytes(), + ); + data.extend_from_slice(&wav_16k(samples)); + data.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes()); + HttpRequest { + follow_redirects: true, + body: Some(Body::Bytes { + content_type: format!("multipart/form-data; boundary={boundary}"), + data, + }), + ..HttpRequest::new(Method::Post, format!("{BASE}/audio/transcriptions")) + .header("Authorization", format!("Bearer {}", key.expose())) + } +} + +fn lines_of(answer: &Value, offset_ms: u64) -> Result, CloudError> { + let ms = |seconds: &Value| seconds.as_f64().map(|s| (s.max(0.0) * 1000.0).round() as u64); + if let Some(segments) = answer["segments"].as_array() { + return Ok(segments + .iter() + .filter_map(|segment| { + let text = segment["text"].as_str()?.trim().to_owned(); + (!text.is_empty()).then(|| TranscriptLine { + start_ms: offset_ms + ms(&segment["start"]).unwrap_or(0), + end_ms: offset_ms + ms(&segment["end"]).unwrap_or(0), + text, + speaker: None, + }) + }) + .collect()); + } + let text = answer["text"].as_str().ok_or(CloudError::Answer)?.trim(); + Ok(if text.is_empty() { + Vec::new() + } else { + vec![TranscriptLine { + start_ms: offset_ms, + end_ms: offset_ms, + text: text.to_owned(), + speaker: None, + }] + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::connectors::{HttpResponse, MemoryStore}; + + /// One request as the stand-in saw it: the address, the headers, and the size of the body. + type Seen = (String, Vec<(String, String)>, usize); + + /// Answers every request with one reply, and keeps what was sent. + struct Fake { + reply: (u16, Value), + sent: Mutex>, + } + + impl Http for Fake { + fn send(&self, policy: &HostPolicy, request: &HttpRequest, _max: usize) -> Result { + if !policy.allows(&request.url) { + return Err(HttpError::ForeignHost); + } + let len = match &request.body { + Some(Body::Bytes { data, .. }) => data.len(), + _ => 0, + }; + self.sent + .lock() + .unwrap() + .push((request.url.clone(), request.headers.clone(), len)); + Ok(HttpResponse { + status: self.reply.0, + content_type: Some("application/json".to_owned()), + body: self.reply.1.to_string().into_bytes(), + location: None, + }) + } + } + + fn key() -> String { + // Built from pieces so the source holds nothing shaped like a key. + ["test", "only", "0123456789", "abcdef"].join("-") + } + + fn cloud(dir: &Path, status: u16, reply: Value) -> (Cloud, Arc) { + let fake = Arc::new(Fake { + reply: (status, reply), + sent: Mutex::default(), + }); + let http: Arc = fake.clone(); + (Cloud::new(dir, Arc::new(MemoryStore::default()), http), fake) + } + + #[test] + fn a_key_is_kept_in_the_store_and_only_its_presence_is_reported() { + let dir = tempfile::tempdir().unwrap(); + let (cloud, _) = cloud(dir.path(), 200, json!({})); + assert!(cloud.status().iter().all(|status| !status.has_key)); + assert_eq!(cloud.set_key(CloudFeature::Summaries, "short"), Err(CloudError::BadKey)); + assert_eq!( + cloud.set_key(CloudFeature::Summaries, "has a space in it, 12345678"), + Err(CloudError::BadKey) + ); + cloud + .set_key(CloudFeature::Summaries, &format!(" {} ", key())) + .unwrap(); + let status = cloud.status(); + let json = serde_json::to_string(&status).unwrap(); + assert!(!json.contains(&key()), "the status never carries the key"); + assert!(status.iter().any(|s| s.feature == "summaries" && s.has_key)); + assert!(status.iter().any(|s| s.feature == "transcription" && !s.has_key)); + cloud.forget(CloudFeature::Summaries).unwrap(); + assert!(cloud.status().iter().all(|status| !status.has_key)); + // Nothing about the key lands in the device folder. + for entry in fs::read_dir(dir.path()).unwrap() { + let text = fs::read_to_string(entry.unwrap().path()).unwrap_or_default(); + assert!(!text.contains(&key())); + } + } + + #[test] + fn nothing_is_sent_offline_in_safe_mode_or_without_a_key() { + let dir = tempfile::tempdir().unwrap(); + let (cloud, fake) = cloud(dir.path(), 200, json!({})); + assert_eq!(cloud.summarize("notes", false, false), Err(CloudError::NoKey)); + cloud.set_key(CloudFeature::Summaries, &key()).unwrap(); + assert_eq!(cloud.summarize("notes", true, false), Err(CloudError::Offline)); + assert_eq!(cloud.summarize("notes", false, true), Err(CloudError::SafeMode)); + assert!(fake.sent.lock().unwrap().is_empty()); + } + + #[test] + fn a_summary_goes_to_the_pinned_host_with_the_key_and_marks_when_it_ran() { + let dir = tempfile::tempdir().unwrap(); + let reply = json!({ "choices": [{ "message": { "content": " The cell makes energy. " } }] }); + let (cloud, fake) = cloud(dir.path(), 200, reply); + cloud.set_key(CloudFeature::Summaries, &key()).unwrap(); + assert_eq!( + cloud.summarize("notes", false, false).unwrap(), + "The cell makes energy." + ); + let sent = fake.sent.lock().unwrap(); + assert_eq!(sent[0].0, "https://api.openai.com/v1/chat/completions"); + assert!(sent[0] + .1 + .iter() + .any(|(name, value)| name == "Authorization" && value.ends_with(&key()))); + drop(sent); + let used = cloud.status(); + assert!(used + .iter() + .any(|s| s.feature == "summaries" && s.last_used_unix.is_some())); + assert!(used + .iter() + .any(|s| s.feature == "transcription" && s.last_used_unix.is_none())); + } + + #[test] + fn a_refused_key_and_a_busy_service_say_so() { + let dir = tempfile::tempdir().unwrap(); + let (refused, _) = cloud(dir.path(), 401, json!({ "error": {} })); + refused.set_key(CloudFeature::Summaries, &key()).unwrap(); + assert_eq!(refused.summarize("notes", false, false), Err(CloudError::Refused)); + let (busy, _) = cloud(dir.path(), 429, json!({})); + busy.set_key(CloudFeature::Summaries, &key()).unwrap(); + assert_eq!(busy.summarize("notes", false, false), Err(CloudError::Limited)); + } + + #[test] + fn audio_goes_up_in_pieces_and_the_lines_keep_their_place() { + let dir = tempfile::tempdir().unwrap(); + let reply = json!({ "text": "x", "segments": [{ "start": 1.0, "end": 2.5, "text": " Hello. " }] }); + let (cloud, fake) = cloud(dir.path(), 200, reply); + cloud.set_key(CloudFeature::Transcription, &key()).unwrap(); + let total = 16_000 * (PIECE_SECONDS + 30); + let mut left = total; + let mut read = |out: &mut [f32]| -> Result { + let count = out.len().min(left); + out[..count].fill(0.1); + left -= count; + Ok(count) + }; + let fractions = Mutex::new(Vec::new()); + let lines = cloud + .transcribe( + &mut read, + Some(total as u64), + Some("en-US"), + "ATP", + &|| false, + &|| false, + &|f| fractions.lock().unwrap().push(f), + &|| false, + ) + .unwrap(); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].start_ms, 1000); + assert_eq!(lines[1].start_ms, PIECE_SECONDS as u64 * 1000 + 1000); + assert_eq!(lines[1].text, "Hello."); + assert_eq!(fake.sent.lock().unwrap().len(), 2); + assert_eq!(fractions.lock().unwrap().last().copied(), Some(1.0)); + let canceled = cloud.transcribe(&mut |_| Ok(0), None, None, "", &|| false, &|| false, &|_| {}, &|| true); + assert_eq!(canceled, Err(CloudError::Canceled)); + } + + #[test] + fn turning_on_safe_mode_or_offline_part_way_stops_the_pieces_still_to_go() { + for offline_mode in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let reply = json!({ "text": "x", "segments": [] }); + let (cloud, fake) = cloud(dir.path(), 200, reply); + cloud.set_key(CloudFeature::Transcription, &key()).unwrap(); + let mut left = 16_000 * PIECE_SECONDS * 3; + let mut read = |out: &mut [f32]| -> Result { + let count = out.len().min(left); + left -= count; + Ok(count) + }; + // The mode turns on once the first piece has gone. + let on = || fake.sent.lock().unwrap().len() >= 1; + let result = cloud.transcribe( + &mut read, + None, + None, + "", + &|| offline_mode && on(), + &|| !offline_mode && on(), + &|_| {}, + &|| false, + ); + let expected = if offline_mode { CloudError::Offline } else { CloudError::SafeMode }; + assert_eq!(result, Err(expected)); + assert_eq!(fake.sent.lock().unwrap().len(), 1); + } + } + + #[test] + fn the_wav_header_matches_the_samples() { + let wav = wav_16k(&[0.0, 1.0, -1.0]); + assert_eq!(&wav[..4], b"RIFF"); + assert_eq!(wav.len(), 44 + 6); + assert_eq!(i16::from_le_bytes([wav[46], wav[47]]), 32_767); + } +} diff --git a/app/src-tauri/src/intel/ext.rs b/app/src-tauri/src/intel/ext.rs new file mode 100644 index 00000000..11bf0727 --- /dev/null +++ b/app/src-tauri/src/intel/ext.rs @@ -0,0 +1,532 @@ +//! The one command behind the quality-of-life parts of on-device intelligence (Phase 12): model downloads and a +//! small store for the lists and indexes the interface keeps on this device. Each call names a method, so a new +//! method needs no new command, capability, or permission. Nothing here reads a note, and the store holds only what +//! the interface hands it. + +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +use serde::Deserialize; +use serde_json::{json, Value}; + +use super::cloud::{Cloud, CloudError, CloudFeature}; +use super::llm::{Llm, LlmError}; +use super::models::{self, DownloadError, Models}; +use super::native::{self, Native, NativeError}; +use crate::{ + ipc::{codes, IpcError}, + settings::file::write_atomic, +}; + +/// The most text one stored item may hold. +const MAX_STORED_BYTES: usize = 16 * 1024 * 1024; + +/// A call from the interface: the method and its arguments. +#[derive(Debug, Clone, Deserialize)] +pub struct ExtRequest { + pub method: String, + #[serde(default)] + pub params: Value, +} + +/// The state behind the calls. +#[derive(Clone)] +pub struct Ext { + store: PathBuf, + models: Models, + cloud: Cloud, + llm: Llm, + native: std::sync::Arc, +} + +fn invalid(message: &str, field: &str) -> IpcError { + IpcError { + code: codes::INVALID.to_owned(), + message: message.to_owned(), + field: Some(field.to_owned()), + } +} + +fn io_error(error: &io::Error) -> IpcError { + IpcError::new(codes::IO, error.to_string()) +} + +fn text<'a>(params: &'a Value, key: &str) -> Result<&'a str, IpcError> { + params + .get(key) + .and_then(Value::as_str) + .ok_or_else(|| invalid("A text argument is missing.", key)) +} + +/// A stored item's file name must be a plain name, so a call cannot reach outside the store. +fn store_name(name: &str) -> Result<&str, IpcError> { + let plain = !name.is_empty() + && name.len() <= 80 + && !name.starts_with('.') + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')); + if plain { + Ok(name) + } else { + Err(invalid( + "A stored name has letters, digits, dots, dashes, and underscores only.", + "name", + )) + } +} + +impl Ext { + /// `device` is this device's folder. The store and the models live beneath it. + pub fn new(device: &Path) -> Ext { + Ext::with_models(device, Models::new(device.join("models"))) + } + + pub fn with_models(device: &Path, models: Models) -> Ext { + let cloud = Cloud::new( + &device.join("intel"), + crate::connectors::platform_store(), + std::sync::Arc::new(crate::connectors::UreqHttp::default()), + ); + Ext::with_parts(device, models, cloud) + } + + pub fn with_parts(device: &Path, models: Models, cloud: Cloud) -> Ext { + Ext { + store: device.join("intel"), + models, + cloud, + llm: Llm::new(&device.join("engines")), + native: std::sync::Arc::new(Native::default()), + } + } + + /// The cloud calls, which the transcription command shares. + pub fn cloud(&self) -> &Cloud { + &self.cloud + } + + fn cloud_error(error: &CloudError) -> IpcError { + IpcError::new(error.code(), error.message()) + } + + fn cloud_feature(params: &Value) -> Result { + CloudFeature::parse(text(params, "feature")?) + .ok_or_else(|| invalid("That feature can't use a cloud key.", "feature")) + } + + /// Replaces the language model program, so a test can use a stand-in. + #[cfg(test)] + pub fn with_llm(mut self, llm: Llm) -> Ext { + self.llm = llm; + self + } + + fn llm_error(error: &LlmError) -> IpcError { + IpcError::new(error.code(), error.message()) + } + + fn native_error(error: &NativeError) -> IpcError { + IpcError::new(error.code(), error.message()) + } + + /// The language model the person picked, or the best one that is installed. + fn language_model(&self) -> Option { + let chosen = fs::read_to_string(self.store.join("language-model.txt")).unwrap_or_default(); + let found = [chosen.trim(), "language-balanced", "language-small"] + .into_iter() + .filter(|id| models::spec_of(id).is_some_and(|spec| spec.kind == "language")) + .find_map(|id| self.models.installed_path(id)); + found + } + + fn strings(params: &Value, key: &str) -> Result, IpcError> { + params + .get(key) + .and_then(Value::as_array) + .ok_or_else(|| invalid("A list of texts is missing.", key))? + .iter() + .map(|one| { + one.as_str() + .map(str::to_owned) + .ok_or_else(|| invalid("A text is not text.", key)) + }) + .collect() + } + + /// The folder of an installed pack: its files are beside its first one. + fn pack_dir(&self, id: &str, missing: &str) -> Result { + let main = self + .models + .installed_path(id) + .ok_or_else(|| IpcError::new("model", missing))?; + Ok(main + .parent() + .map_or_else(|| self.models.dir().to_path_buf(), Path::to_path_buf)) + } + + fn file(&self, name: &str) -> Result { + Ok(self.store.join(store_name(name)?)) + } + + fn download_error(error: &DownloadError) -> IpcError { + let code = match error { + DownloadError::Disk(_) => codes::IO, + other => other.code(), + }; + IpcError::new(code, format!("{error:?}")) + } + + /// Runs one method. `offline` and `safe_mode` come from the app, so tests can set them. + pub fn call(&self, request: &ExtRequest, offline: bool, safe_mode: bool) -> Result { + let (method, params) = (request.method.as_str(), &request.params); + match method.split('.').next().unwrap_or_default() { + "store" | "vocabulary" => self.call_device(method, params), + "models" => self.call_models(method, params, offline, safe_mode), + "engines" | "llm" | "translate" | "embed" => self.call_engines(method, params, safe_mode), + "cloud" => self.call_cloud(method, params, offline, safe_mode), + _ => Err(invalid(&format!("There is no method called {method}."), "method")), + } + } + + /// The small store and the vocabulary. + fn call_device(&self, method: &str, params: &Value) -> Result { + match method { + "store.get" => match fs::read_to_string(self.file(text(params, "name")?)?) { + Ok(content) => Ok(json!(content)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(Value::Null), + Err(error) => Err(io_error(&error)), + }, + "store.put" => { + let content = text(params, "text")?; + if content.len() > MAX_STORED_BYTES { + return Err(invalid("That is too much to store.", "text")); + } + write_atomic(&self.file(text(params, "name")?)?, content.as_bytes()) + .map_err(|error| io_error(&error))?; + Ok(Value::Null) + } + "store.remove" => match fs::remove_file(self.file(text(params, "name")?)?) { + Ok(()) => Ok(Value::Null), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(Value::Null), + Err(error) => Err(io_error(&error)), + }, + "store.list" => { + let prefix = text(params, "prefix")?; + if !prefix.is_empty() { + store_name(prefix)?; + } + let entries = match fs::read_dir(&self.store) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(json!([])), + Err(error) => return Err(io_error(&error)), + }; + let mut names: Vec = entries + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_file())) + .filter_map(|entry| entry.file_name().into_string().ok()) + .filter(|name| name.starts_with(prefix) && store_name(name).is_ok()) + .collect(); + names.sort(); + Ok(json!(names)) + } + "vocabulary.correct" => { + let request: opennote_intel::wire::VocabularyCorrectRequest = + serde_json::from_value(params.clone()).map_err(|error| invalid(&error.to_string(), "params"))?; + serde_json::to_value(request.correct()) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string())) + } + other => Err(invalid(&format!("There is no method called {other}."), "method")), + } + } + + /// Model downloads. + fn call_models(&self, method: &str, params: &Value, offline: bool, safe_mode: bool) -> Result { + match method { + "models.list" => serde_json::to_value(self.models.list()) + .map(|mut list| { + list["offline"] = json!(offline); + list + }) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string())), + "models.start" => { + let blocked = if safe_mode { + Some(DownloadError::SafeMode) + } else if offline { + Some(DownloadError::Offline) + } else { + None + }; + self.models + .start(text(params, "id")?, blocked) + .map(|()| Value::Null) + .map_err(|error| Self::download_error(&error)) + } + "models.cancel" => { + self.models.cancel(text(params, "id")?); + Ok(Value::Null) + } + "models.remove" => { + // Let go of the loaded models first, so their files can be deleted. + self.native.unload(); + self.llm.stop(); + self.models + .remove(text(params, "id")?) + .map(|()| Value::Null) + .map_err(|error| Self::download_error(&error)) + } + other => Err(invalid(&format!("There is no method called {other}."), "method")), + } + } + + /// The language model, translation, and meaning vectors. + fn call_engines(&self, method: &str, params: &Value, safe_mode: bool) -> Result { + match method { + "engines.info" => { + let mut info = native::capabilities(); + info["llmHost"] = json!(self.llm.has_host()); + info["languageModel"] = json!(self.language_model().is_some()); + Ok(info) + } + "llm.generate" => { + if safe_mode { + return Err(IpcError::new(DownloadError::SafeMode.code(), "Safe mode is on.")); + } + let model = self + .language_model() + .ok_or_else(|| IpcError::new("model", "No language model is downloaded."))?; + let number = |key: &str, default: f64| params.get(key).and_then(Value::as_f64).unwrap_or(default); + self.llm + .generate( + &model, + text(params, "system")?, + text(params, "prompt")?, + number("maxTokens", 400.0).clamp(1.0, 1024.0) as u32, + number("temperature", 0.2).clamp(0.0, 1.0) as f32, + ) + .map(|answer| json!({ "text": answer })) + .map_err(|error| Self::llm_error(&error)) + } + "llm.stop" => { + self.llm.stop(); + Ok(Value::Null) + } + "translate.run" => { + if safe_mode { + return Err(IpcError::new(DownloadError::SafeMode.code(), "Safe mode is on.")); + } + let id = format!("translate-{}", text(params, "pack")?); + if !models::spec_of(&id).is_some_and(|spec| spec.kind == "translate") { + return Err(invalid("That language pack is not offered.", "pack")); + } + let lines = Self::strings(params, "lines")?; + let dir = self.pack_dir(&id, "That language pack is not downloaded.")?; + self.native + .translate(&id, &dir, &lines) + .map(|lines| json!({ "lines": lines })) + .map_err(|error| Self::native_error(&error)) + } + "embed.run" => { + if safe_mode { + return Err(IpcError::new(DownloadError::SafeMode.code(), "Safe mode is on.")); + } + let texts = Self::strings(params, "texts")?; + let dir = self.pack_dir("meaning-minilm", "The meaning model is not downloaded.")?; + self.native + .embed(&dir, &texts) + .map(|vectors| json!({ "vectors": vectors })) + .map_err(|error| Self::native_error(&error)) + } + other => Err(invalid(&format!("There is no method called {other}."), "method")), + } + } + + /// The person's own cloud key. + fn call_cloud(&self, method: &str, params: &Value, offline: bool, safe_mode: bool) -> Result { + match method { + "cloud.status" => serde_json::to_value(self.cloud.status()) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string())), + "cloud.setKey" => { + let feature = Ext::cloud_feature(params)?; + self.cloud + .set_key(feature, text(params, "key")?) + .map(|()| Value::Null) + .map_err(|error| Ext::cloud_error(&error)) + } + "cloud.forget" => self + .cloud + .forget(Ext::cloud_feature(params)?) + .map(|()| Value::Null) + .map_err(|error| Ext::cloud_error(&error)), + "cloud.summarize" => self + .cloud + .summarize(text(params, "text")?, offline, safe_mode) + .map(|summary| json!({ "summary": summary })) + .map_err(|error| Ext::cloud_error(&error)), + other => Err(invalid(&format!("There is no method called {other}."), "method")), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ext() -> (tempfile::TempDir, Ext) { + let dir = tempfile::tempdir().unwrap(); + let ext = Ext::new(dir.path()); + (dir, ext) + } + + fn call(ext: &Ext, method: &str, params: Value) -> Result { + ext.call( + &ExtRequest { + method: method.to_owned(), + params, + }, + false, + false, + ) + } + + #[test] + fn the_store_keeps_text_by_name() { + let (_dir, ext) = ext(); + assert_eq!( + call(&ext, "store.get", json!({ "name": "words.txt" })).unwrap(), + Value::Null + ); + call( + &ext, + "store.put", + json!({ "name": "words.txt", "text": "Ångström\nOpenNote" }), + ) + .unwrap(); + assert_eq!( + call(&ext, "store.get", json!({ "name": "words.txt" })).unwrap(), + json!("Ångström\nOpenNote") + ); + call(&ext, "store.remove", json!({ "name": "words.txt" })).unwrap(); + assert_eq!( + call(&ext, "store.get", json!({ "name": "words.txt" })).unwrap(), + Value::Null + ); + } + + #[test] + fn the_store_lists_the_names_that_start_with_a_prefix() { + let (_dir, ext) = ext(); + assert_eq!( + call(&ext, "store.list", json!({ "prefix": "study.deck." })).unwrap(), + json!([]) + ); + for name in ["study.deck.b.json", "study.deck.a.json", "study.states.a.json"] { + call(&ext, "store.put", json!({ "name": name, "text": "{}" })).unwrap(); + } + assert_eq!( + call(&ext, "store.list", json!({ "prefix": "study.deck." })).unwrap(), + json!(["study.deck.a.json", "study.deck.b.json"]) + ); + let error = call(&ext, "store.list", json!({ "prefix": "../" })).unwrap_err(); + assert_eq!(error.code, codes::INVALID); + } + + #[test] + fn a_name_cannot_leave_the_store() { + let (_dir, ext) = ext(); + for name in ["../x", "a/b", "a\\b", ".hidden", "", "x y"] { + let error = call(&ext, "store.get", json!({ "name": name })).unwrap_err(); + assert_eq!(error.code, codes::INVALID, "{name}"); + } + } + + #[test] + fn a_vocabulary_fixes_a_mishearing_and_reports_it() { + let (_dir, ext) = ext(); + let fixed = call( + &ext, + "vocabulary.correct", + json!({ "vocabulary": "Calvin cycle | calvin psyche", "text": "The calvin psyche makes sugar." }), + ) + .unwrap(); + assert_eq!(fixed["text"], "The Calvin cycle makes sugar."); + assert_eq!(fixed["changes"][0]["from"], "calvin psyche"); + } + + fn safe(ext: &Ext, method: &str, params: Value) -> Result { + let request = ExtRequest { + method: method.to_owned(), + params, + }; + ext.call(&request, false, true) + } + + #[test] + fn the_engines_say_what_this_build_and_install_carry() { + let (_dir, ext) = ext(); + let info = call(&ext, "engines.info", Value::Null).unwrap(); + assert_eq!(info["languageModel"], false); + assert!(info["onnx"].is_boolean() && info["whisperCpp"].is_boolean() && info["llmHost"].is_boolean()); + } + + #[test] + fn the_language_model_and_the_packs_need_a_download_and_refuse_in_safe_mode() { + let (_dir, ext) = ext(); + let write = json!({ "system": "s", "prompt": "p" }); + assert_eq!(call(&ext, "llm.generate", write.clone()).unwrap_err().code, "model"); + assert_eq!(safe(&ext, "llm.generate", write).unwrap_err().code, "safeMode"); + let translate = json!({ "pack": "en-es", "lines": ["hello"] }); + assert_eq!( + call(&ext, "translate.run", translate.clone()).unwrap_err().code, + "model" + ); + assert_eq!(safe(&ext, "translate.run", translate).unwrap_err().code, "safeMode"); + let unknown = json!({ "pack": "xx-yy", "lines": [] }); + assert_eq!(call(&ext, "translate.run", unknown).unwrap_err().code, codes::INVALID); + let embed = json!({ "texts": ["hello"] }); + assert_eq!(call(&ext, "embed.run", embed.clone()).unwrap_err().code, "model"); + assert_eq!(safe(&ext, "embed.run", embed).unwrap_err().code, "safeMode"); + assert_eq!( + call(&ext, "translate.run", json!({ "pack": "en-es", "lines": [1] })) + .unwrap_err() + .code, + codes::INVALID + ); + } + + #[test] + fn an_installed_language_model_is_found_and_asked_through_the_program() { + let (dir, ext) = ext(); + let spec = models::spec_of("language-small").unwrap(); + let path = dir.path().join("models").join(spec.main().file); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::File::create(&path).unwrap().set_len(spec.size()).unwrap(); + assert_eq!(call(&ext, "engines.info", Value::Null).unwrap()["languageModel"], true); + // No program is in a test install, so the model is found and the missing program is what fails. + let ext = ext.with_llm(Llm::with_host(None)); + let error = call(&ext, "llm.generate", json!({ "system": "s", "prompt": "p" })).unwrap_err(); + assert_eq!(error.code, "unavailable"); + } + + #[test] + fn an_unknown_method_is_refused() { + let (_dir, ext) = ext(); + assert_eq!(call(&ext, "nope", Value::Null).unwrap_err().code, codes::INVALID); + } + + #[test] + fn downloads_refuse_to_start_offline_or_in_safe_mode() { + let (_dir, ext) = ext(); + let request = ExtRequest { + method: "models.start".to_owned(), + params: json!({ "id": "speech-base-en" }), + }; + assert_eq!(ext.call(&request, true, false).unwrap_err().code, "offline"); + assert_eq!(ext.call(&request, false, true).unwrap_err().code, "safeMode"); + let listed = call(&ext, "models.list", Value::Null).unwrap(); + assert_eq!(listed["models"][0]["state"], "notInstalled"); + assert_eq!(listed["offline"], false); + } +} diff --git a/app/src-tauri/src/intel/llm.rs b/app/src-tauri/src/intel/llm.rs new file mode 100644 index 00000000..9dc2c168 --- /dev/null +++ b/app/src-tauri/src/intel/llm.rs @@ -0,0 +1,475 @@ +//! The language model program, from the app's side (A3-23). Ask and the writing tools send a prompt and get text +//! back. The program (`opennote-llm`, in `crates/llm-host`) runs llama.cpp in a process of its own and talks over +//! its standard input and output. It starts on the first request, ends after a quiet spell so its memory goes back to +//! the computer, and is started only with a language model the person downloaded. Nothing it is sent is written to +//! disk or logged here. + +use std::{ + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, + process::{Child, ChildStdin, Command, Stdio}, + sync::{ + atomic::{AtomicU64, Ordering}, + mpsc::{self, Receiver, RecvTimeoutError}, + Arc, Mutex, MutexGuard, PoisonError, + }, + thread, + time::{Duration, Instant}, +}; + +use opennote_intel_engines::protocol::{self, FailureCode, Request, Response}; + +/// How long the program waits for its first word that it is ready. +const START_TIMEOUT: Duration = Duration::from_secs(30); +/// The longest one answer may take. +const ANSWER_TIMEOUT: Duration = Duration::from_secs(180); +/// How long the program stays loaded when nothing is asked of it. +const IDLE_LIMIT: Duration = Duration::from_secs(120); + +/// Why a request to the program failed. The interface picks its own words from the code. +#[derive(Debug, PartialEq, Eq)] +pub enum LlmError { + /// The program is not in this install. + NoHost, + /// The program would not start or stopped. + Host(String), + /// The model is missing or cannot be read. + Model(String), + /// The request is over a limit. + Invalid(String), + /// The program took too long. + Timeout, +} + +impl LlmError { + pub fn code(&self) -> &'static str { + match self { + LlmError::NoHost => "unavailable", + LlmError::Host(_) | LlmError::Timeout => "engine", + LlmError::Model(_) => "model", + LlmError::Invalid(_) => "invalid", + } + } + + pub fn message(&self) -> String { + match self { + LlmError::NoHost => "This install has no language model program.".to_owned(), + LlmError::Host(message) | LlmError::Model(message) | LlmError::Invalid(message) => message.clone(), + LlmError::Timeout => "The language model took too long.".to_owned(), + } + } +} + +struct Running { + child: Child, + stdin: ChildStdin, + lines: Receiver, + last_used: Instant, +} + +impl Drop for Running { + fn drop(&mut self) { + let _ = writeln!( + self.stdin, + "{}", + serde_json::to_string(&Request::Quit).unwrap_or_default() + ); + // A program that is mid-answer would take its time. The computer has the memory back sooner this way. + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +struct Inner { + host: Option, + running: Mutex>, + next_id: AtomicU64, +} + +/// The language model program, started when needed. +#[derive(Clone)] +pub struct Llm(Arc); + +/// The program's file name. +const HOST_NAME: &str = if cfg!(windows) { + "opennote-llm.exe" +} else { + "opennote-llm" +}; + +/// The program, inside the app's own file. A release is one signed exe and nothing else, so the program travels in +/// it and is written to this device's folder the first time it is needed. +#[cfg(feature = "embed-llm-host")] +static EMBEDDED_HOST: &[u8] = include_bytes!(env!("OPENNOTE_LLM_HOST_BIN")); +#[cfg(not(feature = "embed-llm-host"))] +static EMBEDDED_HOST: &[u8] = &[]; + +/// Writes `bytes` as a program in `dir`, under a name that holds their SHA-256, so a different build gets a file of +/// its own and a file that is there is never run unless it is exactly these bytes. +pub fn extract_host(bytes: &[u8], dir: &Path) -> std::io::Result { + use sha2::{Digest, Sha256}; + let hash: String = Sha256::digest(bytes) + .iter() + .take(8) + .map(|b| format!("{b:02x}")) + .collect(); + let stem = HOST_NAME.trim_end_matches(".exe"); + let name = if cfg!(windows) { + format!("{stem}-{hash}.exe") + } else { + format!("{stem}-{hash}") + }; + let target = dir.join(name); + if std::fs::read(&target).is_ok_and(|have| have == bytes) { + return Ok(target); + } + std::fs::create_dir_all(dir)?; + let temporary = dir.join(format!(".{hash}.tmp")); + std::fs::write(&temporary, bytes)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&temporary, std::fs::Permissions::from_mode(0o755))?; + } + match std::fs::rename(&temporary, &target) { + Ok(()) => {} + // A running copy keeps the file locked on Windows. If it is the same program, it is fine. + Err(_) if std::fs::read(&target).is_ok_and(|have| have == bytes) => { + let _ = std::fs::remove_file(&temporary); + } + Err(error) => { + let _ = std::fs::remove_file(&temporary); + return Err(error); + } + } + Ok(target) +} + +/// Where the program is in an install: beside the app if a build left it there, and otherwise the copy in the app's +/// own file, written to `dir`. +pub fn default_host(dir: &Path) -> Option { + if let Some(beside) = std::env::current_exe() + .ok() + .and_then(|exe| exe.parent().map(|parent| parent.join(HOST_NAME))) + .filter(|path| path.is_file()) + { + return Some(beside); + } + if EMBEDDED_HOST.is_empty() { + return None; + } + extract_host(EMBEDDED_HOST, dir).ok() +} + +impl Llm { + /// The program, found when first needed. `dir` is where a copy inside the app is written. + pub fn new(dir: &Path) -> Llm { + Llm::with_host(default_host(dir)) + } + + pub fn with_host(host: Option) -> Llm { + Llm(Arc::new(Inner { + host, + running: Mutex::new(None), + next_id: AtomicU64::new(1), + })) + } + + /// Whether this install has the program. + pub fn has_host(&self) -> bool { + self.0.host.is_some() + } + + fn running(&self) -> MutexGuard<'_, Option> { + self.0.running.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Ends the program, if it is running. + pub fn stop(&self) { + *self.running() = None; + } + + fn start(host: &Path) -> Result { + let mut command = Command::new(host); + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + const CREATE_NO_WINDOW: u32 = 0x0800_0000; + command.creation_flags(CREATE_NO_WINDOW); + } + let mut child = command.spawn().map_err(|error| LlmError::Host(error.to_string()))?; + let (Some(stdin), Some(stdout)) = (child.stdin.take(), child.stdout.take()) else { + let _ = child.kill(); + return Err(LlmError::Host("The program has no pipes.".to_owned())); + }; + let (sender, lines) = mpsc::channel(); + thread::Builder::new() + .name("llm-reader".to_owned()) + .spawn(move || { + for line in BufReader::new(stdout).lines() { + let Ok(line) = line else { break }; + // A line that is not a message, such as noise from a library, is skipped. + if let Ok(response) = serde_json::from_str::(&line) { + if sender.send(response).is_err() { + break; + } + } + } + }) + .map_err(|error| LlmError::Host(error.to_string()))?; + let running = Running { + child, + stdin, + lines, + last_used: Instant::now(), + }; + match running.lines.recv_timeout(START_TIMEOUT) { + Ok(Response::Ready { .. }) => Ok(running), + Ok(Response::Failed { message, .. }) => Err(LlmError::Host(message)), + Ok(Response::Done { .. }) | Err(RecvTimeoutError::Disconnected) => { + Err(LlmError::Host("The program stopped at start.".to_owned())) + } + Err(RecvTimeoutError::Timeout) => Err(LlmError::Timeout), + } + } + + /// Asks the model in `model` to write an answer. One request runs at a time. + pub fn generate( + &self, + model: &Path, + system: &str, + prompt: &str, + max_tokens: u32, + temperature: f32, + ) -> Result { + let host = self.0.host.as_deref().ok_or(LlmError::NoHost)?; + let id = self.0.next_id.fetch_add(1, Ordering::Relaxed); + let request = Request::Generate { + id, + model: model.to_string_lossy().into_owned(), + system: system.to_owned(), + prompt: prompt.to_owned(), + max_tokens, + temperature, + }; + protocol::check(&request).map_err(LlmError::Invalid)?; + let mut slot = self.running(); + let result = Self::ask(&mut slot, host, &request, id); + match &result { + Ok(_) | Err(LlmError::Model(_) | LlmError::Invalid(_)) => { + if let Some(running) = slot.as_mut() { + running.last_used = Instant::now(); + } + drop(slot); + self.reap_later(); + } + // A program that failed or timed out may be stuck. The next request starts a fresh one. + Err(_) => *slot = None, + } + result + } + + fn ask(slot: &mut Option, host: &Path, request: &Request, id: u64) -> Result { + if slot.is_none() { + *slot = Some(Self::start(host)?); + } + let Some(running) = slot.as_mut() else { + return Err(LlmError::NoHost); + }; + // Answers left over from an earlier request that timed out are not this one's. + while running.lines.try_recv().is_ok() {} + let line = serde_json::to_string(request).map_err(|error| LlmError::Host(error.to_string()))?; + writeln!(running.stdin, "{line}") + .and_then(|()| running.stdin.flush()) + .map_err(|error| LlmError::Host(error.to_string()))?; + let deadline = Instant::now() + ANSWER_TIMEOUT; + loop { + let left = deadline.saturating_duration_since(Instant::now()); + match running.lines.recv_timeout(left) { + Ok(Response::Done { id: answered, text }) if answered == id => return Ok(text), + Ok(Response::Failed { + id: answered, + code, + message, + }) if answered == id => { + return Err(match code { + FailureCode::Model => LlmError::Model(message), + FailureCode::Invalid => LlmError::Invalid(message), + FailureCode::Engine => LlmError::Host(message), + }); + } + Ok(_) => {} + Err(RecvTimeoutError::Timeout) => return Err(LlmError::Timeout), + Err(RecvTimeoutError::Disconnected) => { + return Err(LlmError::Host("The program stopped.".to_owned())); + } + } + } + } + + /// Ends the program once it has been quiet for [`IDLE_LIMIT`]. + fn reap_later(&self) { + let this = self.clone(); + let _ = thread::Builder::new() + .name("llm-reaper".to_owned()) + .spawn(move || loop { + thread::sleep(IDLE_LIMIT / 4); + let mut slot = this.running(); + match slot.as_ref() { + None => return, + Some(running) if running.last_used.elapsed() >= IDLE_LIMIT => { + *slot = None; + return; + } + Some(_) => {} + } + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The test program starts itself again as the language model program when this variable is set. + const AS_HOST: &str = "OPENNOTE_TEST_AS_LLM_HOST"; + + /// Not a test of its own: when the variable is set, this process is the program, and it answers "echo: " and the + /// prompt, fails for a model file whose name has `missing`, and never answers for one with `hang`. + #[test] + fn the_stand_in_program() { + if std::env::var_os(AS_HOST).is_none() { + return; + } + let stdin = std::io::stdin(); + let mut out = std::io::stdout(); + let mut say = |response: &Response| { + let _ = writeln!(out, "{}", serde_json::to_string(response).unwrap()); + let _ = out.flush(); + }; + say(&Response::Ready { + runtime: "stand-in".to_owned(), + }); + for line in stdin.lock().lines() { + let Ok(Request::Generate { id, model, prompt, .. }) = serde_json::from_str(&line.unwrap()) else { + return; + }; + if model.contains("hang") { + std::thread::sleep(Duration::from_secs(3600)); + } else if model.contains("missing") { + say(&Response::Failed { + id, + code: FailureCode::Model, + message: "no file".to_owned(), + }); + } else { + say(&Response::Done { + id, + text: format!("echo: {prompt}"), + }); + } + } + } + + /// A stand-in program is a copy of this test program told to act as one. Libtest prints its own lines first, + /// which the reader skips because they are not messages. + fn stand_in() -> Llm { + let exe = std::env::current_exe().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let script = if cfg!(windows) { + let path = dir.path().join("host.cmd"); + std::fs::write( + &path, + format!( + "@echo off\r\nset {AS_HOST}=1\r\n\"{}\" the_stand_in_program --nocapture\r\n", + exe.display() + ), + ) + .unwrap(); + path + } else { + let path = dir.path().join("host.sh"); + std::fs::write( + &path, + format!( + "#!/bin/sh\n{AS_HOST}=1 exec \"{}\" the_stand_in_program --nocapture\n", + exe.display() + ), + ) + .unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + path + }; + std::mem::forget(dir); + Llm::with_host(Some(script)) + } + + #[test] + fn a_request_goes_out_and_the_answer_comes_back() { + let llm = stand_in(); + let model = Path::new("model.gguf"); + assert_eq!(llm.generate(model, "s", "one", 10, 0.0).as_deref(), Ok("echo: one")); + // The program is kept for the next request. + assert_eq!(llm.generate(model, "s", "two", 10, 0.0).as_deref(), Ok("echo: two")); + llm.stop(); + } + + #[test] + fn a_missing_model_is_reported_and_the_program_goes_on() { + let llm = stand_in(); + let error = llm.generate(Path::new("missing.gguf"), "s", "x", 10, 0.0).unwrap_err(); + assert_eq!(error.code(), "model"); + assert_eq!( + llm.generate(Path::new("m.gguf"), "s", "after", 10, 0.0).as_deref(), + Ok("echo: after") + ); + llm.stop(); + } + + #[test] + fn limits_are_checked_before_anything_starts() { + let llm = Llm::with_host(Some(PathBuf::from("does-not-exist"))); + let error = llm.generate(Path::new("m.bin"), "s", "x", 10, 0.0).unwrap_err(); + assert_eq!(error.code(), "invalid"); + let error = llm.generate(Path::new("m.gguf"), "s", "x", 10, 0.0).unwrap_err(); + assert_eq!( + error.code(), + "engine", + "a program that will not start is an engine failure" + ); + } + + #[test] + fn a_program_inside_the_app_is_written_once_under_its_hash_and_never_trusted_when_it_differs() { + let dir = tempfile::tempdir().unwrap(); + let first = extract_host(b"program one", dir.path()).unwrap(); + assert_eq!(std::fs::read(&first).unwrap(), b"program one"); + assert_eq!(extract_host(b"program one", dir.path()).unwrap(), first); + // A different build gets a file of its own, and a damaged file is written again. + let second = extract_host(b"program two", dir.path()).unwrap(); + assert_ne!(first, second); + std::fs::write(&first, b"tampered").unwrap(); + assert_eq!(extract_host(b"program one", dir.path()).unwrap(), first); + assert_eq!(std::fs::read(&first).unwrap(), b"program one"); + let leftovers = std::fs::read_dir(dir.path()).unwrap().count(); + assert_eq!(leftovers, 2, "no temporary file stays behind"); + } + + #[test] + fn an_install_without_the_program_says_so() { + let llm = Llm::with_host(None); + assert!(!llm.has_host()); + assert_eq!( + llm.generate(Path::new("m.gguf"), "s", "x", 10, 0.0).unwrap_err(), + LlmError::NoHost + ); + } +} diff --git a/app/src-tauri/src/intel/mod.rs b/app/src-tauri/src/intel/mod.rs new file mode 100644 index 00000000..f6c8ce89 --- /dev/null +++ b/app/src-tauri/src/intel/mod.rs @@ -0,0 +1,432 @@ +//! On-device intelligence for the interface (Phase 12): text in images, handwriting, read aloud, and summaries. +//! +//! Every command goes through `Engines`, which refuses a feature the person has not turned on, so no command checks +//! a switch itself. Nothing here opens a connection, and the crate behind it holds no network code. The person's +//! choices live in `intel.json` beside this device's state, because which engines work depends on the language +//! packs and voices installed on this computer. Every feature starts off. + +use std::{ + fs, + path::{Path, PathBuf}, + sync::{Arc, Mutex, OnceLock, PoisonError, RwLock, RwLockReadGuard}, +}; + +use opennote_intel::{ + wire::{ + ActionItem, ActionItemsRequest, Chapter, ChaptersRequest, InkRecognition, InkRequest, Keyword, KeywordsRequest, + OcrRequest, OcrResult, Offer, ReadAloudNotice, ReadAloudRequest, ReadAloudStarted, SpeechClip, SpeechHub, + SummarizeRequest, Summary, SynthesizeRequest, TidyPlan, TidyRequest, VocabularyOfferRequest, Voice, + }, + Engines, Feature, FeatureStatus, IntelError, IntelSettings, +}; +use serde::Deserialize; +use tauri::{async_runtime::spawn_blocking, ipc::Response, State}; + +use crate::{ + ipc::{codes, IpcError, IpcResult}, + paths::Paths, + settings::file::write_json, +}; + +pub(crate) mod caption_style; +mod catalog; +mod cloud; +mod ext; +mod llm; +mod models; +mod native; +pub(crate) mod stream; +pub(crate) mod transcribe; + +#[cfg(test)] +mod tests; + +/// The file that holds the person's choices, in this device's folder. +const FILE_NAME: &str = "intel.json"; + +/// The features to change. A feature left out stays as it is. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct IntelSettingsPatch { + pub ocr: Option, + pub handwriting: Option, + pub read_aloud: Option, + pub summaries: Option, + pub transcription: Option, +} + +impl IntelSettingsPatch { + fn apply(&self, settings: &mut IntelSettings) { + let choices = [ + (Feature::Ocr, self.ocr), + (Feature::Handwriting, self.handwriting), + (Feature::ReadAloud, self.read_aloud), + (Feature::Summaries, self.summaries), + (Feature::Transcription, self.transcription), + ]; + for (feature, on) in choices { + if let Some(on) = on { + settings.set(feature, on); + } + } + } +} + +/// The error the interface receives: the crate's code, its message, and the feature to offer in `field`. +pub fn ipc_error(error: &IntelError) -> IpcError { + let info = error.info(); + let field = info + .feature + .and_then(|feature| serde_json::to_value(feature).ok()) + .and_then(|value| value.as_str().map(str::to_owned)); + IpcError { + code: info.code, + message: info.message, + field, + } +} + +/// Reads the saved choices. A missing or unreadable file means everything is off, which is the safe answer. +fn load(file: &Path) -> IntelSettings { + match fs::read_to_string(file) { + Ok(text) => serde_json::from_str(&text).unwrap_or_else(|error| { + ::log::warn!( + "Couldn't read {}: {error}. Every on-device feature stays off.", + file.display() + ); + IntelSettings::default() + }), + Err(_) => IntelSettings::default(), + } +} + +struct Inner { + file: PathBuf, + /// The choices, which are also the source for the engines once they exist. + settings: Mutex, + /// Made on first use, so start-up does not wait for Windows to list engines. + engines: OnceLock>, + hub: SpeechHub, + ext: ext::Ext, + /// This device's folder, which holds the downloaded models. + device: PathBuf, + /// The transcription jobs, made on the first one, when the app handle for their events is known. + transcribe: OnceLock, +} + +/// The app's on-device intelligence: the person's choices, the engines behind them, and the read-aloud hub. +#[derive(Clone)] +pub struct IntelState(Arc); + +impl IntelState { + /// The cloud calls with the person's own keys (A1-33). + pub fn cloud(&self) -> &cloud::Cloud { + self.0.ext.cloud() + } + + pub fn new(paths: &Paths) -> IntelState { + IntelState::at(paths.local.join(FILE_NAME)) + } + + fn at(file: PathBuf) -> IntelState { + // Whisper.cpp, when this build carries it, takes over speech from the pure Rust reader. It is gated like + // every other engine: nothing runs until the person turns transcription on. + opennote_intel_engines::install(); + let settings = load(&file); + let device = file.parent().unwrap_or_else(|| Path::new(".")).to_path_buf(); + let ext = ext::Ext::new(&device); + IntelState(Arc::new(Inner { + file, + settings: Mutex::new(settings), + engines: OnceLock::new(), + hub: SpeechHub::new(), + ext, + device, + transcribe: OnceLock::new(), + })) + } + + fn choices(&self) -> IntelSettings { + // Safe mode turns the on-device models off (Phase 13), whatever the person chose. + if crate::hardening::safe_mode() { + return IntelSettings::default(); + } + *self.0.settings.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn engines(&self) -> RwLockReadGuard<'_, Engines> { + let engines = match self.0.engines.get() { + Some(engines) => engines, + None => { + // Made while holding the choices, so a change that lands meanwhile waits and then updates them. + let settings = self.0.settings.lock().unwrap_or_else(PoisonError::into_inner); + self.0.engines.get_or_init(|| { + let mut engines = Engines::platform(*settings); + engines.set_speech_model(transcribe::installed_speech_model(&self.0.device).as_deref()); + RwLock::new(engines) + }) + } + }; + engines.read().unwrap_or_else(PoisonError::into_inner) + } + + /// This device's folder. + fn device(&self) -> &Path { + &self.0.device + } + + /// Makes `model` the speech model the engines use, if it isn't already. + fn use_speech_model(&self, model: &Path) { + // Make the engines first, so the change below lands on them. + drop(self.engines()); + let engines = self.0.engines.get(); + if let Some(engines) = engines { + let mut engines = engines.write().unwrap_or_else(PoisonError::into_inner); + if engines.speech_model().as_deref() != Some(model) { + engines.set_speech_model(Some(model)); + } + } + } + + /// What the person has turned on. + pub fn settings(&self) -> IntelSettings { + self.choices() + } + + /// Changes the choices, saves them, and tells the engines. The change takes effect at the next request. + pub fn update(&self, patch: &IntelSettingsPatch) -> Result { + let mut settings = self.0.settings.lock().unwrap_or_else(PoisonError::into_inner); + let mut next = *settings; + patch.apply(&mut next); + if next != *settings { + let json = serde_json::to_value(next).map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))?; + write_json(&self.0.file, &json)?; + *settings = next; + if let Some(engines) = self.0.engines.get() { + engines + .write() + .unwrap_or_else(PoisonError::into_inner) + .set_settings(next); + } + } + Ok(next) + } + + pub fn status(&self) -> Vec { + self.engines().status() + } + + pub fn ocr_languages(&self) -> Result, IntelError> { + let languages = self.engines().ocr()?.available_languages()?; + Ok(languages.iter().map(|language| language.to_string()).collect()) + } + + pub fn ocr_recognize(&self, request: OcrRequest) -> Result { + let engine = self.engines().ocr()?; + let (image, options) = request.into_parts()?; + engine.recognize(&image, &options) + } + + pub fn ink_recognize(&self, request: &InkRequest) -> Result { + let engine = self.engines().ink()?; + engine.recognize(&request.strokes, &request.options()) + } + + pub fn ink_tidy(&self, request: &TidyRequest) -> Result { + self.engines() + .tidy(&request.strokes, &request.recognition, &request.operation) + } + + pub fn summarize(&self, request: &SummarizeRequest) -> Result { + let engine = self.engines().summarizer()?; + engine.summarize(&request.text, &request.options) + } + + pub fn keywords(&self, request: &KeywordsRequest) -> Result, IntelError> { + let engine = self.engines().summarizer()?; + engine.keywords(&request.text, &request.options) + } + + pub fn action_items(&self, request: &ActionItemsRequest) -> Result, IntelError> { + self.engines().action_items(&request.transcript) + } + + pub fn chapters(&self, request: &ChaptersRequest) -> Result, IntelError> { + self.engines().chapters(&request.transcript, &request.options) + } + + pub fn vocabulary_offer(&self, request: &VocabularyOfferRequest) -> Result, IntelError> { + request.offer(&self.engines()) + } + + pub fn voices(&self) -> Result, IntelError> { + self.engines().speech()?.voices() + } + + pub fn synthesize(&self, request: SynthesizeRequest) -> Result { + self.0.hub.synthesize(&self.engines(), request) + } + + pub fn read_aloud_start(&self, request: ReadAloudRequest) -> Result { + self.0.hub.start(&self.engines(), request) + } + + /// Waits for the session's next chunk, so it must not run on the interface thread. + pub fn read_aloud_next(&self, session_id: &str) -> Result { + self.0.hub.next(session_id) + } + + /// Ends a session. It waits for the chunk being made, so it must not run on the interface thread. + pub fn read_aloud_cancel(&self, session_id: &str) { + self.0.hub.cancel(session_id); + } + + pub fn clip_audio(&self, clip_id: &str) -> Result, IntelError> { + self.0.hub.clip_audio(clip_id) + } +} + +/// Runs engine work off the interface thread, because recognition and speech take from milliseconds to seconds. +async fn blocking(work: impl FnOnce() -> Result + Send + 'static) -> IpcResult { + match spawn_blocking(work).await { + Ok(result) => result.map_err(|error| ipc_error(&error)), + Err(error) => Err(IpcError::new(codes::INTERNAL, error.to_string())), + } +} + +#[tauri::command] +pub async fn intel_settings_get(state: State<'_, IntelState>) -> IpcResult { + Ok(state.settings()) +} + +#[tauri::command] +pub async fn intel_settings_set(state: State<'_, IntelState>, patch: IntelSettingsPatch) -> IpcResult { + let state = state.inner().clone(); + match spawn_blocking(move || state.update(&patch)).await { + Ok(result) => result, + Err(error) => Err(IpcError::new(codes::INTERNAL, error.to_string())), + } +} + +#[tauri::command] +pub async fn intel_status(state: State<'_, IntelState>) -> IpcResult> { + let state = state.inner().clone(); + blocking(move || Ok(state.status())).await +} + +#[tauri::command] +pub async fn intel_ocr_languages(state: State<'_, IntelState>) -> IpcResult> { + let state = state.inner().clone(); + blocking(move || state.ocr_languages()).await +} + +#[tauri::command] +pub async fn intel_ocr_recognize(state: State<'_, IntelState>, request: OcrRequest) -> IpcResult { + let state = state.inner().clone(); + blocking(move || state.ocr_recognize(request)).await +} + +#[tauri::command] +pub async fn intel_ink_recognize(state: State<'_, IntelState>, request: InkRequest) -> IpcResult { + let state = state.inner().clone(); + blocking(move || state.ink_recognize(&request)).await +} + +#[tauri::command] +pub async fn intel_ink_tidy(state: State<'_, IntelState>, request: TidyRequest) -> IpcResult { + let state = state.inner().clone(); + blocking(move || state.ink_tidy(&request)).await +} + +#[tauri::command] +pub async fn intel_summarize(state: State<'_, IntelState>, request: SummarizeRequest) -> IpcResult { + let state = state.inner().clone(); + blocking(move || state.summarize(&request)).await +} + +#[tauri::command] +pub async fn intel_keywords(state: State<'_, IntelState>, request: KeywordsRequest) -> IpcResult> { + let state = state.inner().clone(); + blocking(move || state.keywords(&request)).await +} + +#[tauri::command] +pub async fn intel_action_items( + state: State<'_, IntelState>, + request: ActionItemsRequest, +) -> IpcResult> { + let state = state.inner().clone(); + blocking(move || state.action_items(&request)).await +} + +#[tauri::command] +pub async fn intel_chapters(state: State<'_, IntelState>, request: ChaptersRequest) -> IpcResult> { + let state = state.inner().clone(); + blocking(move || state.chapters(&request)).await +} + +#[tauri::command] +pub async fn intel_vocabulary_offer( + state: State<'_, IntelState>, + request: VocabularyOfferRequest, +) -> IpcResult> { + let state = state.inner().clone(); + blocking(move || state.vocabulary_offer(&request)).await +} + +#[tauri::command] +pub async fn intel_speech_voices(state: State<'_, IntelState>) -> IpcResult> { + let state = state.inner().clone(); + blocking(move || state.voices()).await +} + +#[tauri::command] +pub async fn intel_speech_synthesize( + state: State<'_, IntelState>, + request: SynthesizeRequest, +) -> IpcResult { + let state = state.inner().clone(); + blocking(move || state.synthesize(request)).await +} + +#[tauri::command] +pub async fn intel_read_aloud_start( + state: State<'_, IntelState>, + request: ReadAloudRequest, +) -> IpcResult { + let state = state.inner().clone(); + blocking(move || state.read_aloud_start(request)).await +} + +#[tauri::command] +pub async fn intel_read_aloud_next(state: State<'_, IntelState>, session_id: String) -> IpcResult { + let state = state.inner().clone(); + blocking(move || state.read_aloud_next(&session_id)).await +} + +#[tauri::command] +pub async fn intel_read_aloud_cancel(state: State<'_, IntelState>, session_id: String) -> IpcResult<()> { + let state = state.inner().clone(); + blocking(move || { + state.read_aloud_cancel(&session_id); + Ok(()) + }) + .await +} + +/// The WAV bytes of a clip, raw and not as JSON numbers. The hub forgets the clip once it hands the bytes over. +#[tauri::command] +pub async fn intel_clip_audio(state: State<'_, IntelState>, clip_id: String) -> IpcResult { + let state = state.inner().clone(); + blocking(move || state.clip_audio(&clip_id).map(Response::new)).await +} + +/// Model downloads and the device store (see `ext.rs`). One command, so a new method adds no permission. +#[tauri::command] +pub async fn intel_ext_call(state: State<'_, IntelState>, request: ext::ExtRequest) -> IpcResult { + let ext = state.0.ext.clone(); + match spawn_blocking(move || ext.call(&request, crate::hardening::offline(), crate::hardening::safe_mode())).await { + Ok(result) => result, + Err(error) => Err(IpcError::new(codes::INTERNAL, error.to_string())), + } +} diff --git a/app/src-tauri/src/intel/models.rs b/app/src-tauri/src/intel/models.rs new file mode 100644 index 00000000..09e60a87 --- /dev/null +++ b/app/src-tauri/src/intel/models.rs @@ -0,0 +1,409 @@ +//! Model downloads (Phase 12): the speech models, the language models for Ask and the writing tools, the meaning +//! model, and the translation packs. A download starts only when the person chose it after seeing the size. It +//! resumes where it stopped, checks each file against a published checksum and size before it keeps it, and never +//! runs while Work offline or safe mode is on. Files go in this device's folder, so a portable install keeps them +//! beside its profile. The crate behind the other commands holds no network code, so this is the one place a model +//! can come from, and only from the pinned addresses in the catalog. + +use std::collections::HashMap; +use std::{ + fs, io, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, PoisonError, + }, + thread, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use serde::Serialize; + +use super::catalog; + +/// How a downloaded file is checked. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Digest { + /// The SHA-256 of the file, in hex. + Sha256(&'static str), + /// The git blob hash of the file, in hex: SHA-1 over `blob `, a zero byte, and the bytes. Hugging Face + /// publishes only this for a small file it keeps outside its large-file storage. + GitBlob(&'static str), +} + +impl Digest { + fn hex(self) -> &'static str { + match self { + Digest::Sha256(hex) | Digest::GitBlob(hex) => hex, + } + } +} + +/// One file of a model. +#[derive(Debug)] +pub struct Part { + /// The path under the models folder. + pub file: &'static str, + /// The exact size. A server that sends more is cut off, so this is also the cap. + pub size: u64, + pub digest: Digest, + pub url: &'static str, +} + +/// One model the person can download. +#[derive(Debug)] +pub struct ModelSpec { + pub id: &'static str, + /// What it is for: `speech`, `language`, `meaning`, or `translate`. + pub kind: &'static str, + pub name: &'static str, + pub detail: &'static str, + pub parts: &'static [Part], +} + +impl ModelSpec { + /// All the bytes the model takes. + pub fn size(&self) -> u64 { + self.parts.iter().map(|part| part.size).sum() + } + + /// The model's first file, the one a runtime opens. + pub fn main(&self) -> &'static Part { + &self.parts[0] + } +} + +/// The largest model the catalog may hold, as a guard against a typo in a size. +#[cfg(test)] +pub const MAX_MODEL_BYTES: u64 = 4 * 1024 * 1024 * 1024; + +/// Pinned to one commit, so the files cannot change under the checksums. +#[cfg(test)] +const SPEECH_HOST: &str = "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1"; + +macro_rules! speech { + ($id:literal, $name:literal, $detail:literal, $size:literal, $sha:literal, $file:literal) => { + ModelSpec { + id: $id, + kind: "speech", + name: $name, + detail: $detail, + parts: &[Part { + file: $file, + size: $size, + digest: Digest::Sha256($sha), + url: concat!("https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/", $file), + }], + } + }; +} + +/// The speech models, with the sizes and checksums Hugging Face publishes for each file. +const SPEECH: &[ModelSpec] = &[ + speech!( + "speech-tiny-en", + "Speech, smallest (English)", + "Fastest. Good for clear voices.", + 77_704_715, + "921e4cf8686fdd993dcd081a5da5b6c365bfde1162e72b08d75ac75289920b1f", + "ggml-tiny.en.bin" + ), + speech!( + "speech-base-en", + "Speech, balanced (English)", + "A good mix of speed and accuracy. Recommended.", + 147_964_211, + "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002", + "ggml-base.en.bin" + ), + speech!( + "speech-small-en", + "Speech, accurate (English)", + "Most accurate for English, and slower.", + 487_614_201, + "c6138d6d58ecc8322097e0f987c32f1be8bb0a18532a3f88f734d1bbf9c41e5d", + "ggml-small.en.bin" + ), + speech!( + "speech-base", + "Speech, balanced (many languages)", + "Understands many languages, and detects which one is spoken.", + 147_951_465, + "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe", + "ggml-base.bin" + ), + speech!( + "speech-small", + "Speech, accurate (many languages)", + "Most accurate across languages, and slower.", + 487_601_967, + "1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b", + "ggml-small.bin" + ), +]; + +/// Every model, speech first. +pub fn catalog() -> impl Iterator { + SPEECH + .iter() + .chain(catalog::LANGUAGE) + .chain(catalog::MEANING) + .chain(catalog::TRANSLATE) +} + +/// The model with this ID. +pub fn spec_of(id: &str) -> Option<&'static ModelSpec> { + catalog().find(|spec| spec.id == id) +} + +/// Why a download did not finish. The interface picks its own words from the code. +#[derive(Debug, PartialEq, Eq)] +pub enum DownloadError { + Canceled, + Offline, + SafeMode, + Network(String), + Disk(String), + Checksum, + /// A download is still winding down, so the model can't be removed yet. + Stopping, +} + +impl DownloadError { + pub fn code(&self) -> &'static str { + match self { + DownloadError::Canceled => "canceled", + DownloadError::Offline => "offline", + DownloadError::SafeMode => "safeMode", + DownloadError::Network(_) => "network", + DownloadError::Disk(_) => "disk", + DownloadError::Checksum => "checksum", + DownloadError::Stopping => "stopping", + } + } +} + +impl From for DownloadError { + fn from(error: io::Error) -> Self { + DownloadError::Disk(error.to_string()) + } +} + +/// How many 20 ms steps `remove` waits for a download to stop: a little over the 60 s read timeout. +const REMOVE_WAIT_STEPS: u32 = 3100; + +mod download; +mod source; +#[cfg(test)] +mod tests; + +pub use self::download::{download, is_installed}; +use self::download::{part_path, size_of}; +#[cfg(test)] +use self::source::Opened; +pub use self::source::{HttpSource, Source}; + +/// How one model looks to the interface. +#[derive(Serialize, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ModelView { + pub id: &'static str, + pub kind: &'static str, + pub name: &'static str, + pub detail: &'static str, + pub size_bytes: u64, + /// `notInstalled`, `partial`, `downloading`, `installed`, or `failed`. + pub state: &'static str, + /// Bytes on disk or downloaded so far. + pub bytes: u64, + /// The code of the last failure, for `failed`. + pub error: Option<&'static str>, +} + +/// The list the interface shows, with when a download last ran. +#[derive(Serialize, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ModelList { + pub models: Vec, + /// Seconds since 1970 when a download last started, or none. + pub last_ran_unix: Option, +} + +struct Job { + cancel: Arc, + bytes: Arc, + running: bool, + error: Option<&'static str>, +} + +struct Inner { + dir: PathBuf, + source: Arc, + jobs: Mutex>, +} + +/// The downloads of this device, one at a time for each model. +#[derive(Clone)] +pub struct Models(Arc); + +impl Models { + pub fn new(dir: PathBuf) -> Models { + Models::with_source(dir, Arc::new(HttpSource)) + } + + pub fn with_source(dir: PathBuf, source: Arc) -> Models { + Models(Arc::new(Inner { + dir, + source, + jobs: Mutex::new(HashMap::new()), + })) + } + + fn jobs(&self) -> std::sync::MutexGuard<'_, HashMap<&'static str, Job>> { + self.0.jobs.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn last_ran_file(&self) -> PathBuf { + self.0.dir.join("last-ran.txt") + } + + /// The folder the models are in. + pub fn dir(&self) -> &Path { + &self.0.dir + } + + /// The path of the model's first file, if the whole model is on disk. + pub fn installed_path(&self, id: &str) -> Option { + let spec = spec_of(id)?; + is_installed(&self.0.dir, spec).then(|| self.0.dir.join(spec.main().file)) + } + + pub fn list(&self) -> ModelList { + let jobs = self.jobs(); + let models = catalog() + .map(|spec| { + let installed = is_installed(&self.0.dir, spec); + let job = jobs.get(spec.id); + let on_disk: u64 = spec + .parts + .iter() + .map(|part| { + size_of(&self.0.dir.join(part.file)) + .or_else(|| size_of(&part_path(&self.0.dir, part))) + .unwrap_or(0) + }) + .sum(); + let (state, bytes, error) = match job { + _ if installed => ("installed", spec.size(), None), + Some(job) if job.running => ("downloading", job.bytes.load(Ordering::Relaxed), None), + Some(Job { error: Some(code), .. }) if *code != "canceled" => ("failed", on_disk, Some(*code)), + _ if on_disk > 0 => ("partial", on_disk, None), + _ => ("notInstalled", 0, None), + }; + ModelView { + id: spec.id, + kind: spec.kind, + name: spec.name, + detail: spec.detail, + size_bytes: spec.size(), + state, + bytes, + error, + } + }) + .collect(); + let last_ran_unix = fs::read_to_string(self.last_ran_file()) + .ok() + .and_then(|text| text.trim().parse().ok()); + ModelList { models, last_ran_unix } + } + + /// Starts or resumes a download on a thread of its own. `blocked` says why none may run now. + pub fn start(&self, id: &str, blocked: Option) -> Result<(), DownloadError> { + let spec = spec_of(id).ok_or_else(|| DownloadError::Disk("That model isn't in the catalog.".to_owned()))?; + if let Some(reason) = blocked { + return Err(reason); + } + let (cancel, bytes) = { + let mut jobs = self.jobs(); + if jobs.get(spec.id).is_some_and(|job| job.running) { + return Ok(()); + } + let cancel = Arc::new(AtomicBool::new(false)); + let bytes = Arc::new(AtomicU64::new(0)); + jobs.insert( + spec.id, + Job { + cancel: cancel.clone(), + bytes: bytes.clone(), + running: true, + error: None, + }, + ); + (cancel, bytes) + }; + let now = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |d| d.as_secs()); + let _ = fs::create_dir_all(&self.0.dir); + let _ = fs::write(self.last_ran_file(), now.to_string()); + let this = self.clone(); + thread::Builder::new() + .name(format!("model-download-{}", spec.id)) + .spawn(move || { + let result = download(this.0.source.as_ref(), spec, &this.0.dir, &cancel, &bytes); + let mut jobs = this.jobs(); + if let Some(job) = jobs.get_mut(spec.id) { + job.running = false; + job.error = result.err().map(|error| error.code()); + } + }) + .map_err(|error| DownloadError::Disk(error.to_string()))?; + Ok(()) + } + + /// Stops a running download and keeps the part that arrived, so the next start resumes. + pub fn cancel(&self, id: &str) { + if let Some(job) = self.jobs().get(id) { + job.cancel.store(true, Ordering::Relaxed); + } + } + + /// Deletes the model and any part of it, after stopping a running download. + pub fn remove(&self, id: &str) -> Result<(), DownloadError> { + self.remove_waiting(id, REMOVE_WAIT_STEPS) + } + + fn remove_waiting(&self, id: &str, wait_steps: u32) -> Result<(), DownloadError> { + let spec = spec_of(id).ok_or_else(|| DownloadError::Disk("That model isn't in the catalog.".to_owned()))?; + self.cancel(id); + // Wait for the thread to let go of the file. It sees the flag between reads, and a read can last as long as + // the connection's timeout, so the wait is longer than that. If it still runs, nothing is deleted, and the + // job stays, so the list tells the truth and a second Start can't begin another thread on the same file. + for _ in 0..wait_steps { + if !self.jobs().get(id).is_some_and(|job| job.running) { + break; + } + thread::sleep(Duration::from_millis(20)); + } + if self.jobs().get(id).is_some_and(|job| job.running) { + return Err(DownloadError::Stopping); + } + for part in spec.parts { + for path in [self.0.dir.join(part.file), part_path(&self.0.dir, part)] { + match fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + } + } + // A pack lives in a folder of its own. Take it away once it is empty. + if let Some(folder) = Path::new(spec.main().file) + .parent() + .filter(|p| !p.as_os_str().is_empty()) + { + let _ = fs::remove_dir(self.0.dir.join(folder)); + } + self.jobs().remove(id); + Ok(()) + } +} diff --git a/app/src-tauri/src/intel/models/download.rs b/app/src-tauri/src/intel/models/download.rs new file mode 100644 index 00000000..72631a7d --- /dev/null +++ b/app/src-tauri/src/intel/models/download.rs @@ -0,0 +1,155 @@ +//! Downloading the files of a model: resumed from a `.part` file, cut off at the published size, and kept only when the +//! checksum matches. + +use std::{ + fs::{self, File, OpenOptions}, + io::{self, Read, Write}, + path::{Path, PathBuf}, + sync::atomic::{AtomicBool, AtomicU64, Ordering}, +}; + +use sha1::Sha1; +use sha2::{Digest as _, Sha256}; + +use super::{Digest, DownloadError, ModelSpec, Part, Source}; + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Whether the file at `path` matches `digest` and `size`. +fn matches(path: &Path, size: u64, digest: Digest) -> io::Result { + let mut file = File::open(path)?; + let mut buffer = vec![0u8; 1 << 16]; + let found = match digest { + Digest::Sha256(_) => { + let mut hasher = Sha256::new(); + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + hex(&hasher.finalize()) + } + Digest::GitBlob(_) => { + let mut hasher = Sha1::new(); + hasher.update(format!("blob {size}\0").as_bytes()); + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + hex(&hasher.finalize()) + } + }; + Ok(found == digest.hex()) +} + +pub(super) fn part_path(dir: &Path, part: &Part) -> PathBuf { + let mut name = dir.join(part.file).into_os_string(); + name.push(".part"); + PathBuf::from(name) +} + +pub(super) fn size_of(path: &Path) -> Option { + fs::metadata(path).ok().map(|meta| meta.len()) +} + +/// Whether every file of the model is on disk at its published size. +pub fn is_installed(dir: &Path, spec: &ModelSpec) -> bool { + spec.parts + .iter() + .all(|part| size_of(&dir.join(part.file)) == Some(part.size)) +} + +/// Downloads one file into `dir`, resuming a `.part` file left by an earlier try. The file is kept only when its +/// size and checksum match the catalog. Cancel leaves the `.part` file so the next try resumes. `progress` holds +/// the bytes of the whole model: `base` is what the earlier files took. +fn download_part( + source: &dyn Source, + part: &Part, + dir: &Path, + cancel: &AtomicBool, + progress: &AtomicU64, + base: u64, +) -> Result<(), DownloadError> { + let target = dir.join(part.file); + if size_of(&target) == Some(part.size) { + progress.store(base + part.size, Ordering::Relaxed); + return Ok(()); + } + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + let partial = part_path(dir, part); + let mut have = size_of(&partial).unwrap_or(0); + if have > part.size { + fs::remove_file(&partial)?; + have = 0; + } + if have < part.size { + let opened = source.open(part.url, have).map_err(DownloadError::Network)?; + let mut file = if opened.resumed && have > 0 { + OpenOptions::new().append(true).open(&partial)? + } else { + have = 0; + File::create(&partial)? + }; + progress.store(base + have, Ordering::Relaxed); + // Never read past the published size, whatever the server sends. + let mut reader = opened.reader.take(part.size - have + 1); + let mut buffer = vec![0u8; 1 << 16]; + while have <= part.size { + if cancel.load(Ordering::Relaxed) { + file.flush()?; + return Err(DownloadError::Canceled); + } + let read = reader + .read(&mut buffer) + .map_err(|error| DownloadError::Network(error.to_string()))?; + if read == 0 { + break; + } + file.write_all(&buffer[..read])?; + have += read as u64; + progress.store(base + have.min(part.size), Ordering::Relaxed); + } + file.flush()?; + drop(file); + if have != part.size { + return if have > part.size { + fs::remove_file(&partial)?; + Err(DownloadError::Checksum) + } else { + Err(DownloadError::Network("The connection ended early.".to_owned())) + }; + } + } + if !matches(&partial, part.size, part.digest)? { + fs::remove_file(&partial)?; + return Err(DownloadError::Checksum); + } + fs::rename(&partial, &target)?; + Ok(()) +} + +/// Downloads every file of a model, one after another. +pub fn download( + source: &dyn Source, + spec: &ModelSpec, + dir: &Path, + cancel: &AtomicBool, + progress: &AtomicU64, +) -> Result<(), DownloadError> { + fs::create_dir_all(dir)?; + let mut base = 0; + for part in spec.parts { + download_part(source, part, dir, cancel, progress, base)?; + base += part.size; + } + Ok(()) +} diff --git a/app/src-tauri/src/intel/models/source.rs b/app/src-tauri/src/intel/models/source.rs new file mode 100644 index 00000000..06ff6c16 --- /dev/null +++ b/app/src-tauri/src/intel/models/source.rs @@ -0,0 +1,56 @@ +//! Where model bytes come from. + +use std::{io::Read, time::Duration}; + +/// What a server answered to a request for a file from some byte on. +pub struct Opened { + /// True when the answer starts at the byte asked for, and false when the server sent the file from the start. + pub resumed: bool, + pub reader: Box, +} + +/// Where model bytes come from. The app uses [`HttpSource`], and tests use a stand-in. +pub trait Source: Send + Sync { + fn open(&self, url: &str, from: u64) -> Result; +} + +/// Downloads over HTTPS with Windows' TLS stack, as the updater does. No cookies and no proxy settings are read. +pub struct HttpSource; + +impl Source for HttpSource { + fn open(&self, url: &str, from: u64) -> Result { + use ureq::tls::{RootCerts, TlsConfig, TlsProvider}; + // Only the catalog's own addresses are ever opened. + if !url.starts_with("https://huggingface.co/") { + return Err("That address is not one OpenNote downloads from.".to_owned()); + } + let tls = TlsConfig::builder() + .provider(TlsProvider::NativeTls) + .root_certs(RootCerts::PlatformVerifier) + .build(); + let agent: ureq::Agent = ureq::Agent::config_builder() + .tls_config(tls) + .max_redirects(5) + .http_status_as_error(false) + .timeout_connect(Some(Duration::from_secs(20))) + .timeout_recv_response(Some(Duration::from_secs(60))) + .user_agent(concat!("OpenNote/", env!("CARGO_PKG_VERSION"))) + .build() + .into(); + let mut request = agent.get(url); + if from > 0 { + request = request.header("Range", format!("bytes={from}-")); + } + let response = request.call().map_err(|error| error.to_string())?; + let status = response.status().as_u16(); + let resumed = match status { + 200 => false, + 206 => true, + other => return Err(format!("The server answered with status {other}.")), + }; + Ok(Opened { + resumed, + reader: Box::new(response.into_body().into_reader()), + }) + } +} diff --git a/app/src-tauri/src/intel/models/tests.rs b/app/src-tauri/src/intel/models/tests.rs new file mode 100644 index 00000000..414c5e6c --- /dev/null +++ b/app/src-tauri/src/intel/models/tests.rs @@ -0,0 +1,365 @@ +use std::{ + fs::{self, File}, + io::{self}, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, + }, +}; + +use super::*; + +const DATA: &[u8] = b"hello world"; +const HELLO_SHA: &str = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"; +/// `printf 'blob 11\0hello world' | sha1sum` +const HELLO_BLOB: &str = "95d09f2b10159347eece71399a7e2e907ea3df4f"; + +const SPEC: ModelSpec = ModelSpec { + id: "test", + kind: "speech", + name: "Test", + detail: "", + parts: &[Part { + file: "test.bin", + size: DATA.len() as u64, + digest: Digest::Sha256(HELLO_SHA), + url: "https://example.invalid/test.bin", + }], +}; + +const PACK: ModelSpec = ModelSpec { + id: "pack", + kind: "translate", + name: "Pack", + detail: "", + parts: &[ + Part { + file: "pack/one.bin", + size: DATA.len() as u64, + digest: Digest::Sha256(HELLO_SHA), + url: "https://example.invalid/one", + }, + Part { + file: "pack/two.json", + size: DATA.len() as u64, + digest: Digest::GitBlob(HELLO_BLOB), + url: "https://example.invalid/two", + }, + ], +}; + +/// Serves `data`, honoring a start offset unless `ignore_range` is set, and asks `after` to run first. +struct Memory { + data: Vec, + ignore_range: bool, + asked: Mutex>, +} + +impl Source for Memory { + fn open(&self, _url: &str, from: u64) -> Result { + self.asked.lock().unwrap().push(from); + let start = if self.ignore_range { 0 } else { from as usize }; + Ok(Opened { + resumed: !self.ignore_range && from > 0, + reader: Box::new(io::Cursor::new(self.data[start..].to_vec())), + }) + } +} + +fn memory(data: &[u8], ignore_range: bool) -> Memory { + Memory { + data: data.to_vec(), + ignore_range, + asked: Mutex::new(Vec::new()), + } +} + +#[test] +fn the_catalog_lists_checksums_sizes_and_pinned_https_addresses() { + for spec in catalog() { + assert!(!spec.parts.is_empty(), "{}", spec.id); + assert!(spec.size() <= MAX_MODEL_BYTES, "{}", spec.id); + for part in spec.parts { + let expected = match part.digest { + Digest::Sha256(_) => 64, + Digest::GitBlob(_) => 40, + }; + assert_eq!(part.digest.hex().len(), expected, "{}", part.file); + assert!( + part.digest.hex().bytes().all(|b| b.is_ascii_hexdigit()), + "{}", + part.file + ); + assert!(part.url.starts_with("https://huggingface.co/"), "{}", part.file); + assert!(part.url.ends_with(part.file.rsplit('/').next().unwrap()) || spec.kind != "speech"); + assert!(part.size > 0 && part.size <= MAX_MODEL_BYTES, "{}", part.file); + assert!( + !part.file.contains("..") && !part.file.starts_with('/'), + "{}", + part.file + ); + if spec.kind == "speech" { + assert!(part.url.starts_with(SPEECH_HOST), "{}", spec.id); + } + // Everything is pinned to a commit, not to a branch that can change under the checksum. + let revision = part + .url + .split("/resolve/") + .nth(1) + .and_then(|rest| rest.split('/').next()); + let revision = revision.expect("a resolve address"); + assert_eq!(revision.len(), 40, "{}", part.url); + assert!(revision.bytes().all(|b| b.is_ascii_hexdigit()), "{}", part.url); + // Hugging Face stores every file over 10 MB in its large-file storage, which publishes a sha256 for it. Only a + // smaller file may have just git's own hash. + if part.size > 10 * 1024 * 1024 { + assert!(matches!(part.digest, Digest::Sha256(_)), "{}", part.file); + } + } + } + let mut ids: Vec<_> = catalog().map(|spec| spec.id).collect(); + ids.sort_unstable(); + let total = ids.len(); + ids.dedup(); + assert_eq!(ids.len(), total); + for kind in ["speech", "language", "meaning", "translate"] { + assert!(catalog().any(|spec| spec.kind == kind), "{kind}"); + } + // No two models write the same file. + let mut files: Vec<_> = catalog() + .flat_map(|spec| spec.parts.iter().map(|part| part.file)) + .collect(); + files.sort_unstable(); + let total = files.len(); + files.dedup(); + assert_eq!(files.len(), total); +} + +#[test] +fn the_web_previews_pack_list_matches_the_catalog() { + // Each row of the preview's list reads `['en', 'es', 119006287],`. + let listed = include_str!("../../../../src/services/intel/packs.ts"); + let mut in_preview: Vec<(String, u64)> = listed + .lines() + .filter_map(|line| { + let row = line.trim().strip_prefix('[')?.strip_suffix("],")?; + let mut cells = row.split(',').map(|cell| cell.trim().trim_matches('\'')); + let (from, to, size) = (cells.next()?, cells.next()?, cells.next()?); + Some((format!("translate-{from}-{to}"), size.parse().ok()?)) + }) + .collect(); + let mut in_catalog: Vec<(String, u64)> = catalog() + .filter(|spec| spec.kind == "translate") + .map(|spec| (spec.id.to_owned(), spec.size())) + .collect(); + in_preview.sort(); + in_catalog.sort(); + assert_eq!(in_preview, in_catalog); +} + +#[test] +fn a_download_is_kept_after_its_checksum_matches() { + let dir = tempfile::tempdir().unwrap(); + let progress = AtomicU64::new(0); + download( + &memory(DATA, false), + &SPEC, + dir.path(), + &AtomicBool::new(false), + &progress, + ) + .unwrap(); + assert_eq!(fs::read(dir.path().join("test.bin")).unwrap(), DATA); + assert!(!dir.path().join("test.bin.part").exists()); + assert_eq!(progress.load(Ordering::Relaxed), DATA.len() as u64); +} + +#[test] +fn a_pack_downloads_every_file_and_checks_each_kind_of_checksum() { + let dir = tempfile::tempdir().unwrap(); + let progress = AtomicU64::new(0); + download( + &memory(DATA, false), + &PACK, + dir.path(), + &AtomicBool::new(false), + &progress, + ) + .unwrap(); + assert_eq!(fs::read(dir.path().join("pack/one.bin")).unwrap(), DATA); + assert_eq!(fs::read(dir.path().join("pack/two.json")).unwrap(), DATA); + assert_eq!(progress.load(Ordering::Relaxed), PACK.size()); + assert!(is_installed(dir.path(), &PACK)); +} + +#[test] +fn a_wrong_git_hash_deletes_the_file() { + const BAD: ModelSpec = ModelSpec { + id: "bad", + kind: "translate", + name: "Bad", + detail: "", + parts: &[Part { + file: "bad/two.json", + size: DATA.len() as u64, + digest: Digest::GitBlob("0000000000000000000000000000000000000000"), + url: "https://example.invalid/two", + }], + }; + let dir = tempfile::tempdir().unwrap(); + let result = download( + &memory(DATA, false), + &BAD, + dir.path(), + &AtomicBool::new(false), + &AtomicU64::new(0), + ); + assert_eq!(result, Err(DownloadError::Checksum)); + assert!(!dir.path().join("bad/two.json").exists()); + assert!(!dir.path().join("bad/two.json.part").exists()); +} + +#[test] +fn a_server_that_sends_more_than_the_published_size_is_cut_off() { + let dir = tempfile::tempdir().unwrap(); + let mut long = DATA.to_vec(); + long.extend(std::iter::repeat_n(b'x', 1000)); + let result = download( + &memory(&long, false), + &SPEC, + dir.path(), + &AtomicBool::new(false), + &AtomicU64::new(0), + ); + assert_eq!(result, Err(DownloadError::Checksum)); + assert!(!dir.path().join("test.bin").exists()); + assert!(!dir.path().join("test.bin.part").exists()); +} + +#[test] +fn a_stopped_download_resumes_from_its_part() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("test.bin.part"), &DATA[..5]).unwrap(); + let source = memory(DATA, false); + download(&source, &SPEC, dir.path(), &AtomicBool::new(false), &AtomicU64::new(0)).unwrap(); + assert_eq!(*source.asked.lock().unwrap(), vec![5]); + assert_eq!(fs::read(dir.path().join("test.bin")).unwrap(), DATA); +} + +#[test] +fn a_server_that_ignores_the_range_restarts_the_file() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("test.bin.part"), &DATA[..5]).unwrap(); + download( + &memory(DATA, true), + &SPEC, + dir.path(), + &AtomicBool::new(false), + &AtomicU64::new(0), + ) + .unwrap(); + assert_eq!(fs::read(dir.path().join("test.bin")).unwrap(), DATA); +} + +#[test] +fn a_wrong_checksum_deletes_the_file() { + let dir = tempfile::tempdir().unwrap(); + let result = download( + &memory(b"HELLO WORLD", false), + &SPEC, + dir.path(), + &AtomicBool::new(false), + &AtomicU64::new(0), + ); + assert_eq!(result, Err(DownloadError::Checksum)); + assert!(!dir.path().join("test.bin").exists()); + assert!(!dir.path().join("test.bin.part").exists()); +} + +#[test] +fn cancel_keeps_the_part_for_the_next_try() { + let dir = tempfile::tempdir().unwrap(); + let result = download( + &memory(DATA, false), + &SPEC, + dir.path(), + &AtomicBool::new(true), + &AtomicU64::new(0), + ); + assert_eq!(result, Err(DownloadError::Canceled)); + assert!(dir.path().join("test.bin.part").exists()); + assert!(!dir.path().join("test.bin").exists()); +} + +#[test] +fn a_short_answer_is_an_error_that_can_resume() { + let dir = tempfile::tempdir().unwrap(); + let result = download( + &memory(&DATA[..4], false), + &SPEC, + dir.path(), + &AtomicBool::new(false), + &AtomicU64::new(0), + ); + assert!(matches!(result, Err(DownloadError::Network(_)))); + assert_eq!(size_of(&dir.path().join("test.bin.part")), Some(4)); +} + +#[test] +fn a_blocked_start_does_not_run_and_the_list_shows_what_is_on_disk() { + let dir = tempfile::tempdir().unwrap(); + let models = Models::with_source(dir.path().to_path_buf(), Arc::new(memory(DATA, false))); + assert_eq!( + models.start("speech-base-en", Some(DownloadError::Offline)), + Err(DownloadError::Offline) + ); + let first = &models.list().models[0]; + assert_eq!((first.state, first.bytes), ("notInstalled", 0)); + let spec = catalog().next().unwrap(); + fs::write(dir.path().join(format!("{}.part", spec.main().file)), b"abc").unwrap(); + let listed = models.list(); + assert_eq!((listed.models[0].state, listed.models[0].bytes), ("partial", 3)); + models.remove(spec.id).unwrap(); + assert_eq!(models.list().models[0].state, "notInstalled"); +} + +#[test] +fn a_model_whose_download_will_not_stop_is_kept_and_says_so() { + let dir = tempfile::tempdir().unwrap(); + let models = Models::with_source(dir.path().to_path_buf(), Arc::new(memory(DATA, false))); + let spec = catalog().next().unwrap(); + models.jobs().insert( + spec.id, + Job { + cancel: Arc::new(AtomicBool::new(false)), + bytes: Arc::new(AtomicU64::new(0)), + running: true, + error: None, + }, + ); + let part = dir.path().join(format!("{}.part", spec.main().file)); + fs::write(&part, b"abc").unwrap(); + assert_eq!(models.remove_waiting(spec.id, 2), Err(DownloadError::Stopping)); + assert!(part.exists()); + assert!(models.jobs().contains_key(spec.id)); + assert_eq!(DownloadError::Stopping.code(), "stopping"); +} + +#[test] +fn a_pack_is_installed_only_when_every_file_is_there_and_remove_takes_its_folder() { + let dir = tempfile::tempdir().unwrap(); + let models = Models::with_source(dir.path().to_path_buf(), Arc::new(memory(DATA, false))); + let spec = spec_of("translate-en-es").unwrap(); + assert!(models.installed_path(spec.id).is_none()); + for part in spec.parts { + let path = dir.path().join(part.file); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + File::create(&path).unwrap().set_len(part.size).unwrap(); + } + assert!(models.installed_path(spec.id).is_some()); + let one = dir.path().join(spec.parts[1].file); + fs::remove_file(&one).unwrap(); + assert!(models.installed_path(spec.id).is_none()); + models.remove(spec.id).unwrap(); + assert!(!dir.path().join("translate-en-es").exists()); +} diff --git a/app/src-tauri/src/intel/native.rs b/app/src-tauri/src/intel/native.rs new file mode 100644 index 00000000..74e5d483 --- /dev/null +++ b/app/src-tauri/src/intel/native.rs @@ -0,0 +1,236 @@ +//! Translation and meaning vectors, from the app's side (A3-25, A3-23). Both run in this process on the model files +//! the person downloaded: `opennote-intel-engines` holds the engines, and builds without its `onnx` feature answer +//! `unavailable`. One translation pack and the meaning model stay loaded after use, and a different pack replaces +//! the one before it. + +use std::path::Path; + +#[cfg(feature = "native-engines")] +use std::sync::{Arc, Mutex, PoisonError}; + +#[cfg(feature = "native-engines")] +use opennote_intel_engines::onnx::{Embedder, OnnxError, Translator}; + +/// The most lines one translation request may carry, and the most characters in all. +pub const MAX_LINES: usize = 400; +pub const MAX_CHARS: usize = 400_000; +/// The most texts one request for vectors may carry. +pub const MAX_TEXTS: usize = 64; + +/// Why a call to an engine failed. +#[derive(Debug, PartialEq, Eq)] +#[allow(dead_code)] +pub enum NativeError { + /// This build does not carry the engine. + Unavailable, + /// The model is not on this computer. + Model(String), + /// The engine failed. + Engine(String), + /// The request is over a limit. + Invalid(String), +} + +impl NativeError { + pub fn code(&self) -> &'static str { + match self { + NativeError::Unavailable => "unavailable", + NativeError::Model(_) => "model", + NativeError::Engine(_) => "engine", + NativeError::Invalid(_) => "invalid", + } + } + + pub fn message(&self) -> String { + match self { + NativeError::Unavailable => "This build does not carry that engine.".to_owned(), + NativeError::Model(message) | NativeError::Engine(message) | NativeError::Invalid(message) => { + message.clone() + } + } + } +} + +#[cfg(feature = "native-engines")] +impl From for NativeError { + fn from(error: OnnxError) -> Self { + match error { + OnnxError::Model(message) => NativeError::Model(message), + OnnxError::Runtime(message) => NativeError::Engine(message), + OnnxError::Input(message) => NativeError::Invalid(message), + } + } +} + +/// Checks the size of a translation request. +pub fn check_lines(lines: &[String]) -> Result<(), NativeError> { + if lines.len() > MAX_LINES { + return Err(NativeError::Invalid( + "There are too many lines to translate at once.".to_owned(), + )); + } + if lines.iter().map(String::len).sum::() > MAX_CHARS { + return Err(NativeError::Invalid( + "The text is too long to translate at once.".to_owned(), + )); + } + Ok(()) +} + +/// Checks the size of a request for vectors. +pub fn check_texts(texts: &[String]) -> Result<(), NativeError> { + if texts.len() > MAX_TEXTS { + return Err(NativeError::Invalid("There are too many texts at once.".to_owned())); + } + Ok(()) +} + +/// The engines of this process. +#[derive(Default)] +pub struct Native { + #[cfg(feature = "native-engines")] + translator: Mutex)>>, + #[cfg(feature = "native-engines")] + embedder: Mutex>>, +} + +#[cfg(feature = "native-engines")] +impl Native { + /// Translates `lines` with the pack in `dir` (its id is `id`). + pub fn translate(&self, id: &str, dir: &Path, lines: &[String]) -> Result, NativeError> { + check_lines(lines)?; + let translator = { + let mut held = self.translator.lock().unwrap_or_else(PoisonError::into_inner); + match held.as_ref() { + Some((have, translator)) if have == id => Arc::clone(translator), + _ => { + // Let the old pack go before the new one is read. + *held = None; + let translator = Arc::new(Translator::open(dir)?); + *held = Some((id.to_owned(), Arc::clone(&translator))); + translator + } + } + }; + Ok(translator.translate(lines)?) + } + + /// The vector of each text, using the meaning model in `dir`. + pub fn embed(&self, dir: &Path, texts: &[String]) -> Result>, NativeError> { + check_texts(texts)?; + let embedder = { + let mut held = self.embedder.lock().unwrap_or_else(PoisonError::into_inner); + match held.as_ref() { + Some(embedder) => Arc::clone(embedder), + None => { + let embedder = Arc::new(Embedder::open(dir)?); + *held = Some(Arc::clone(&embedder)); + embedder + } + } + }; + Ok(embedder.embed(texts)?) + } + + /// Lets go of the loaded models, such as when the person removes one. + pub fn unload(&self) { + *self.translator.lock().unwrap_or_else(PoisonError::into_inner) = None; + *self.embedder.lock().unwrap_or_else(PoisonError::into_inner) = None; + } +} + +#[cfg(not(feature = "native-engines"))] +impl Native { + pub fn translate(&self, _id: &str, _dir: &Path, lines: &[String]) -> Result, NativeError> { + check_lines(lines)?; + Err(NativeError::Unavailable) + } + + pub fn embed(&self, _dir: &Path, texts: &[String]) -> Result>, NativeError> { + check_texts(texts)?; + Err(NativeError::Unavailable) + } + + pub fn unload(&self) {} +} + +/// What this build carries, for the interface. +pub fn capabilities() -> serde_json::Value { + serde_json::json!({ + "whisperCpp": opennote_intel_engines::HAS_WHISPER_CPP, + "whisperGpu": opennote_intel_engines::HAS_WHISPER_GPU, + "onnx": opennote_intel_engines::HAS_ONNX, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn requests_over_a_limit_are_refused_before_any_engine_is_touched() { + let many = vec!["a".to_owned(); MAX_LINES + 1]; + let native = Native::default(); + let dir = Path::new("nowhere"); + assert_eq!(native.translate("x", dir, &many).unwrap_err().code(), "invalid"); + let long = vec!["x".repeat(MAX_CHARS + 1)]; + assert_eq!(native.translate("x", dir, &long).unwrap_err().code(), "invalid"); + let texts = vec![String::new(); MAX_TEXTS + 1]; + assert_eq!(native.embed(dir, &texts).unwrap_err().code(), "invalid"); + } + + #[cfg(not(feature = "native-engines"))] + #[test] + fn a_build_without_the_engines_says_so() { + let native = Native::default(); + let error = native + .translate("x", Path::new("nowhere"), &["hi".to_owned()]) + .unwrap_err(); + assert_eq!(error, NativeError::Unavailable); + assert_eq!(capabilities()["onnx"], false); + } + + #[cfg(feature = "native-engines")] + fn fixtures(name: &str) -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../crates/intel-engines/tests/fixtures") + .join(name) + } + + #[cfg(feature = "native-engines")] + #[test] + fn a_pack_translates_and_the_meaning_model_embeds_through_the_shell() { + let native = Native::default(); + let lines = ["hello world".to_owned()]; + assert_eq!( + native + .translate("translate-test", &fixtures("translate"), &lines) + .unwrap(), + ["hola mundo"] + ); + // A second pack replaces the first, and the first can be loaded again. + assert_eq!( + native + .translate("translate-other", &fixtures("translate"), &lines) + .unwrap(), + ["hola mundo"] + ); + let vectors = native.embed(&fixtures("embed"), &["cat".to_owned()]).unwrap(); + assert_eq!(vectors[0].len(), 384); + native.unload(); + assert_eq!( + native.embed(&fixtures("embed"), &["kitten".to_owned()]).unwrap().len(), + 1 + ); + } + + #[cfg(feature = "native-engines")] + #[test] + fn a_pack_that_is_not_there_is_a_model_error() { + let native = Native::default(); + let error = native + .translate("x", Path::new("nowhere"), &["hi".to_owned()]) + .unwrap_err(); + assert_eq!(error.code(), "model"); + } +} diff --git a/app/src-tauri/src/intel/stream.rs b/app/src-tauri/src/intel/stream.rs new file mode 100644 index 00000000..557b1b7c --- /dev/null +++ b/app/src-tauri/src/intel/stream.rs @@ -0,0 +1,268 @@ +//! Speech to text while the person talks (A2-24 dictation and A2-25 live captions). The interface asks to start a +//! session, the shell opens the default microphone, and the Whisper model on this device reads each stretch of +//! speech. Words reach the interface as `intel-stream` events with the session's ID. The audio is read in memory, +//! never written to disk, and never leaves this device; a session ends when the interface stops it, when +//! transcription is turned off, or when the window closes. +//! +//! The microphone opens only after the interface asks, and the interface asks only after the person pressed +//! Dictate or turned on live captions. Windows shows its own microphone indicator while the stream is open. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock, Mutex, PoisonError}; + +use opennote_intel::wire::{StreamEvent, StreamOptions}; +use opennote_intel::{Feature, IntelError}; +use serde::{Deserialize, Serialize}; +use tauri::{AppHandle, Emitter, State}; + +use super::transcribe::speech_model_path; +use super::IntelState; +use crate::ipc::{codes, IpcError, IpcResult}; + +/// The event every session update arrives as. +pub const EVENT: &str = "intel-stream"; + +/// The most sessions open at once: one of dictation and one of captions. +const MAX_SESSIONS: usize = 2; +/// The most vocabulary text a request may carry. +const MAX_VOCABULARY_BYTES: usize = 64 * 1024; + +/// What the session is for, which sets how it finds the ends of phrases. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Purpose { + Dictation, + Captions, +} + +/// What to listen for. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamRequest { + pub purpose: Purpose, + /// The catalog ID of the downloaded speech model. + pub model: String, + /// A BCP 47 tag, or none to let the model detect it. + #[serde(default)] + pub language: Option, + /// The notebook's vocabulary list, as plain text. + #[serde(default)] + pub vocabulary: String, +} + +/// A session's news, for the interface. +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum StreamUpdate { + /// The microphone is open and the model is ready. + Listening, + /// Speech was noticed. + Speaking, + /// A rough reading of the phrase so far. + Partial { text: String, behind_ms: u64 }, + /// The finished words of a phrase. + Final { + text: String, + start_ms: u64, + end_ms: u64, + behind_ms: u64, + }, + /// The session ended on its own, with why. + Ended { error: Option }, +} + +#[derive(Serialize, Clone)] +#[serde(rename_all = "camelCase")] +struct Message<'a> { + session: &'a str, + update: &'a StreamUpdate, +} + +struct Session { + purpose: Purpose, + stop: Arc, +} + +static SESSIONS: LazyLock>> = LazyLock::new(Default::default); +static NEXT: AtomicU64 = AtomicU64::new(1); + +fn lock() -> std::sync::MutexGuard<'static, HashMap> { + SESSIONS.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Starts listening and returns the session's ID. Needs transcription turned on and a downloaded speech model. +#[tauri::command] +pub async fn intel_stream_start( + app: AppHandle, + state: State<'_, IntelState>, + request: StreamRequest, +) -> IpcResult { + if request.vocabulary.len() > MAX_VOCABULARY_BYTES { + return Err(IpcError::invalid("vocabulary", "The vocabulary list is too long.")); + } + let language = request.language.clone().filter(|tag| !tag.is_empty()); + if let Some(tag) = language.as_deref() { + opennote_intel::wire::check_language(tag).map_err(|error| super::ipc_error(&error))?; + } + state + .engines() + .require(Feature::Transcription) + .map_err(|error| super::ipc_error(&error))?; + let model = speech_model_path(state.device(), &request.model).map_err(|error| super::ipc_error(&error))?; + state.use_speech_model(&model); + let id = { + let mut sessions = lock(); + if sessions.values().any(|s| s.purpose == request.purpose) { + return Err(IpcError::new(codes::INVALID, "That is already listening.")); + } + if sessions.len() >= MAX_SESSIONS { + return Err(IpcError::new(codes::INVALID, "Too many things are listening.")); + } + let id = format!("s{}", NEXT.fetch_add(1, Ordering::Relaxed)); + let stop = Arc::new(AtomicBool::new(false)); + sessions.insert( + id.clone(), + Session { + purpose: request.purpose, + stop: Arc::clone(&stop), + }, + ); + (id, stop) + }; + let (id, stop) = id; + let (state, key) = (state.inner().clone(), id.clone()); + let prompt = opennote_intel::wire::vocabulary_prompt(&request.vocabulary); + let purpose = request.purpose; + let spawned = std::thread::Builder::new() + .name("opennote-stream".to_owned()) + .spawn(move || { + let send = |update: &StreamUpdate| { + let _ = app.emit(EVENT, Message { session: &key, update }); + }; + let outcome = run(&state, language, prompt, purpose, &stop, &send); + lock().remove(&key); + send(&StreamUpdate::Ended { + error: outcome.err().map(|error| error.to_string()), + }); + }); + if let Err(error) = spawned { + lock().remove(&id); + return Err(IpcError::new(codes::INTERNAL, error.to_string())); + } + Ok(id) +} + +/// Stops a session and closes the microphone. Resolves false when the session was already over. +#[tauri::command] +pub async fn intel_stream_stop(session: String) -> IpcResult { + match lock().get(&session) { + Some(found) => { + found.stop.store(true, Ordering::Relaxed); + Ok(true) + } + None => Ok(false), + } +} + +#[cfg(windows)] +fn run( + state: &IntelState, + language: Option, + prompt: String, + purpose: Purpose, + stop: &AtomicBool, + send: &dyn Fn(&StreamUpdate), +) -> Result<(), IntelError> { + use opennote_media::audio::{ + device::CpalSource, + resample::Resampler, + ring::{packet_channel, SinkHandle}, + AudioSource, + }; + use std::time::{Duration, Instant}; + + let options = match purpose { + Purpose::Dictation => StreamOptions::DICTATION, + Purpose::Captions => StreamOptions::CAPTIONS, + }; + // Load the model before the microphone opens, so a missing or broken model never listens. + let mut streamer = state.engines().live_stream(language.as_deref(), prompt, options)?; + let mut source = CpalSource::microphone().map_err(|error| IntelError::Audio(error.to_string()))?; + let format = source.format(); + let (sink, mut reader) = packet_channel( + format, + usize::try_from(format.rate).unwrap_or(48_000) * 2, + Arc::default(), + ); + source + .start(SinkHandle::new(sink)) + .map_err(|error| IntelError::Audio(error.to_string()))?; + send(&StreamUpdate::Listening); + let (mut resampler, mut rate) = (Resampler::new(format.rate), format.rate); + let (mut packet, mut converted) = (Vec::new(), Vec::new()); + let started = Instant::now(); + let mut first_capture: Option = None; + let result = (|| -> Result<(), IntelError> { + while !stop.load(Ordering::Relaxed) { + // Turning transcription off, or safe mode, ends the session at once. + if !state.choices().transcription { + return Ok(()); + } + let Some(info) = reader.next(&mut packet) else { + std::thread::sleep(Duration::from_millis(10)); + continue; + }; + if info.rate != rate { + resampler = Resampler::new(info.rate); + rate = info.rate; + } + let base = *first_capture.get_or_insert(info.capture_ns); + resampler.process(&packet, &mut converted); + // How far the words trail the sound: the time since this packet was heard, less the time since it began. + let behind = || { + let heard_at = Duration::from_nanos(info.capture_ns.saturating_sub(base)); + u64::try_from(started.elapsed().saturating_sub(heard_at).as_millis()).unwrap_or(u64::MAX) + }; + streamer.push_48k(&converted, &mut |event| { + send(&match event { + StreamEvent::Speaking => StreamUpdate::Speaking, + StreamEvent::Partial { text } => StreamUpdate::Partial { + text, + behind_ms: behind(), + }, + StreamEvent::Final { text, start_ms, end_ms } => StreamUpdate::Final { + text, + start_ms, + end_ms, + behind_ms: behind(), + }, + }); + })?; + } + streamer.finish(&mut |event| { + if let StreamEvent::Final { text, start_ms, end_ms } = event { + send(&StreamUpdate::Final { + text, + start_ms, + end_ms, + behind_ms: 0, + }); + } + }) + })(); + source.stop(); + result +} + +#[cfg(not(windows))] +fn run( + _state: &IntelState, + _language: Option, + _prompt: String, + _purpose: Purpose, + _stop: &AtomicBool, + _send: &dyn Fn(&StreamUpdate), +) -> Result<(), IntelError> { + Err(IntelError::Unsupported { feature: "listening" }) +} diff --git a/app/src-tauri/src/intel/tests.rs b/app/src-tauri/src/intel/tests.rs new file mode 100644 index 00000000..c829f9ad --- /dev/null +++ b/app/src-tauri/src/intel/tests.rs @@ -0,0 +1,129 @@ +use opennote_intel::{ + wire::{OcrRequest, ReadAloudRequest, SummarizeRequest}, + Feature, IntelError, +}; +use serde_json::json; + +use super::{ipc_error, IntelSettingsPatch, IntelState, FILE_NAME}; + +fn state_in(folder: &tempfile::TempDir) -> IntelState { + IntelState::at(folder.path().join(FILE_NAME)) +} + +fn turn_on(state: &IntelState, patch: serde_json::Value) { + let patch: IntelSettingsPatch = serde_json::from_value(patch).expect("a settings patch"); + state.update(&patch).expect("the settings are saved"); +} + +fn summary_request() -> SummarizeRequest { + serde_json::from_value(json!({ + "text": "Cells make energy. The mitochondria is the powerhouse of the cell. Plants also use chloroplasts.", + "options": { "maxSentences": 1 }, + })) + .expect("a summarize request") +} + +#[test] +fn every_feature_starts_off() { + let folder = tempfile::tempdir().unwrap(); + let state = state_in(&folder); + assert!(Feature::ALL.iter().all(|&feature| !state.settings().is_on(feature))); + let status = state.status(); + assert_eq!(status.len(), Feature::ALL.len()); + assert!(status.iter().all(|one| !one.enabled)); +} + +#[test] +fn a_feature_that_is_off_refuses_before_it_does_any_work() { + let folder = tempfile::tempdir().unwrap(); + let state = state_in(&folder); + assert_eq!( + state.summarize(&summary_request()).unwrap_err(), + IntelError::Disabled(Feature::Summaries) + ); + let image: OcrRequest = serde_json::from_value(json!({ + "source": { "kind": "pixels", "width": 1, "height": 1, "format": "gray8", "pixels": "AA==" }, + })) + .unwrap(); + assert_eq!( + state.ocr_recognize(image).unwrap_err(), + IntelError::Disabled(Feature::Ocr) + ); + let read: ReadAloudRequest = serde_json::from_value(json!({ "text": "Hello there." })).unwrap(); + assert_eq!( + state.read_aloud_start(read).unwrap_err(), + IntelError::Disabled(Feature::ReadAloud) + ); + assert_eq!(state.voices().unwrap_err(), IntelError::Disabled(Feature::ReadAloud)); +} + +#[test] +fn the_error_the_interface_gets_carries_the_feature_to_offer() { + let error = ipc_error(&IntelError::Disabled(Feature::ReadAloud)); + assert_eq!( + (error.code.as_str(), error.field.as_deref()), + ("disabled", Some("readAloud")) + ); + let error = ipc_error(&IntelError::VoiceUnavailable); + assert_eq!((error.code.as_str(), error.field), ("voiceUnavailable", None)); +} + +#[test] +fn turning_a_feature_on_saves_it_and_a_restart_keeps_it() { + let folder = tempfile::tempdir().unwrap(); + let state = state_in(&folder); + turn_on(&state, json!({ "summaries": true })); + let again = state_in(&folder); + assert!(again.settings().summaries); + assert!(!again.settings().ocr); +} + +#[test] +fn a_summary_runs_only_while_the_feature_is_on() { + let folder = tempfile::tempdir().unwrap(); + let state = state_in(&folder); + // Engines exist from the first request, so turning the feature on and off reaches ones already made. + assert!(state.summarize(&summary_request()).is_err()); + turn_on(&state, json!({ "summaries": true })); + let summary = state.summarize(&summary_request()).expect("a summary"); + assert_eq!(summary.sentences.len(), 1); + turn_on(&state, json!({ "summaries": false })); + assert_eq!( + state.summarize(&summary_request()).unwrap_err(), + IntelError::Disabled(Feature::Summaries) + ); +} + +#[test] +fn a_patch_changes_only_the_features_it_names() { + let folder = tempfile::tempdir().unwrap(); + let state = state_in(&folder); + turn_on(&state, json!({ "ocr": true, "readAloud": true })); + turn_on(&state, json!({ "ocr": false })); + let settings = state.settings(); + assert!(!settings.ocr && settings.read_aloud && !settings.summaries); +} + +#[test] +fn a_patch_that_changes_nothing_writes_nothing() { + let folder = tempfile::tempdir().unwrap(); + let state = state_in(&folder); + turn_on(&state, json!({})); + assert!(!folder.path().join(FILE_NAME).exists()); +} + +#[test] +fn a_damaged_file_leaves_everything_off() { + let folder = tempfile::tempdir().unwrap(); + std::fs::write(folder.path().join(FILE_NAME), "{ not json").unwrap(); + let state = state_in(&folder); + assert_eq!(state.settings(), opennote_intel::IntelSettings::default()); +} + +#[test] +fn an_unknown_session_is_an_error_not_a_hang() { + let folder = tempfile::tempdir().unwrap(); + let state = state_in(&folder); + assert!(state.read_aloud_next("session-404").is_err()); + assert!(state.clip_audio("clip-404").is_err()); +} diff --git a/app/src-tauri/src/intel/transcribe.rs b/app/src-tauri/src/intel/transcribe.rs new file mode 100644 index 00000000..c64ad454 --- /dev/null +++ b/app/src-tauri/src/intel/transcribe.rs @@ -0,0 +1,234 @@ +//! Transcribing a recording on this device (Phase 12, A1-3): the interface names the page's recording and a +//! downloaded speech model, and the job runs in the background, one at a time. Each step of a job reaches the +//! interface as an `intel-transcribe` event with the job's ID: started, progress, each line heard, then done with +//! the whole transcript, failed, or canceled. The audio is read from the page's own files and never copied. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use opennote_intel::{ + wire::{Pcm48k, TranscribeChoices, TranscribeHub, TranscribeUpdate}, + IntelError, +}; +use opennote_media::{ + layout::RecordingEntry, + playback::open_recording, + transcribe::{MixedPcm, PcmSource}, +}; +use serde::{Deserialize, Serialize}; +use tauri::{AppHandle, Emitter, Manager, State}; + +use super::{models, IntelState}; +use crate::{ + audio::{checked_dir, AudioState}, + core_bridge::CoreBridge, + ipc::{codes, IpcError, IpcResult}, +}; + +/// The event every job update arrives as. +pub const EVENT: &str = "intel-transcribe"; + +/// A job's update, as the interface receives it. +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JobUpdate<'a> { + pub job: &'a str, + pub update: &'a TranscribeUpdate, +} + +/// What to transcribe. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TranscribeRequest { + /// The page's `assets` folder, which holds the recording's files. + pub assets_dir: String, + /// The recording, with the tracks to read: the original's, or the enhanced copy's. + pub entry: RecordingEntry, + /// The catalog ID of the downloaded speech model to use. + pub model: String, + #[serde(default)] + pub choices: TranscribeChoices, + /// `cloud` sends the audio to the service the person gave a key for (A1-33). Anything else runs on this device. + #[serde(default)] + pub engine: Option, +} + +/// The cloud jobs still running, by ID, with the flag that stops each. +static CLOUD_JOBS: std::sync::LazyLock< + std::sync::Mutex>>, +> = std::sync::LazyLock::new(Default::default); +static CLOUD_NEXT: AtomicU64 = AtomicU64::new(1); + +/// Runs a job on the cloud service with the person's key, in the background, with the same updates as a job on this +/// device. Transcription must be on, as for the on-device engine. +fn start_cloud(app: &AppHandle, state: &IntelState, dir: &Path, request: &TranscribeRequest) -> IpcResult { + state + .engines() + .require(opennote_intel::Feature::Transcription) + .map_err(|error| super::ipc_error(&error))?; + if let Some(tag) = request.choices.language.as_deref().filter(|tag| !tag.is_empty()) { + opennote_intel::wire::check_language(tag).map_err(|error| super::ipc_error(&error))?; + } + let mut audio = open_audio(app, dir, &request.entry)?; + let id = format!("c{}", CLOUD_NEXT.fetch_add(1, Ordering::Relaxed)); + let stop = std::sync::Arc::new(AtomicBool::new(false)); + CLOUD_JOBS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(id.clone(), std::sync::Arc::clone(&stop)); + let cloud = state.cloud().clone(); + let (app, key, choices) = (app.clone(), id.clone(), request.choices.clone()); + std::thread::Builder::new() + .name("opennote-cloud-transcribe".to_owned()) + .spawn(move || { + let send = |update: &TranscribeUpdate| { + let _ = app.emit(EVENT, JobUpdate { job: &key, update }); + }; + send(&TranscribeUpdate::Started { + device: opennote_intel::wire::Device::Cpu, + }); + let total = audio.samples_total(); + let mut read = |out: &mut [f32]| audio.read_samples(out).map_err(|error| error.to_string()); + let result = cloud.transcribe( + &mut read, + total, + choices.language.as_deref().filter(|tag| !tag.is_empty()), + &opennote_intel::wire::vocabulary_prompt(&choices.vocabulary), + &crate::hardening::offline, + &crate::hardening::safe_mode, + &|fraction| send(&TranscribeUpdate::Progress { fraction }), + &|| stop.load(Ordering::Relaxed), + ); + let update = match result { + Ok(lines) => TranscribeUpdate::Done { + language: choices.language.clone().filter(|tag| !tag.is_empty()), + lines: opennote_intel::wire::corrected_lines(&choices.vocabulary, lines), + }, + Err(super::cloud::CloudError::Canceled) => TranscribeUpdate::Canceled, + Err(error) => TranscribeUpdate::Failed { + error: opennote_intel::ErrorInfo { + code: error.code().to_owned(), + message: error.message().to_owned(), + feature: None, + }, + }, + }; + CLOUD_JOBS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&key); + send(&update); + }) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))?; + Ok(id) +} + +/// The most vocabulary text a request may carry, as the crate's list limit allows. +const MAX_VOCABULARY_BYTES: usize = 512 * 1024; + +/// The file of an installed speech model by catalog ID, or why it can't be used. +pub fn speech_model_path(device: &Path, id: &str) -> Result { + let spec = models::spec_of(id) + .filter(|spec| spec.kind == "speech") + .ok_or_else(|| IntelError::InvalidInput("that isn't a speech model OpenNote offers".to_owned()))?; + let path = device.join("models").join(spec.main().file); + let complete = models::is_installed(&device.join("models"), spec); + if complete { + Ok(path) + } else { + Err(IntelError::ModelMissing { + model: spec.name.to_owned(), + }) + } +} + +/// The first installed speech model, in the order of preference: the balanced English model, then the others. +pub fn installed_speech_model(device: &Path) -> Option { + const PREFERENCE: [&str; 5] = [ + "speech-base-en", + "speech-small-en", + "speech-tiny-en", + "speech-base", + "speech-small", + ]; + PREFERENCE.iter().find_map(|id| speech_model_path(device, id).ok()) +} + +fn hub<'a>(state: &'a IntelState, app: &AppHandle) -> &'a TranscribeHub { + state.0.transcribe.get_or_init(|| { + let app = app.clone(); + TranscribeHub::new(move |job, update| { + // A closed window can't receive it; the job's end is in the interface's own state then. + let _ = app.emit(EVENT, JobUpdate { job, update }); + }) + }) +} + +fn open_audio(app: &AppHandle, dir: &Path, entry: &RecordingEntry) -> IpcResult { + let audio = app.state::(); + let summary = entry.summary().map_err(crate::audio::audio_error)?; + let decoder = audio.services().decoder.clone(); + let player = open_recording(dir, &summary, &decoder).map_err(crate::audio::audio_error)?; + let pcm = MixedPcm::new(player); + let total = pcm.total(); + let (dir, summary) = (dir.to_path_buf(), summary.clone()); + let shared = std::sync::Arc::new(std::sync::Mutex::new(pcm)); + let reader = std::sync::Arc::clone(&shared); + Ok(Pcm48k::new(total, move |out| { + reader + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .read(out) + .map_err(|error| error.to_string()) + }) + .with_rewind(move || { + let player = open_recording(&dir, &summary, &decoder).map_err(|error| error.to_string())?; + *shared.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = MixedPcm::new(player); + Ok(()) + })) +} + +/// Starts transcribing a recording and returns the job's ID. Updates follow as `intel-transcribe` events. +#[tauri::command] +pub async fn intel_transcribe( + app: AppHandle, + state: State<'_, IntelState>, + bridge: State<'_, CoreBridge>, + request: TranscribeRequest, +) -> IpcResult { + if request.choices.vocabulary.len() > MAX_VOCABULARY_BYTES { + return Err(IpcError::invalid("choices", "The vocabulary list is too long.")); + } + let (dir, _) = bridge.with(|bridge| checked_dir(bridge, &request.assets_dir))?; + let state = state.inner().clone(); + let work = move || -> IpcResult { + if request.engine.as_deref() == Some("cloud") { + return start_cloud(&app, &state, &dir, &request); + } + let model = speech_model_path(state.device(), &request.model).map_err(|error| super::ipc_error(&error))?; + state.use_speech_model(&model); + let audio = open_audio(&app, &dir, &request.entry)?; + let engines = state.engines(); + hub(&state, &app) + .submit(&engines, &model, &request.choices, audio, None) + .map_err(|error| super::ipc_error(&error)) + }; + match tauri::async_runtime::spawn_blocking(work).await { + Ok(result) => result, + Err(error) => Err(IpcError::new(codes::INTERNAL, error.to_string())), + } +} + +/// Cancels a transcription job, queued, or running. Its `canceled` update follows. +#[tauri::command] +pub async fn intel_transcribe_cancel(app: AppHandle, state: State<'_, IntelState>, job: String) -> IpcResult { + if let Some(stop) = CLOUD_JOBS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&job) + { + stop.store(true, Ordering::Relaxed); + return Ok(true); + } + Ok(hub(state.inner(), &app).cancel(&job)) +} diff --git a/app/src-tauri/src/interop/commands.rs b/app/src-tauri/src/interop/commands.rs new file mode 100644 index 00000000..9c1e25f0 --- /dev/null +++ b/app/src-tauri/src/interop/commands.rs @@ -0,0 +1,603 @@ +//! The commands behind the import and export dialogs (Phase 11; interop crate README, "What the interface and +//! the app wiring need"). The long ones take a job name from the interface, send `interop://progress` events +//! under it, and stop when `interop_cancel` names it. A canceled job answers `{ status: "canceled" }` and leaves +//! nothing behind. + +use std::{ + path::{Path, PathBuf}, + sync::{Arc, Mutex, PoisonError}, + time::Duration, +}; + +use opennote_core::{model::DeviceRef, SystemClock}; +use opennote_interop::{ + Control, Detected, DiskSink, Exported, ImportEnv, ImportOptions, InteropError, LossGroup, Preview, PreviewSection, + Report, WordPages, +}; +use serde::{Deserialize, Serialize}; +use tauri::{AppHandle, Emitter, State, WebviewWindow}; + +use super::{ + export::{self, ExportRequest, TreeSource}, + grants, + jobs::{self, Emit, Job}, + pick::{self, PickKind}, + restore::{self, ImportedTree}, +}; +use crate::{ + core_bridge::{notes_folder, run_notes_named, CoreBridge}, + ipc::{codes, IpcError, IpcResult}, +}; + +/// How long saving the open pages may take before an export reads their files. +const SAVE_OPEN_PAGES: Duration = Duration::from_secs(5); + +/// How a job ended. A canceled job is not an error. +#[derive(Debug, Serialize)] +#[serde(tag = "status", rename_all = "camelCase")] +pub enum Outcome { + Done { result: T }, + Canceled, +} + +/// What the person chose besides the source. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct ImportChoices { + /// Add an "Import report" page to the notebook. + pub report_page: bool, + /// How to cut Word files into pages: `auto`, `single`, `byTitle`, or `byPageBreak`. + pub word_pages: Option, + /// The password of a locked shared file (`.opennote`). It is never logged. + pub password: Option, +} + +impl ImportChoices { + pub(super) fn options(&self) -> ImportOptions { + let word_pages = match self.word_pages.as_deref() { + Some("single") => WordPages::Single, + Some("byTitle") => WordPages::ByTitle, + Some("byPageBreak") => WordPages::ByPageBreak, + _ => WordPages::Auto, + }; + ImportOptions { + kind: None, + word_pages, + report_page: self.report_page, + password: self + .password + .as_ref() + .and_then(super::export::Secret::get) + .filter(|p| p.chars().count() <= super::export::MAX_PASSWORD_CHARS) + .map(str::to_owned), + } + } +} + +fn internal(error: impl std::fmt::Display) -> IpcError { + IpcError::new(codes::INTERNAL, error.to_string()) +} + +/// An interop error with the code the interface reads. The message is for logs and for the "details" line. +fn failure(error: InteropError) -> IpcError { + let code = match &error { + InteropError::Unsupported { .. } => "unsupported", + InteropError::Format { .. } => "unreadable", + InteropError::Io { .. } => codes::IO, + InteropError::TooBig(_) => "tooBig", + InteropError::NoTables(_) => "noTables", + InteropError::Canceled => "canceled", + _ => codes::INTERNAL, + }; + IpcError::new(code, error.to_string()) +} + +/// Turns a finished job into the command's answer: canceled is an outcome, the rest of the errors are errors. +fn outcome(result: Result, done: impl FnOnce(T) -> IpcResult) -> IpcResult> { + match result { + Ok(value) => Ok(Outcome::Done { result: done(value)? }), + Err(InteropError::Canceled) => Ok(Outcome::Canceled), + Err(error) => Err(failure(error)), + } +} + +fn emitter(app: &AppHandle) -> Emit { + let app = app.clone(); + Arc::new(move |payload| { + if let Err(error) = app.emit(jobs::PROGRESS_EVENT, payload) { + ::log::warn!("Couldn't send a progress event to the interface: {error}"); + } + }) +} + +async fn blocking(work: impl FnOnce() -> T + Send + 'static) -> IpcResult { + tauri::async_runtime::spawn_blocking(work).await.map_err(internal) +} + +/// Opens the file or folder picker. Returns `None` when the person cancels. +#[tauri::command] +pub async fn interop_pick(window: WebviewWindow, kind: PickKind, initial: Option) -> IpcResult> { + // A window handle isn't `Send`, so it crosses to the dialog's thread as a number. + let owner = window.hwnd().map(|hwnd| hwnd.0 as isize).unwrap_or(0); + let picked = blocking(move || pick::pick(owner, kind, initial)).await??; + if let Some(path) = &picked { + grants::grant(path); + // A file the person picks may also open as a page that saves back (A3-18). + if kind == PickKind::File { + let _ = super::open_files::grant(Path::new(path)); + } + } + Ok(picked) +} + +/// What a file, folder, or archive is, and whether it can be imported. +#[tauri::command] +pub async fn interop_detect(path: String) -> IpcResult { + let path = grants::require(&path)?; + blocking(move || opennote_interop::detect(&path)) + .await? + .map_err(failure) +} + +/// Sources that live on this PC, offered without browsing. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LocalSources { + /// The Sticky Notes app's database, when this PC has one. + sticky_notes: Option, +} + +/// Looks for notes that other apps keep on this PC. +#[tauri::command] +pub async fn interop_local_sources() -> IpcResult { + blocking(|| LocalSources { + sticky_notes: opennote_interop::sticky_notes_database().map(|path| { + grants::grant(&path); + path.to_string_lossy().into_owned() + }), + }) + .await +} + +/// What a dry run found, for the review step. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PreviewView { + detected: Detected, + notebook_title: String, + sections: Vec, + pages: usize, + blocks: usize, + assets: usize, + asset_bytes: u64, + losses: Vec, + /// How many pages lose something. + lost_pages: usize, + /// How many parts do not come over at all. + skipped: usize, +} + +impl From for PreviewView { + fn from(preview: Preview) -> PreviewView { + let (lost_pages, skipped) = preview.report.loss_counts(); + PreviewView { + detected: preview.detected, + notebook_title: preview.notebook_title, + sections: preview.sections, + pages: preview.pages, + blocks: preview.blocks, + assets: preview.assets, + asset_bytes: preview.asset_bytes, + losses: preview.losses, + lost_pages, + skipped, + } + } +} + +/// A dry run: reads the source as an import would and writes nothing. +#[tauri::command] +pub async fn interop_preview( + app: AppHandle, + bridge: State<'_, CoreBridge>, + job: String, + path: String, + choices: Option, +) -> IpcResult> { + let path = grants::require(&path)?; + let device = bridge + .run(|bridge| bridge.with(|bridge| Ok(bridge.core.device()))) + .await?; + let (guard, control) = Job::start(&job, emitter(&app)); + let options = choices.unwrap_or_default().options(); + let result = blocking(move || { + let _guard = guard; + let clock = SystemClock::new(); + let env = ImportEnv::new(&clock, device).with_control(control); + opennote_interop::preview(&path, &options, &env) + }) + .await?; + outcome(result, |preview| Ok(PreviewView::from(preview))) +} + +/// What an import made. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportDone { + /// The new notebook, for the interface to build its tree from. + tree: ImportedTree, + pages: usize, + losses: Vec, + lost_pages: usize, + skipped: usize, +} + +/// Writes an import into a new notebook folder under `parent`, the notes folder. A canceled or failed import +/// leaves no folder. +pub(super) fn run_import( + parent: &Path, + source: &Path, + options: &ImportOptions, + device: DeviceRef, + control: Control, +) -> Result<(Report, PathBuf), InteropError> { + std::fs::create_dir_all(parent).map_err(|error| InteropError::io(parent, error))?; + let clock = SystemClock::new(); + let env = ImportEnv::new(&clock, device).with_control(control); + let mut sink = DiskSink::standard(parent); + let report = opennote_interop::import(source, options, &env, &mut sink)?; + let dir = sink + .notebook_dir() + .map(Path::to_path_buf) + .ok_or_else(|| InteropError::Missing("notebook folder after the import".to_owned()))?; + Ok((report, dir)) +} + +/// Imports one at a time: each import first removes the working folders crashed imports left, which would +/// otherwise remove the working folder of an import still running. +static IMPORTS: Mutex<()> = Mutex::new(()); + +/// Runs `work` as the only import, after cleaning up what crashed imports left in `folder`. +pub(super) fn import_alone(folder: &Path, work: impl FnOnce() -> T) -> T { + let _only = IMPORTS.lock().unwrap_or_else(PoisonError::into_inner); + restore::clean_staging(folder); + work() +} + +/// Imports into a new notebook folder in the notes folder, opens it in the core, and answers with its tree. The +/// notebook is in the notes tree when this answers, and the notes events have told the interface. +#[tauri::command] +pub async fn interop_import( + app: AppHandle, + bridge: State<'_, CoreBridge>, + job: String, + path: String, + choices: Option, +) -> IpcResult> { + let Some(parent) = notes_folder(&app) else { + return Err(IpcError::invalid("folder", "Choose a notes folder first.")); + }; + let path = grants::require(&path)?; + let device = bridge + .run(|bridge| bridge.with(|bridge| Ok(bridge.core.device()))) + .await?; + let (guard, control) = Job::start(&job, emitter(&app)); + let options = choices.unwrap_or_default().options(); + let folder = parent.clone(); + let result = blocking(move || { + let _guard = guard; + import_alone(&folder, || run_import(&folder, &path, &options, device, control)) + }) + .await?; + let (report, dir) = match result { + Ok(made) => made, + Err(InteropError::Canceled) => return Ok(Outcome::Canceled), + Err(error) => return Err(failure(error)), + }; + let tree = bridge + .run(move |bridge| { + run_notes_named(&app, bridge, "interop_import", |bridge| { + let handle = bridge.core.open_notebook(&dir).map_err(internal)?; + Ok(restore::tree_of(&handle)) + }) + }) + .await?; + let (lost_pages, skipped) = report.loss_counts(); + jobs::remember_report(&job, report.to_markdown()); + Ok(Outcome::Done { + result: ImportDone { + tree, + pages: report.pages.len(), + losses: report.loss_groups(), + lost_pages, + skipped, + }, + }) +} + +/// Asks a running job to stop. Does nothing when the job has finished. +#[tauri::command] +pub fn interop_cancel(job: String) { + jobs::cancel(&job); +} + +/// What an export made. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportDone { + /// The folder or file to show in Explorer. + pub(super) reveal: String, + pages: usize, + files: usize, + losses: Vec, + lost_pages: usize, + skipped: usize, +} + +impl From for ExportDone { + fn from(exported: Exported) -> ExportDone { + let (lost_pages, skipped) = exported.report.loss_counts(); + // One file exports show that file, which "Update the copy" also needs; folders show the new folder. The + // root of a one-file format is the chosen folder itself, so the number of files decides. + let reveal = match exported.files.as_slice() { + [file] => file.clone(), + _ => exported.root.clone(), + }; + ExportDone { + reveal: reveal.to_string_lossy().into_owned(), + pages: exported.report.pages.len(), + files: exported.files.len(), + losses: exported.report.loss_groups(), + lost_pages, + skipped, + } + } +} + +/// Exports a notebook or a section of the interface's tree into a folder. +#[tauri::command] +pub async fn interop_export( + app: AppHandle, + bridge: State<'_, CoreBridge>, + job: String, + request: ExportRequest, +) -> IpcResult> { + if !Path::new(&request.folder).is_dir() { + return Err(IpcError::invalid("folder", "Choose a folder that exists.")); + } + let (source, request) = bridge + .run(move |bridge| { + bridge.with(|bridge| { + // The export reads the page files, so the open pages are saved first. + bridge.core.flush_all(SAVE_OPEN_PAGES).map_err(internal)?; + let source = TreeSource::build(bridge, &request).map_err(failure)?; + Ok((source, request)) + }) + }) + .await?; + // The pages a PDF export prints arrived before it, under its job name. + let pdf = super::pdf_stage::take(&job); + let (guard, control) = Job::start(&job, emitter(&app)); + let result = blocking(move || { + let _guard = guard; + let what = format!("Export {}", request.title); + control.run( + &what, + |exported: &Exported| exported.report.pages.len() as u64, + |control| export::run(&source, &request, &pdf, control), + ) + }) + .await?; + outcome(result, |exported| { + let done = ExportDone::from(exported); + grants::grant(&done.reveal); + Ok(done) + }) +} + +/// Shows a file or folder in Explorer. +#[tauri::command] +pub async fn interop_reveal(path: String) -> IpcResult<()> { + let target = grants::require_within(&path)?; + if !target.is_absolute() || !target.exists() { + return Err(IpcError::invalid("path", "That file or folder isn't there.")); + } + reveal(&target) +} + +/// What Explorer is given to show a file: `/select,` and its path, so the file is selected in its folder. Explorer +/// given a file's path by itself opens the file as a double-click does, which for an attachment a note won't open +/// (F2-4: a program, a script, a type it doesn't know) is just what showing it in its folder must never do. A folder +/// is given as it is, and opens. Explorer reads the argument as one piece, so it is passed as it is. +#[cfg_attr(not(windows), allow(dead_code))] +fn explorer_select(target: &Path) -> Option { + (!target.is_dir()).then(|| format!("/select,\"{}\"", target.display())) +} + +#[cfg(windows)] +pub(crate) fn reveal(target: &Path) -> IpcResult<()> { + use std::os::windows::process::CommandExt; + use std::process::Command; + + let mut command = Command::new("explorer.exe"); + match explorer_select(target) { + Some(select) => command.raw_arg(select), + None => command.arg(target), + }; + command.spawn()?; + Ok(()) +} + +#[cfg(not(windows))] +pub(crate) fn reveal(_target: &Path) -> IpcResult<()> { + Err(IpcError::not_implemented("interop_reveal")) +} + +#[cfg(test)] +mod tests { + use opennote_interop::{Outcome as LossOutcome, ReportKind, SourceKind}; + use serde_json::json; + + use super::*; + use crate::interop::restore::{ImportedPageNode, ImportedSection}; + + #[test] + fn a_file_is_only_ever_selected_in_its_folder() { + // F2-4: an attachment a note won't open is shown in its folder. Were its path given to Explorer by itself, + // Explorer would open it, and a program or script would run. + let dir = tempfile::tempdir().expect("a temp folder"); + for name in ["Run me.bat", "Budget.unknown", "no extension", "a,b.cmd"] { + let file = dir.path().join(name); + std::fs::write(&file, b"x").expect("writes"); + assert_eq!( + explorer_select(&file), + Some(format!("/select,\"{}\"", file.display())), + "{name}" + ); + } + // A folder opens, even one named like a program. + let folder = dir.path().join("Setup.exe"); + std::fs::create_dir(&folder).expect("a folder"); + assert_eq!(explorer_select(&folder), None); + assert_eq!(explorer_select(dir.path()), None); + } + + #[test] + fn requests_from_the_interface_deserialize() { + let request: ExportRequest = serde_json::from_value(json!({ + "format": "htmlSingle", "scope": "page", "title": "Biology", + "sections": [{ "title": "Lectures", "pages": [{ "ui": "p-1", "title": "Mitosis", "level": 1 }] }], + "folder": "C:/Out", + })) + .expect("an export request"); + assert_eq!(request.sections[0].pages[0].ui, "p-1"); + let choices: ImportChoices = + serde_json::from_value(json!({ "reportPage": true, "wordPages": "byTitle" })).expect("import choices"); + assert!(choices.report_page); + assert!(matches!(choices.options().word_pages, WordPages::ByTitle)); + let none: ImportChoices = serde_json::from_value(json!({})).expect("empty choices"); + assert!(!none.report_page); + let kind: PickKind = serde_json::from_value(json!("folder")).expect("a pick kind"); + assert_eq!(kind, PickKind::Folder); + } + + fn loss() -> LossGroup { + LossGroup { + outcome: LossOutcome::Skipped, + why: "No reminders yet.".to_owned(), + examples: vec!["2 reminders".to_owned()], + pages: 2, + } + } + + #[test] + fn a_dry_run_answers_with_the_names_the_interface_reads() { + let mut preview = Preview { + detected: Detected { + needs_password: false, + kind: SourceKind::GoogleKeep, + label: "Google Keep export".to_owned(), + supported: true, + zipped: false, + advice: None, + legacy: false, + }, + notebook_title: "Keep".to_owned(), + sections: vec![PreviewSection { + title: "Notes".to_owned(), + pages: 4, + }], + pages: 4, + blocks: 9, + assets: 1, + asset_bytes: 2048, + losses: vec![loss()], + report: Report::new(ReportKind::Import, "Keep"), + }; + preview.report.general.skipped("2 reminders", "No reminders yet."); + let answer = serde_json::to_value(Outcome::Done { + result: PreviewView::from(preview), + }) + .expect("serializes"); + assert_eq!( + answer, + json!({ + "status": "done", + "result": { + "detected": { + "kind": "googleKeep", "label": "Google Keep export", "supported": true, + "zipped": false, "advice": null, "needsPassword": false, + "legacy": false + }, + "notebookTitle": "Keep", + "sections": [{ "title": "Notes", "pages": 4 }], + "pages": 4, "blocks": 9, "assets": 1, "assetBytes": 2048, + "losses": [{ "outcome": "skipped", "why": "No reminders yet.", "examples": ["2 reminders"], "pages": 2 }], + "lostPages": 0, "skipped": 1, + }, + }) + ); + } + + #[test] + fn an_import_answers_with_its_tree_and_a_canceled_job_with_only_its_status() { + let done = ImportDone { + tree: ImportedTree { + notebook_id: "n-1".to_owned(), + dir: "C:/Imported/Keep".to_owned(), + title: "Keep".to_owned(), + color: None, + sections: vec![ImportedSection { + id: "s-1".to_owned(), + title: "Notes".to_owned(), + color: Some("fern".to_owned()), + pages: vec![ImportedPageNode { + core: "c-1".to_owned(), + title: "Groceries".to_owned(), + level: 0, + }], + }], + }, + pages: 1, + losses: vec![], + lost_pages: 0, + skipped: 0, + }; + let answer = serde_json::to_value(Outcome::Done { result: done }).expect("serializes"); + assert_eq!( + answer["result"]["tree"], + json!({ + "notebookId": "n-1", "dir": "C:/Imported/Keep", "title": "Keep", "color": null, + "sections": [{ + "id": "s-1", "title": "Notes", "color": "fern", + "pages": [{ "core": "c-1", "title": "Groceries", "level": 0 }], + }], + }) + ); + assert_eq!(answer["result"]["lostPages"], 0); + assert_eq!( + serde_json::to_value(Outcome::::Canceled).expect("serializes"), + json!({ "status": "canceled" }) + ); + } + + #[test] + fn local_sources_use_the_names_the_interface_reads() { + let sources = LocalSources { + sticky_notes: Some("C:/Packages/plum.sqlite".to_owned()), + }; + assert_eq!( + serde_json::to_value(sources).expect("serializes"), + json!({ "stickyNotes": "C:/Packages/plum.sqlite" }) + ); + } + + #[test] + fn errors_carry_the_codes_the_interface_maps_to_sentences() { + let code = |error: InteropError| failure(error).code; + assert_eq!(code(InteropError::unsupported("a.one", "no")), "unsupported"); + assert_eq!(code(InteropError::format("a.enex", "bad")), "unreadable"); + assert_eq!(code(InteropError::TooBig("a.zip".to_owned())), "tooBig"); + assert_eq!(code(InteropError::Missing("page".to_owned())), codes::INTERNAL); + } +} diff --git a/app/src-tauri/src/interop/export.rs b/app/src-tauri/src/interop/export.rs new file mode 100644 index 00000000..ed366699 --- /dev/null +++ b/app/src-tauri/src/interop/export.rs @@ -0,0 +1,472 @@ +//! Exports: the interface's tree of a notebook or section, read from the pages of the core's notebooks. +//! +//! The interface sends the sections and pages it shows, and `TreeSource` answers the interop crate's `NoteSource` +//! from them: the tree gives the structure and the titles, and each page's files give its content. The page IDs +//! the interface sends are the core's, so each page is found in whichever open notebook holds it. A page that has +//! no files to read exports as an empty page with its title. + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Arc, +}; + +use opennote_core::{ + format::CanonicalCodec, + model::{Asset, FormatInfo, JsonMap, NotebookFile, Page, PageEntry, SectionFile}, + seams::Codec, + session::{notebook::NotebookHandle, page::read_page_dir}, + store::{ + assets::read_asset, + fs::Fs, + history::{list_versions, open_version}, + layout::NotebookLayout, + std_fs::StdFs, + PageFiles, + }, + Clock, Limits, NotebookId, OrderKey, PageId, SectionId, SystemClock, Timings, +}; +use opennote_interop::{ + page_builder::PageBuilder, Control, Exported, Format, ImportEnv, InteropError, NoteSource, PdfRenderer, Result, + Scope, TableFormat, +}; +use serde::Deserialize; + +use crate::core_bridge::Bridge; + +/// Which files an export writes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ExportFormat { + /// A folder of Markdown files. + Markdown, + /// A folder of web pages with an index. + Html, + /// One web page file. + HtmlSingle, + /// One Word file. + Docx, + /// A folder of PDF files, which the interface printed page by page, with an index. + Pdf, + /// One PowerPoint file with a slide for each page, or part of a page. + Pptx, + /// One Excel workbook with a sheet for each table. + Xlsx, + /// One CSV file for a table, or a folder of them. + Csv, + /// One `.opennote` file that opens in OpenNote as a new notebook (ADR 0035), with a password if one is given. + Share, +} + +/// A password the person typed. It is never printed: its debug form hides it, so a logged request can't show it. +#[derive(Clone, Deserialize)] +#[serde(transparent)] +pub struct Secret(String); + +impl Secret { + /// The password, or `None` when it is empty. + pub fn get(&self) -> Option<&str> { + Some(self.0.as_str()).filter(|p| !p.is_empty()) + } +} + +impl std::fmt::Debug for Secret { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Secret(..)") + } +} + +/// The longest password a shared file takes. +pub const MAX_PASSWORD_CHARS: usize = 1_024; + +/// The most earlier versions of each page a shared file holds. +const HISTORY_PER_PAGE: usize = 20; + +/// What the person exports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ExportScope { + Notebook, + Section, + /// One page, sent as the only page of the only section. + Page, +} + +/// A page of the interface's tree. +#[derive(Debug, Clone, Deserialize)] +pub struct ExportPage { + /// The page's ID in the tree, which is the core's. + pub ui: String, + pub title: String, + /// 0 for a page, 1 and 2 for subpages. + pub level: u8, +} + +/// A section of the interface's tree. +#[derive(Debug, Clone, Deserialize)] +pub struct ExportSection { + pub title: String, + pub pages: Vec, +} + +/// What the interface asks for. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportRequest { + pub format: ExportFormat, + pub scope: ExportScope, + /// The notebook's title. + pub title: String, + /// All the notebook's sections, or just the one that is exported. + pub sections: Vec, + /// The folder the export goes into. + pub folder: String, + /// A file sent earlier that this export replaces ("Update the copy"). Only the formats that make one file. + #[serde(default)] + pub replace: Option, + /// Share as a file: the password that locks the file. + #[serde(default)] + pub password: Option, + /// Share as a file: put each page's earlier versions in the file, as pages of a "Page history" section. + #[serde(default)] + pub history: bool, +} + +/// The notebook, sections, and pages of an export, read from the bridge's page files. +pub struct TreeSource { + fs: Arc, + codec: Arc, + limits: Limits, + notebook: NotebookFile, + sections: Vec, + /// The folder of each page that has files. + dirs: HashMap, + /// Pages with no files yet. + empty: HashMap, + /// The title the interface shows for each page. It wins over the title inside the page. + titles: HashMap, + /// Earlier versions of pages, under IDs of their own. Their assets are read from `dirs`. + versions: HashMap, +} + +/// The folder of every page in a notebook. +fn page_dirs(handle: &NotebookHandle) -> HashMap { + let layout = NotebookLayout::new(handle.path()); + let mut dirs = HashMap::new(); + for section in handle.tree().sections { + for page in section.pages { + dirs.insert(page.id, layout.page_dir(section.id, page.id)); + } + } + dirs +} + +fn invalid(what: &str, error: impl std::fmt::Display) -> InteropError { + InteropError::format(what, error.to_string()) +} + +impl TreeSource { + /// Reads the tree the interface sent. Call after the core has saved its open pages. + pub fn build(bridge: &Bridge, request: &ExportRequest) -> Result { + let clock = SystemClock::new(); + let now = clock.now(); + let env = ImportEnv::new(&clock, bridge.core.device()); + let fs: Arc = Arc::new(StdFs::new(&Timings::default())); + let mut notebook_dirs: HashMap> = HashMap::new(); + let mut source = TreeSource { + fs, + codec: Arc::new(CanonicalCodec), + limits: Limits::default(), + notebook: NotebookFile::new(NotebookId::generate(&clock), request.title.clone(), now), + sections: Vec::new(), + dirs: HashMap::new(), + empty: HashMap::new(), + titles: HashMap::new(), + versions: HashMap::new(), + }; + let orders = OrderKey::spread(None, None, request.sections.len()).map_err(|e| invalid("section order", e))?; + for (section, order) in request.sections.iter().zip(orders) { + let page_orders = + OrderKey::spread(None, None, section.pages.len()).map_err(|e| invalid("page order", e))?; + let mut entries = Vec::with_capacity(section.pages.len()); + // The last page at each level, so a subpage finds its parent. + let mut last: [Option; 3] = [None; 3]; + let mut previous_level = 0u8; + for (page, page_order) in section.pages.iter().zip(page_orders) { + let level = page.level.min(2).min(previous_level.saturating_add(1)); + let known = PageId::parse(&page.ui).ok(); + let dir = known.and_then(|id| { + let (notebook, _) = bridge.core.find_node(id.0)?; + notebook_dirs + .entry(notebook.path().to_path_buf()) + .or_insert_with(|| page_dirs(¬ebook)) + .get(&id) + .cloned() + }); + let id = match (known, dir) { + (Some(id), Some(dir)) => { + source.dirs.insert(id, dir); + id + } + _ => { + let id = PageId::generate(&clock); + let built = PageBuilder::with_id(&env, id, &page.title, now, now).finish()?; + source.empty.insert(id, built.page); + id + } + }; + if !page.title.trim().is_empty() { + source.titles.insert(id, page.title.clone()); + } + let parent = if level == 0 { None } else { last[usize::from(level) - 1] }; + last[usize::from(level)] = Some(id); + previous_level = level; + entries.push(PageEntry { + id, + title: page.title.clone(), + parent, + order: page_order, + pinned: false, + color: None, + changed: now, + moving: None, + extra: JsonMap::new(), + }); + } + source.sections.push(SectionFile { + id: SectionId::generate(&clock), + title: section.title.clone(), + color: None, + group: None, + order, + created: now, + changed: now, + defaults: None, + encryption: None, + pages: entries, + extra: JsonMap::new(), + format: FormatInfo::default(), + }); + } + Ok(source) + } + + /// What the export covers: the first page or section that was sent, else the whole notebook. + pub fn scope(&self, scope: ExportScope) -> Scope { + match (scope, self.sections.first()) { + (ExportScope::Section, Some(section)) => Scope::Section(section.id), + (ExportScope::Page, Some(section)) => section + .pages + .first() + .map_or(Scope::Section(section.id), |entry| Scope::Page(entry.id)), + _ => Scope::Notebook, + } + } + + /// The earlier versions of the pages in `scope`, as one "Page history" section of a source of their own, each + /// titled with its page and the time it was saved. The newest come first, at most [`HISTORY_PER_PAGE`] of each + /// page, and the version that is the page as it is now is left out. `None` when no page has an earlier version. + pub fn history_source(&self, scope: Scope) -> Result> { + let clock = SystemClock::new(); + let now = clock.now(); + let mut history = TreeSource { + fs: Arc::clone(&self.fs), + codec: Arc::clone(&self.codec), + limits: self.limits.clone(), + notebook: self.notebook.clone(), + sections: Vec::new(), + dirs: HashMap::new(), + empty: HashMap::new(), + titles: HashMap::new(), + versions: HashMap::new(), + }; + let mut entries = Vec::new(); + for section in &self.sections { + if matches!(scope, Scope::Section(id) if id != section.id) { + continue; + } + for entry in §ion.pages { + if matches!(scope, Scope::Page(id) if id != entry.id) { + continue; + } + let Some(dir) = self.dirs.get(&entry.id) else { continue }; + for (id, page) in self.versions_of(entry, dir, &clock) { + history.dirs.insert(id, dir.clone()); + entries.push(PageEntry { + id, + title: page.title.clone(), + parent: None, + order: entry.order.clone(), + pinned: false, + color: None, + changed: now, + moving: None, + extra: JsonMap::new(), + }); + history.versions.insert(id, page); + } + } + } + if entries.is_empty() { + return Ok(None); + } + let orders = OrderKey::spread(None, None, entries.len() + 1).map_err(|e| invalid("page order", e))?; + let mut orders = orders.into_iter(); + for entry in &mut entries { + entry.order = orders.next().ok_or_else(|| invalid("page order", "too few keys"))?; + } + let order = orders.next().ok_or_else(|| invalid("section order", "too few keys"))?; + history.sections.push(SectionFile { + id: SectionId::generate(&clock), + title: "Page history".to_owned(), + color: None, + group: None, + order, + created: now, + changed: now, + defaults: None, + encryption: None, + pages: entries, + extra: JsonMap::new(), + format: FormatInfo::default(), + }); + Ok(Some(history)) + } + + /// A page's earlier versions, newest first, each under a new ID and titled with its page and time. History is + /// best effort: a list or a version that can't be read is left out. + fn versions_of(&self, entry: &PageEntry, dir: &Path, clock: &SystemClock) -> Vec<(PageId, Page)> { + let files = PageFiles { + fs: self.fs.as_ref(), + codec: self.codec.as_ref(), + dir, + }; + let Ok(list) = list_versions(&files, &self.limits) else { + return Vec::new(); + }; + let current = self.page(entry.id).ok().map(|p| p.revision.id); + let mut listed: Vec<_> = list + .versions + .into_iter() + .filter(|v| Some(v.revision) != current) + .collect(); + listed.sort_by_key(|v| std::cmp::Reverse(v.saved_at)); + let title = self.titles.get(&entry.id).unwrap_or(&entry.title); + let mut out = Vec::new(); + for version in listed.into_iter().take(HISTORY_PER_PAGE) { + let Ok(read) = open_version(&files, version.revision, &self.limits) else { + continue; + }; + let mut page = read.page; + let id = PageId::generate(clock); + let when = version.saved_at.to_rfc3339(); + let shown = when.get(..16).unwrap_or(&when).replace('T', " "); + page.id = id; + page.title = match &version.name { + Some(name) => format!("{title} ({name}, {shown})"), + None => format!("{title} ({shown})"), + }; + out.push((id, page)); + } + out + } +} + +impl NoteSource for TreeSource { + fn notebook(&self) -> &NotebookFile { + &self.notebook + } + + fn sections(&self) -> &[SectionFile] { + &self.sections + } + + fn page(&self, id: PageId) -> Result { + if let Some(version) = self.versions.get(&id) { + return Ok(version.clone()); + } + let mut page = match (self.dirs.get(&id), self.empty.get(&id)) { + (Some(dir), _) => read_page_dir(self.fs.as_ref(), self.codec.as_ref(), dir, &self.limits) + .map(|loaded| loaded.page) + .map_err(|e| invalid(&format!("page {id}"), format!("{e:?}")))?, + (None, Some(page)) => page.clone(), + (None, None) => return Err(InteropError::Missing(format!("page {id}"))), + }; + if let Some(title) = self.titles.get(&id) { + page.title.clone_from(title); + } + Ok(page) + } + + fn asset_bytes(&self, page: &Page, asset: &Asset) -> Result> { + let dir = self + .dirs + .get(&page.id) + .ok_or_else(|| InteropError::Missing(format!("files of page {}", page.id)))?; + read_asset(self.fs.as_ref(), dir, asset, None).map_err(|e| invalid("asset", e)) + } +} + +/// Runs the export the request names, into `folder`. With `replace`, the new file takes the place of that file. +/// A PDF export draws nothing here: it writes the pages that `pdf` holds, which the interface printed. +pub fn run(source: &TreeSource, request: &ExportRequest, pdf: &dyn PdfRenderer, control: &Control) -> Result { + let Some(target) = request.replace.as_deref().map(Path::new) else { + return run_into(source, request, pdf, Path::new(&request.folder), control); + }; + let parent = target + .parent() + .filter(|p| p.is_dir()) + .ok_or_else(|| InteropError::Missing("folder of the copy".to_owned()))?; + // The export writes into a folder of its own beside the copy, so the copy changes only when the export is whole. + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + let staging = parent.join(format!(".opennote-update-{}-{stamp}", std::process::id())); + std::fs::create_dir(&staging).map_err(|e| InteropError::io(&staging, e))?; + let result = run_into(source, request, pdf, &staging, control).and_then(|mut exported| { + let made = exported + .files + .first() + .cloned() + .ok_or_else(|| InteropError::Missing("file in the export".to_owned()))?; + std::fs::rename(&made, target).map_err(|e| InteropError::io(target, e))?; + exported.root = parent.to_path_buf(); + exported.files = vec![target.to_path_buf()]; + Ok(exported) + }); + let _ = std::fs::remove_dir_all(&staging); + result +} + +fn run_into( + source: &TreeSource, + request: &ExportRequest, + pdf: &dyn PdfRenderer, + folder: &Path, + control: &Control, +) -> Result { + let scope = source.scope(request.scope); + match request.format { + ExportFormat::Markdown => opennote_interop::export_files_with(source, scope, Format::Markdown, folder, control), + ExportFormat::Html => opennote_interop::export_files_with(source, scope, Format::Html, folder, control), + ExportFormat::HtmlSingle => opennote_interop::export_html_single(source, scope, folder, control), + ExportFormat::Docx => opennote_interop::export_docx_with(source, scope, folder, control), + ExportFormat::Pdf => opennote_interop::export_pdf_bundle(source, scope, pdf, folder, control), + ExportFormat::Pptx => opennote_interop::export_pptx(source, scope, folder, control), + ExportFormat::Xlsx => opennote_interop::export_tables(source, scope, TableFormat::Xlsx, folder, control), + ExportFormat::Csv => opennote_interop::export_tables(source, scope, TableFormat::Csv, folder, control), + ExportFormat::Share => { + let password = request.password.as_ref().and_then(Secret::get); + if password.is_some_and(|p| p.chars().count() > MAX_PASSWORD_CHARS) { + return Err(InteropError::unsupported("password", "That password is too long.")); + } + let history = if request.history { + source.history_source(scope)? + } else { + None + }; + let history = history.as_ref().map(|h| h as &dyn NoteSource); + opennote_interop::export_share_with(source, scope, password, history, folder, control) + } + } +} diff --git a/app/src-tauri/src/interop/grants.rs b/app/src-tauri/src/interop/grants.rs new file mode 100644 index 00000000..59c9cbd1 --- /dev/null +++ b/app/src-tauri/src/interop/grants.rs @@ -0,0 +1,96 @@ +//! The paths the person chose in this app's own picker, or that the app found or wrote itself. The import, report, +//! and reveal commands take only these, so a script that got into the WebView can't read or write anywhere else +//! on disk through them. + +use std::{ + collections::HashSet, + path::{Component, Path, PathBuf}, + sync::{Mutex, PoisonError}, +}; + +use crate::ipc::{IpcError, IpcResult}; + +static GRANTED: Mutex>> = Mutex::new(None); + +/// Lets the commands use `path` for the rest of this run. +pub fn grant(path: impl Into) { + GRANTED + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get_or_insert_with(HashSet::new) + .insert(path.into()); +} + +fn is_granted(path: &Path) -> bool { + GRANTED + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .is_some_and(|set| set.contains(path)) +} + +/// `path` itself was granted. +pub fn require(path: &str) -> IpcResult { + let path = PathBuf::from(path); + if is_granted(&path) { + Ok(path) + } else { + Err(IpcError::invalid( + "path", + "Choose the file or folder with Browse first.", + )) + } +} + +/// Whether `path` steps up or stays put anywhere (`..` or `.`), which `Path::ancestors` does not resolve, so +/// `\..\elsewhere` would look inside the granted folder. A verbatim (`\\?\`) path keeps them as +/// plain names, so those are refused too. +fn has_dot_steps(path: &Path) -> bool { + path.components().any(|c| match c { + Component::ParentDir | Component::CurDir => true, + Component::Normal(name) => name == ".." || name == ".", + _ => false, + }) +} + +/// `path` or a folder it is inside was granted, as for what an export wrote into a chosen folder. +pub fn require_within(path: &str) -> IpcResult { + let path = PathBuf::from(path); + if !has_dot_steps(&path) && path.ancestors().any(is_granted) { + Ok(path) + } else { + Err(IpcError::invalid( + "path", + "That file or folder isn't one OpenNote chose or wrote.", + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_granted_paths_and_what_is_inside_granted_folders_pass() { + let base = std::env::temp_dir().join("opennote-grants-test"); + assert!(require(&base.to_string_lossy()).is_err()); + grant(base.clone()); + assert!(require(&base.to_string_lossy()).is_ok()); + let inside = base.join("Export").join("page.md"); + assert!(require(&inside.to_string_lossy()).is_err()); + assert!(require_within(&inside.to_string_lossy()).is_ok()); + assert!(require_within(&std::env::temp_dir().join("elsewhere").to_string_lossy()).is_err()); + } + + /// `..` under a granted folder does not reach outside it (R3-4). + #[test] + fn parent_steps_under_a_granted_folder_are_refused() { + let base = std::env::temp_dir().join("opennote-grants-dots"); + grant(base.clone()); + let escape = base.join("..").join("..").join("secret.docx"); + assert!(require_within(&escape.to_string_lossy()).is_err()); + let verbatim = format!(r"\\?\{}\..\secret.docx", base.display()); + assert!(require_within(&verbatim).is_err()); + assert!(require_within(&base.join("page.md").to_string_lossy()).is_ok()); + } +} diff --git a/app/src-tauri/src/interop/jobs.rs b/app/src-tauri/src/interop/jobs.rs new file mode 100644 index 00000000..99bbcaca --- /dev/null +++ b/app/src-tauri/src/interop/jobs.rs @@ -0,0 +1,118 @@ +//! Running jobs: each import, dry run, and export has a name from the interface, a cancel token, and a stream of +//! progress events named `interop://progress`. + +use std::{ + collections::{HashMap, VecDeque}, + sync::{Arc, Mutex, OnceLock, PoisonError}, + time::{Duration, Instant}, +}; + +use opennote_interop::{CancelToken, Control, Event, ProgressSink}; +use serde_json::Value; + +/// The event the interface listens for: the job's name and one event of the interop crate (`started`, +/// `progress`, `finished`, `canceled`, or `failed`). +pub const PROGRESS_EVENT: &str = "interop://progress"; + +/// At most one progress event goes out in this time, so a thousand small pages don't flood the interface. +const PROGRESS_EVERY: Duration = Duration::from_millis(60); + +/// Sends one payload to the interface. +pub type Emit = Arc; + +fn registry() -> &'static Mutex> { + static JOBS: OnceLock>> = OnceLock::new(); + JOBS.get_or_init(Mutex::default) +} + +/// How many finished jobs keep their report for the Save report button. +const KEPT_REPORTS: usize = 8; + +fn reports() -> &'static Mutex> { + static REPORTS: OnceLock>> = OnceLock::new(); + REPORTS.get_or_init(Mutex::default) +} + +/// Keeps the Markdown report of a finished job, so the person can save it as a file afterwards. +pub fn remember_report(job: &str, markdown: String) { + let mut kept = reports().lock().unwrap_or_else(PoisonError::into_inner); + kept.retain(|(name, _)| name != job); + kept.push_back((job.to_owned(), markdown)); + while kept.len() > KEPT_REPORTS { + kept.pop_front(); + } +} + +/// The report a finished job kept. +pub fn report_for(job: &str) -> Option { + let kept = reports().lock().unwrap_or_else(PoisonError::into_inner); + kept.iter() + .find(|(name, _)| name == job) + .map(|(_, markdown)| markdown.clone()) +} + +/// A job that is running. It leaves the registry when it is dropped. +pub struct Job { + id: String, +} + +impl Job { + /// Registers the job and makes the control that its work reports through. + pub fn start(id: &str, emit: Emit) -> (Job, Control) { + let token = CancelToken::new(); + registry() + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(id.to_owned(), token.clone()); + let relay = Relay { + job: id.to_owned(), + emit, + last: Mutex::new(None), + }; + (Job { id: id.to_owned() }, Control::new(token, Arc::new(relay))) + } +} + +impl Drop for Job { + fn drop(&mut self) { + registry() + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&self.id); + } +} + +/// Asks the job to stop at its next checkpoint. False when no such job is running. +pub fn cancel(id: &str) -> bool { + match registry().lock().unwrap_or_else(PoisonError::into_inner).get(id) { + Some(token) => { + token.cancel(); + true + } + None => false, + } +} + +struct Relay { + job: String, + emit: Emit, + last: Mutex>, +} + +impl ProgressSink for Relay { + fn event(&self, event: &Event) { + if matches!(event, Event::Progress(_)) { + let mut last = self.last.lock().unwrap_or_else(PoisonError::into_inner); + let now = Instant::now(); + if last.is_some_and(|before| now.duration_since(before) < PROGRESS_EVERY) { + return; + } + *last = Some(now); + } + let Ok(mut payload) = serde_json::to_value(event) else { + return; + }; + payload["job"] = Value::String(self.job.clone()); + (self.emit)(payload); + } +} diff --git a/app/src-tauri/src/interop/mod.rs b/app/src-tauri/src/interop/mod.rs new file mode 100644 index 00000000..a59c6254 --- /dev/null +++ b/app/src-tauri/src/interop/mod.rs @@ -0,0 +1,22 @@ +//! Import and export for the interface (Phase 11). The interop crate reads and writes the formats. +//! This module is the app's side of it. It holds the file pickers, the commands behind the dialogs with their +//! progress and Cancel, and the link between the notebooks an import writes and the pages the interface shows. +//! +//! An import writes a real notebook folder (spec 3) into the notes folder that setup chose, and the core opens it +//! like any notebook there, so the notebook, its sections, and its pages are in the tree at once with the core's +//! IDs. The notes events tell the interface. +//! +//! An export goes the other way. The interface sends the tree it shows, and `TreeSource` reads each page's files +//! from the core's notebooks. + +pub mod commands; +mod export; +mod grants; +mod jobs; +pub mod more; +pub mod open_files; +mod pdf_stage; +pub mod pick; +mod restore; +#[cfg(test)] +mod tests; diff --git a/app/src-tauri/src/interop/more.rs b/app/src-tauri/src/interop/more.rs new file mode 100644 index 00000000..5ab66d37 --- /dev/null +++ b/app/src-tauri/src/interop/more.rs @@ -0,0 +1,287 @@ +//! The extra interop operations behind one command, `interop_more`. The interface names an operation and sends its +//! arguments as JSON. One command keeps the list of Tauri commands, the permissions, and the handler list short. +//! The import and export area keeps growing: saving an import report, sending a copy to a folder, sharing as a +//! file, and the rest. Each adds a match arm and a function here. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use tauri::{AppHandle, Manager}; + +use super::{grants, jobs}; +use crate::ipc::{IpcError, IpcResult}; + +/// Runs the operation `op` with its arguments. +#[tauri::command] +pub async fn interop_more(app: AppHandle, op: String, args: Value) -> IpcResult { + let data = app.path().app_data_dir().unwrap_or_else(|_| std::env::temp_dir()); + tauri::async_runtime::spawn_blocking(move || dispatch(&op, args, &data)) + .await + .map_err(|e| IpcError::invalid("op", &e.to_string()))? +} + +fn dispatch(op: &str, args: Value, data: &Path) -> IpcResult { + match op { + "save_report" => save_report(parse(args)?), + "save_text" => save_text(parse(args)?), + "send_state" => send::state(data, parse(args)?), + "send_favorite" => send::favorite(data, parse(args)?), + "send_record" => send::record(data, parse(args)?), + "pdf_stage" => super::pdf_stage::stage(parse(args)?), + "open_launch" => super::open_files::take_launch(data), + "open_picked" => super::open_files::open_picked(parse(args)?), + "open_read" => super::open_files::read(parse(args)?), + "open_write" => super::open_files::write(parse(args)?), + "open_stat" => super::open_files::stat(parse(args)?), + "open_list" => super::open_files::list(data), + "open_link" => super::open_files::link(data, parse(args)?), + "open_default_apps" => super::open_files::default_apps(), + "pdf_unstage" => super::pdf_stage::unstage(parse(args)?), + other => Err(IpcError::invalid( + "op", + &format!("There is no operation called {other}."), + )), + } +} + +/// Reads the arguments of an operation. +pub(super) fn parse Deserialize<'de>>(args: Value) -> IpcResult { + serde_json::from_value(args).map_err(|e| IpcError::invalid("args", &e.to_string())) +} + +/// The first name like `Name.ext`, `Name (2).ext`, and so on that does not exist in `folder`. +pub(super) fn unique_path(folder: &Path, stem: &str, ext: &str) -> PathBuf { + let mut path = folder.join(format!("{stem}.{ext}")); + let mut n = 2; + while path.exists() { + path = folder.join(format!("{stem} ({n}).{ext}")); + n += 1; + } + path +} + +#[derive(Deserialize)] +struct SaveReport { + job: String, + folder: String, +} + +/// Writes the Markdown report of a finished import into a folder the person chose. +fn save_report(args: SaveReport) -> IpcResult { + let folder = grants::require(&args.folder)?; + if !folder.is_dir() { + return Err(IpcError::invalid("folder", "Choose a folder that exists.")); + } + let markdown = jobs::report_for(&args.job) + .ok_or_else(|| IpcError::invalid("job", "That import has no report to save any more."))?; + let path = unique_path(&folder, "OpenNote import report", "md"); + std::fs::write(&path, markdown) + .map_err(|e| IpcError::invalid("folder", &format!("The report could not be saved: {e}")))?; + grants::grant(&path); + Ok(json!({ "path": path.to_string_lossy() })) +} + +/// The biggest Markdown file `save_text` writes. +const MAX_SAVED_TEXT: usize = 8 << 20; + +#[derive(Deserialize)] +struct SaveText { + folder: String, + name: String, + text: String, +} + +/// A file name stem made safe: letters, digits, spaces, and a few marks, at most 80 characters. +fn safe_stem(name: &str) -> String { + let kept: String = name + .chars() + .filter(|c| c.is_alphanumeric() || matches!(c, ' ' | '-' | '_' | '(' | ')' | ',' | '.')) + .take(80) + .collect(); + let trimmed = kept.trim().trim_matches('.').trim(); + if trimmed.is_empty() { + "OpenNote".to_owned() + } else { + trimmed.to_owned() + } +} + +/// Writes Markdown, such as a PDF's highlights, as a new file in a folder the person chose. It never replaces a file. +fn save_text(args: SaveText) -> IpcResult { + let folder = Path::new(&args.folder); + if !folder.is_absolute() || !folder.is_dir() { + return Err(IpcError::invalid("folder", "Choose a folder that exists.")); + } + if args.text.len() > MAX_SAVED_TEXT { + return Err(IpcError::invalid("text", "The text is too long to save.")); + } + let path = unique_path(folder, &safe_stem(&args.name), "md"); + std::fs::write(&path, args.text) + .map_err(|e| IpcError::invalid("folder", &format!("The file could not be saved: {e}")))?; + Ok(json!({ "path": path.to_string_lossy() })) +} + +/// The folders a person sends copies to, and the copies already sent, for "Update the copy". +mod send { + use super::*; + + /// What is kept in `send_to.json`. + #[derive(Default, Serialize, Deserialize)] + struct Store { + favorites: Vec, + copies: Vec, + } + + #[derive(Clone, Serialize, Deserialize)] + pub struct Copy { + /// The node of the interface's tree that was sent. + node: String, + format: String, + path: String, + } + + fn file(data: &Path) -> PathBuf { + data.join("send_to.json") + } + + fn load(data: &Path) -> Store { + std::fs::read(file(data)) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_default() + } + + fn save(data: &Path, store: &Store) -> IpcResult<()> { + std::fs::create_dir_all(data).map_err(|e| IpcError::invalid("folder", &e.to_string()))?; + let bytes = serde_json::to_vec_pretty(store).map_err(|e| IpcError::invalid("store", &e.to_string()))?; + std::fs::write(file(data), bytes).map_err(|e| IpcError::invalid("folder", &e.to_string())) + } + + #[derive(Deserialize)] + pub struct StateArgs { + node: Option, + } + + /// The favorite folders that still exist, and the copies of a node whose files still exist. + pub fn state(data: &Path, args: StateArgs) -> IpcResult { + let store = load(data); + let favorites: Vec<&String> = store.favorites.iter().filter(|f| Path::new(f).is_dir()).collect(); + let copies: Vec<&Copy> = store + .copies + .iter() + .filter(|c| args.node.as_deref().is_none_or(|n| n == c.node) && Path::new(&c.path).is_file()) + .collect(); + Ok(json!({ "favorites": favorites, "copies": copies })) + } + + #[derive(Deserialize)] + pub struct FavoriteArgs { + path: String, + add: bool, + } + + /// Adds a folder to the favorites or removes it. + pub fn favorite(data: &Path, args: FavoriteArgs) -> IpcResult { + let mut store = load(data); + store.favorites.retain(|f| f != &args.path); + if args.add { + if !Path::new(&args.path).is_dir() { + return Err(IpcError::invalid("path", "Choose a folder that exists.")); + } + store.favorites.insert(0, args.path); + store.favorites.truncate(12); + } + save(data, &store)?; + Ok(json!({ "favorites": store.favorites })) + } + + /// Remembers a copy that was sent, replacing an earlier record of the same file. + pub fn record(data: &Path, copy: Copy) -> IpcResult { + let mut store = load(data); + store.copies.retain(|c| c.path != copy.path); + store.copies.insert(0, copy); + store.copies.truncate(50); + save(data, &store)?; + Ok(json!({})) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn favorite_folders_and_sent_copies_are_kept() { + let dir = tempfile::tempdir().expect("a folder"); + let data = dir.path().join("data"); + let target = dir.path().join("OneDrive"); + std::fs::create_dir(&target).expect("a folder"); + let file = target.join("Notes.docx"); + std::fs::write(&file, b"x").expect("a file"); + let path = target.to_string_lossy(); + dispatch("send_favorite", json!({ "path": path, "add": true }), &data).expect("adds"); + let copy = json!({ "node": "n1", "format": "docx", "path": file.to_string_lossy() }); + dispatch("send_record", copy, &data).expect("records"); + let state = dispatch("send_state", json!({ "node": "n1" }), &data).expect("reads"); + assert_eq!(state["favorites"].as_array().map(Vec::len), Some(1)); + assert_eq!(state["copies"].as_array().map(Vec::len), Some(1)); + let other = dispatch("send_state", json!({ "node": "n2" }), &data).expect("reads"); + assert_eq!(other["copies"].as_array().map(Vec::len), Some(0)); + std::fs::remove_file(&file).expect("removes"); + let gone = dispatch("send_state", json!({ "node": "n1" }), &data).expect("reads"); + assert_eq!(gone["copies"].as_array().map(Vec::len), Some(0)); + dispatch("send_favorite", json!({ "path": path, "add": false }), &data).expect("removes"); + let none = dispatch("send_state", json!({}), &data).expect("reads"); + assert_eq!(none["favorites"].as_array().map(Vec::len), Some(0)); + } + + #[test] + fn a_saved_report_gets_a_new_name_each_time() { + let dir = tempfile::tempdir().expect("a folder"); + grants::grant(dir.path()); + jobs::remember_report("import-1", "# Report".to_owned()); + let args = || json!({ "job": "import-1", "folder": dir.path().to_string_lossy() }); + let first = dispatch("save_report", args(), dir.path()).expect("saves"); + let second = dispatch("save_report", args(), dir.path()).expect("saves again"); + assert_ne!(first["path"], second["path"]); + assert_eq!( + std::fs::read_to_string(first["path"].as_str().expect("a path")).expect("reads"), + "# Report" + ); + assert!(dispatch( + "save_report", + json!({ "job": "gone", "folder": dir.path().to_string_lossy() }), + dir.path() + ) + .is_err()); + assert!(dispatch("nothing", json!({}), dir.path()).is_err()); + } + + #[test] + fn saved_text_gets_a_safe_new_name_and_never_replaces_a_file() { + let dir = tempfile::tempdir().unwrap(); + let folder = dir.path().to_string_lossy().into_owned(); + let save = |name: &str| { + let text = "# Highlights\n".to_owned(); + save_text(SaveText { + folder: folder.clone(), + name: name.into(), + text, + }) + .unwrap() + }; + let first = save("..\\..\\Bio: notes?"); + let second = save("..\\..\\Bio: notes?"); + let first = first["path"].as_str().unwrap().to_owned(); + assert!(first.ends_with("Bio notes.md"), "{first}"); + assert_ne!(first, second["path"].as_str().unwrap()); + assert_eq!(safe_stem("..."), "OpenNote"); + let refused = save_text(SaveText { + folder: "relative".into(), + name: "x".into(), + text: String::new(), + }); + assert!(refused.is_err()); + } +} diff --git a/app/src-tauri/src/interop/open_files.rs b/app/src-tauri/src/interop/open_files.rs new file mode 100644 index 00000000..340ac5a2 --- /dev/null +++ b/app/src-tauri/src/interop/open_files.rs @@ -0,0 +1,469 @@ +//! Opening one Markdown or text file as a page that saves back to the file (A3-18, A4-26, A1-27). +//! +//! A file reaches OpenNote three ways: Windows starts the app with it ("Open with", or the default app for `.md` and +//! `.txt`), a second launch forwards it to the running window, or the person picks it with Open file. Each way is the +//! person's own choice of that file, so each grants access to that one path. The interface can then read the file, +//! write it back, and watch it for outside edits, and nothing else: a path that was never granted is refused, and +//! only `.md`, `.markdown`, and `.txt` files are read or written. The pages linked to files are listed in +//! `opened_files.json` in the app's data folder, so a link, and its grant, outlive a restart. +//! +//! A write goes to a temporary file beside the original first and then replaces it, so a crash never leaves half a +//! file. It is refused when the file changed since the interface last read it, so an outside edit is never lost. + +use std::{ + collections::{BTreeMap, HashSet}, + fs, + io::Write as _, + path::{Path, PathBuf}, + sync::{Mutex, OnceLock}, + time::UNIX_EPOCH, +}; + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use crate::ipc::{IpcError, IpcResult}; + +/// The extensions of files that open as a page. +pub const TEXT_EXTENSIONS: [&str; 3] = ["md", "markdown", "txt"]; +/// The extension of a shared notebook file, which opens through Import notes. +pub const SHARE_EXTENSION: &str = "opennote"; +/// The biggest file that opens as a page. +const MAX_FILE_BYTES: u64 = 16 << 20; +/// The most pages linked to files. +const MAX_LINKS: usize = 2_000; +/// The file that lists the links. +const LINKS_FILE: &str = "opened_files.json"; + +#[derive(Default)] +struct State { + grants: HashSet, + launch: Vec, + loaded: bool, +} + +fn state() -> &'static Mutex { + static STATE: OnceLock> = OnceLock::new(); + STATE.get_or_init(Mutex::default) +} + +fn lock() -> std::sync::MutexGuard<'static, State> { + state().lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// The extension in lowercase, without the dot. +fn extension(path: &Path) -> String { + path.extension() + .map(|e| e.to_string_lossy().to_lowercase()) + .unwrap_or_default() +} + +/// Whether a path names a file this module opens: a text file, or a shared notebook file. +pub fn is_openable(path: &Path) -> bool { + let ext = extension(path); + TEXT_EXTENSIONS.contains(&ext.as_str()) || ext == SHARE_EXTENSION +} + +/// The path as the grants keep it: absolute and resolved, so `..` or a different case can't name another file. +fn key(path: &Path) -> Option { + if !path.is_absolute() { + return None; + } + fs::canonicalize(path).ok().filter(|p| p.is_file()) +} + +/// Grants access to one file the person chose. Returns the key it is kept under. +pub fn grant(path: &Path) -> Option { + if !is_openable(path) { + return None; + } + let found = key(path)?; + lock().grants.insert(found.clone()); + Some(found) +} + +/// The files among a launch's arguments: absolute paths of existing files with an extension this module opens. +/// Flags, links, and anything else are left alone. +pub fn files_in(args: &[String]) -> Vec { + args.iter() + .filter(|arg| !arg.starts_with("--") && !crate::deeplink::is_link(arg)) + .map(|arg| PathBuf::from(arg.trim().trim_matches('"'))) + .filter(|path| is_openable(path)) + .filter_map(|path| grant(&path)) + .take(32) + .collect() +} + +/// Keeps the files the first launch was started with, for the interface to take once. +pub fn remember_launch(args: &[String]) { + let files = files_in(args); + if !files.is_empty() { + lock().launch.extend(files); + } +} + +/// Grants the files a second launch forwarded. The interface reads the same arguments from the forwarded event. +pub fn grant_forwarded(args: &[String]) { + let _ = files_in(args); +} + +fn granted(path: &str) -> IpcResult { + let refused = || IpcError::invalid("path", "OpenNote can only open files you chose."); + let found = key(Path::new(path)).ok_or_else(refused)?; + if !TEXT_EXTENSIONS.contains(&extension(&found).as_str()) { + return Err(IpcError::invalid( + "path", + "Only Markdown and text files open as a page.", + )); + } + if lock().grants.contains(&found) { + Ok(found) + } else { + Err(refused()) + } +} + +/// The file's last change, in milliseconds since 1970. +fn modified(path: &Path) -> IpcResult { + let meta = fs::metadata(path).map_err(IpcError::from)?; + let time = meta.modified().map_err(IpcError::from)?; + Ok(time + .duration_since(UNIX_EPOCH) + .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))) +} + +#[derive(Deserialize)] +pub(super) struct PathArgs { + path: String, +} + +/// The files the app was started with, given out once. +pub(super) fn take_launch(data: &Path) -> IpcResult { + load_links(data); + let files: Vec = std::mem::take(&mut lock().launch) + .into_iter() + .map(|p| display(&p)) + .collect(); + Ok(json!({ "files": files })) +} + +/// Resolves a file the person already picked with Open file, or that the app was started with. It grants nothing: +/// the picker (`interop_pick`) and the launch arguments are the only grants, so script in the interface can't name +/// a file of its own choosing and get it opened. +pub(super) fn open_picked(args: PathArgs) -> IpcResult { + let found = granted(&args.path)?; + Ok(json!({ "path": display(&found), "share": false })) +} + +/// A path as the interface shows it, without the `\\?\` prefix that resolving adds on Windows. +fn display(path: &Path) -> String { + let text = path.to_string_lossy(); + text.strip_prefix(r"\\?\").unwrap_or(&text).to_owned() +} + +/// Reads a granted file: its text, decoded as the interop crate decodes notes, and when it last changed. +pub(super) fn read(args: PathArgs) -> IpcResult { + let path = granted(&args.path)?; + let size = fs::metadata(&path).map_err(IpcError::from)?.len(); + if size > MAX_FILE_BYTES { + return Err(IpcError::invalid("path", "The file is too big to open as a page.")); + } + let bytes = fs::read(&path).map_err(IpcError::from)?; + let decoded = opennote_interop::text::decode(&bytes); + Ok(json!({ + "path": display(&path), + "text": decoded.text, + "modified": modified(&path)?, + "markdown": extension(&path) != "txt", + })) +} + +#[derive(Deserialize)] +pub(super) struct WriteArgs { + path: String, + text: String, + /// When the interface last read or wrote the file. A file that changed since is not overwritten. + expected: Option, +} + +/// Writes a page's text back to its granted file, unless the file changed since the interface last saw it. +pub(super) fn write(args: WriteArgs) -> IpcResult { + let path = granted(&args.path)?; + if args.text.len() as u64 > MAX_FILE_BYTES { + return Err(IpcError::invalid("text", "The page is too big to save into the file.")); + } + let now = modified(&path)?; + if args.expected.is_some_and(|expected| expected != now) { + return Err(IpcError::new("changed", "The file changed outside OpenNote.")); + } + let parent = path + .parent() + .ok_or_else(|| IpcError::invalid("path", "The file has no folder."))?; + let mut temp = tempfile::Builder::new() + .prefix(".opennote-save-") + .tempfile_in(parent) + .map_err(IpcError::from)?; + temp.write_all(args.text.as_bytes()).map_err(IpcError::from)?; + temp.as_file().sync_all().map_err(IpcError::from)?; + temp.persist(&path).map_err(|e| IpcError::from(e.error))?; + Ok(json!({ "modified": modified(&path)? })) +} + +/// When a granted file last changed, or that it is gone. A path that was never granted is refused the same way +/// whether it exists, so the interface can't use this to learn which files are on the PC. +pub(super) fn stat(args: PathArgs) -> IpcResult { + let refused = || IpcError::invalid("path", "OpenNote can only open files you chose."); + let Some(asked) = Path::new(&args.path).is_absolute().then(|| PathBuf::from(&args.path)) else { + return Err(refused()); + }; + if let Ok(path) = granted(&args.path) { + return Ok(json!({ "exists": true, "modified": modified(&path)? })); + } + // A granted file that has since gone no longer resolves, so it is matched by the path it was granted under. + let gone = !asked.exists() + && lock() + .grants + .iter() + .any(|g| display(g).eq_ignore_ascii_case(&display(&asked))); + if gone { + Ok(json!({ "exists": false, "modified": null })) + } else { + Err(refused()) + } +} + +/// The pages linked to files: page ID to path. +#[derive(Default, Serialize, Deserialize)] +struct Links { + pages: BTreeMap, +} + +fn links_path(data: &Path) -> PathBuf { + data.join(LINKS_FILE) +} + +fn read_links(data: &Path) -> Links { + fs::read(links_path(data)) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_default() +} + +/// Loads the saved links once, and grants their files again: the person chose each of them before. +fn load_links(data: &Path) { + if lock().loaded { + return; + } + let links = read_links(data); + let mut state = lock(); + state.loaded = true; + for path in links.pages.values() { + if let Some(found) = key(Path::new(path)).filter(|p| is_openable(p)) { + state.grants.insert(found); + } + } +} + +/// Every page linked to a file. +pub(super) fn list(data: &Path) -> IpcResult { + load_links(data); + Ok(json!({ "pages": read_links(data).pages })) +} + +#[derive(Deserialize)] +pub(super) struct LinkArgs { + page: String, + /// The file, or `None` to take the link away. + path: Option, +} + +/// Links a page to a granted file, or takes the link away. +pub(super) fn link(data: &Path, args: LinkArgs) -> IpcResult { + load_links(data); + let valid_page = !args.page.is_empty() + && args.page.len() <= 64 + && args + .page + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); + if !valid_page { + return Err(IpcError::invalid("page", "That isn't a page.")); + } + let mut links = read_links(data); + match args.path { + Some(path) => { + let found = granted(&path)?; + if links.pages.len() >= MAX_LINKS && !links.pages.contains_key(&args.page) { + return Err(IpcError::invalid("page", "Too many pages are linked to files.")); + } + links.pages.insert(args.page, display(&found)); + } + None => { + links.pages.remove(&args.page); + } + } + fs::create_dir_all(data).map_err(IpcError::from)?; + let bytes = serde_json::to_vec_pretty(&links).map_err(|e| IpcError::invalid("links", &e.to_string()))?; + fs::write(links_path(data), bytes).map_err(IpcError::from)?; + Ok(Value::Null) +} + +/// Opens Windows' Default apps settings, where the person can make OpenNote the app for `.md` and `.txt` files. +/// Windows lets only the person choose a default app, so OpenNote shows the page and changes nothing itself. +pub(super) fn default_apps() -> IpcResult { + #[cfg(windows)] + { + std::process::Command::new("explorer.exe") + .arg("ms-settings:defaultapps") + .spawn() + .map_err(IpcError::from)?; + } + Ok(Value::Null) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(dir: &Path, name: &str, text: &str) -> String { + let path = dir.join(name); + fs::write(&path, text).expect("a file"); + path.to_string_lossy().into_owned() + } + + #[test] + fn naming_a_file_grants_nothing_and_says_nothing_about_it() { + let dir = tempfile::tempdir().expect("a folder"); + let there = file( + dir.path(), + "Diary.md", + "private +", + ); + let missing = dir.path().join("Missing.md").to_string_lossy().into_owned(); + assert!( + open_picked(PathArgs { path: there.clone() }).is_err(), + "only the picker grants" + ); + assert!(read(PathArgs { path: there.clone() }).is_err()); + let a = stat(PathArgs { path: there.clone() }).expect_err("refused"); + let b = stat(PathArgs { path: missing }).expect_err("refused the same way"); + assert_eq!(format!("{a:?}"), format!("{b:?}")); + grant(Path::new(&there)).expect("granted by the picker"); + assert!(open_picked(PathArgs { path: there.clone() }).is_ok()); + fs::remove_file(&there).expect("removed"); + assert_eq!(stat(PathArgs { path: there }).expect("known")["exists"], false); + } + + #[test] + fn only_granted_text_files_are_read_and_written() { + let dir = tempfile::tempdir().expect("a folder"); + let notes = file(dir.path(), "Notes.md", "# Notes\n\nHello\n"); + let other = file(dir.path(), "Other.md", "secret\n"); + let refused = read(PathArgs { path: notes.clone() }); + assert!(refused.is_err(), "not granted yet"); + assert!(grant(Path::new(¬es)).is_some()); + let read_back = read(PathArgs { path: notes.clone() }).expect("reads"); + assert_eq!(read_back["text"], "# Notes\n\nHello\n"); + assert_eq!(read_back["markdown"], true); + assert!(read(PathArgs { path: other }).is_err(), "another file stays closed"); + let sneaky = format!("{}\\..\\Other.md", dir.path().join("x").display()); + assert!(read(PathArgs { path: sneaky }).is_err()); + assert!(grant(&dir.path().join("Notes.exe")).is_none()); + assert!( + read(PathArgs { + path: "Notes.md".into() + }) + .is_err(), + "a relative path is refused" + ); + } + + #[test] + fn a_write_replaces_the_file_unless_it_changed_outside() { + let dir = tempfile::tempdir().expect("a folder"); + let notes = file(dir.path(), "Save.md", "one\n"); + grant(Path::new(¬es)).expect("granted"); + let first = read(PathArgs { path: notes.clone() }).expect("reads"); + let when = first["modified"].as_i64(); + let saved = write(WriteArgs { + path: notes.clone(), + text: "two\n".into(), + expected: when, + }) + .expect("saves"); + assert_eq!(fs::read_to_string(¬es).expect("reads"), "two\n"); + let stale = write(WriteArgs { + path: notes.clone(), + text: "three\n".into(), + expected: Some(1), + }); + assert_eq!(stale.expect_err("refused").code, "changed"); + assert_eq!( + fs::read_to_string(¬es).expect("reads"), + "two\n", + "the outside copy is kept" + ); + let again = write(WriteArgs { + path: notes.clone(), + text: "four\n".into(), + expected: saved["modified"].as_i64(), + }); + assert!(again.is_ok()); + let leftovers = fs::read_dir(dir.path()).expect("lists").count(); + assert_eq!(leftovers, 1, "no temporary file is left"); + } + + #[test] + fn launch_arguments_grant_only_existing_text_files() { + let dir = tempfile::tempdir().expect("a folder"); + let notes = file(dir.path(), "Launch.txt", "hi"); + let args = vec![ + "--wait-pid=3".to_owned(), + "opennote://page/abc".to_owned(), + notes.clone(), + dir.path().join("missing.md").to_string_lossy().into_owned(), + "relative.md".to_owned(), + ]; + let files = files_in(&args); + assert_eq!(files.len(), 1); + assert!(read(PathArgs { path: notes }).is_ok()); + } + + #[test] + fn links_are_kept_and_grant_their_files_again() { + let dir = tempfile::tempdir().expect("a folder"); + let data = dir.path().join("data"); + let notes = file(dir.path(), "Linked.md", "x"); + grant(Path::new(¬es)).expect("granted"); + link( + &data, + LinkArgs { + page: "p1".into(), + path: Some(notes.clone()), + }, + ) + .expect("links"); + let listed = list(&data).expect("lists"); + assert!(listed["pages"]["p1"].as_str().is_some_and(|p| p.ends_with("Linked.md"))); + assert!(link( + &data, + LinkArgs { + page: "../x".into(), + path: None + } + ) + .is_err()); + link( + &data, + LinkArgs { + page: "p1".into(), + path: None, + }, + ) + .expect("unlinks"); + assert!(list(&data).expect("lists")["pages"] + .as_object() + .is_some_and(|m| m.is_empty())); + } +} diff --git a/app/src-tauri/src/interop/pdf_stage.rs b/app/src-tauri/src/interop/pdf_stage.rs new file mode 100644 index 00000000..9c8dfba4 --- /dev/null +++ b/app/src-tauri/src/interop/pdf_stage.rs @@ -0,0 +1,160 @@ +//! The PDF files of pages the interface printed for a bundle export (ADR 0006). +//! +//! A section or notebook exports as PDF in two steps. The interface prints each page in the hidden print window, the +//! same way a page's own Export as PDF does, and hands each file here under the export's job name. The export then +//! asks [`take`] for that job's files and writes the bundle with them. Nothing here touches the disk. Each file is +//! checked to be a whole PDF file, and the files of one job, and of all jobs together, have a size limit, so a +//! stuck export can't fill the memory. Finished and canceled exports drop their files. + +use std::{ + collections::HashMap, + str::FromStr, + sync::{Mutex, OnceLock}, +}; + +use base64::Engine as _; +use opennote_core::PageId; +use opennote_interop::PreparedPdfRenderer; +use serde::Deserialize; +use serde_json::{json, Value}; + +use crate::ipc::{IpcError, IpcResult}; + +/// The largest PDF file of one page. +const MAX_PAGE_BYTES: usize = 128 << 20; +/// The most all staged files may take together. +const MAX_TOTAL_BYTES: usize = 768 << 20; +/// The most pages one job may stage. +const MAX_PAGES: usize = 10_000; +/// The most jobs staged at once. +const MAX_JOBS: usize = 4; + +#[derive(Default)] +struct Staged { + pages: HashMap>, + bytes: usize, +} + +fn store() -> &'static Mutex> { + static STORE: OnceLock>> = OnceLock::new(); + STORE.get_or_init(Mutex::default) +} + +/// A job name is what the interface makes: letters, digits, `-`, and `_`, at most 64 of them. +fn valid_job(job: &str) -> bool { + !job.is_empty() && job.len() <= 64 && job.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') +} + +#[derive(Deserialize)] +pub(super) struct StageArgs { + job: String, + page: String, + /// The PDF file, in standard base64. + data: String, +} + +/// Keeps the printed PDF file of one page for the job's export. +pub(super) fn stage(args: StageArgs) -> IpcResult { + if !valid_job(&args.job) { + return Err(IpcError::invalid("job", "The export job's name isn't valid.")); + } + let page = PageId::from_str(&args.page).map_err(|_| IpcError::invalid("page", "That isn't a page."))?; + if args.data.len() > MAX_PAGE_BYTES / 3 * 4 + 4 { + return Err(IpcError::invalid("data", "The page's PDF file is too big.")); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(args.data.as_bytes()) + .map_err(|_| IpcError::invalid("data", "The page's PDF file didn't arrive whole."))?; + if !PreparedPdfRenderer::looks_like_pdf(&bytes) { + return Err(IpcError::invalid("data", "That isn't a PDF file.")); + } + let mut jobs = store() + .lock() + .map_err(|_| IpcError::invalid("job", "The export can't continue."))?; + let total: usize = jobs.values().map(|s| s.bytes).sum(); + if !jobs.contains_key(&args.job) && jobs.len() >= MAX_JOBS { + return Err(IpcError::invalid( + "job", + "Too many exports are running. Wait for one to finish.", + )); + } + let staged = jobs.entry(args.job).or_default(); + let replaced = staged.pages.get(&page).map_or(0, Vec::len); + if total - replaced + bytes.len() > MAX_TOTAL_BYTES || (replaced == 0 && staged.pages.len() >= MAX_PAGES) { + return Err(IpcError::invalid( + "data", + "The export is too big to make in one go. Export a section at a time.", + )); + } + staged.bytes = staged.bytes - replaced + bytes.len(); + staged.pages.insert(page, bytes); + Ok(json!({ "pages": staged.pages.len() })) +} + +#[derive(Deserialize)] +pub(super) struct DropArgs { + job: String, +} + +/// Drops a job's files, such as after the person cancels while pages print. +pub(super) fn unstage(args: DropArgs) -> IpcResult { + if let Ok(mut jobs) = store().lock() { + jobs.remove(&args.job); + } + Ok(Value::Null) +} + +/// Takes the job's files for its export. A job that staged nothing gets an empty renderer, which reports every page. +pub(super) fn take(job: &str) -> PreparedPdfRenderer { + let staged = store().lock().ok().and_then(|mut jobs| jobs.remove(job)); + PreparedPdfRenderer::new(staged.map(|s| s.pages).unwrap_or_default()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const PDF: &[u8] = b"%PDF-1.7\n%%EOF\n"; + + fn encode(bytes: &[u8]) -> String { + base64::engine::general_purpose::STANDARD.encode(bytes) + } + + fn page() -> String { + PageId::generate(&opennote_core::SystemClock::new()).to_string() + } + + #[test] + fn a_staged_page_is_taken_once_by_its_job() { + let id = page(); + let args = StageArgs { + job: "j-stage-1".into(), + page: id.clone(), + data: encode(PDF), + }; + assert_eq!(stage(args).expect("stages")["pages"], 1); + let renderer = take("j-stage-1"); + let again = take("j-stage-1"); + let parsed = PageId::from_str(&id).expect("an ID"); + assert!(renderer.has_page(parsed)); + assert!(!again.has_page(parsed), "taken once"); + } + + #[test] + fn bad_names_pages_and_files_are_refused() { + let bad = |job: &str, page: String, data: String| { + stage(StageArgs { + job: job.into(), + page, + data, + }) + .is_err() + }; + assert!(bad("../x", page(), encode(PDF))); + assert!(bad("", page(), encode(PDF))); + assert!(bad("j-ok", "not a page".into(), encode(PDF))); + assert!(bad("j-ok", page(), encode(b""))); + assert!(bad("j-ok", page(), "%%%".into())); + unstage(DropArgs { job: "j-ok".into() }).expect("drops"); + } +} diff --git a/app/src-tauri/src/interop/pick.rs b/app/src-tauri/src/interop/pick.rs new file mode 100644 index 00000000..481a3b86 --- /dev/null +++ b/app/src-tauri/src/interop/pick.rs @@ -0,0 +1,99 @@ +//! The Windows pickers for a source to import and a folder to export into. + +use serde::Deserialize; + +use crate::ipc::IpcResult; + +/// What the person picks. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PickKind { + /// A file: an export from another app, a Word file, or a database. + File, + /// A folder: a vault, an export, or the place an export goes. + Folder, +} + +/// The file types the picker lists first, as one filter. The interop crate decides what each file really is. +#[cfg(windows)] +const SOURCE_TYPES: &str = concat!( + "*.enex;*.docx;*.odt;*.xlsx;*.xlsm;*.pptx;*.eml;*.opennote;*.md;*.markdown;*.txt;", + "*.html;*.htm;*.mht;*.mhtml;*.csv;*.tsv;*.zip;*.sqlite;*.one;*.onepkg;", + "*.doc;*.rtf;*.xls;*.xlsb;*.ods" +); + +/// Shows the picker and returns the chosen path, or `None` when the person cancels. +#[cfg(windows)] +pub fn pick(owner: isize, kind: PickKind, initial: Option) -> IpcResult> { + use std::path::Path; + + use windows::{ + core::{w, HSTRING, PCWSTR}, + Win32::{ + Foundation::{ERROR_CANCELLED, HWND}, + System::Com::{ + CoCreateInstance, CoInitializeEx, CoTaskMemFree, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_APARTMENTTHREADED, + }, + UI::Shell::{ + Common::COMDLG_FILTERSPEC, FileOpenDialog, IFileOpenDialog, IShellItem, SHCreateItemFromParsingName, + FOS_FORCEFILESYSTEM, FOS_PICKFOLDERS, SIGDN_FILESYSPATH, + }, + }, + }; + + // SAFETY: COM is initialized for this thread and released before it returns, and every object below lives + // within that span. The owner handle names the main window, or is null for none. The filter strings outlive + // the dialog. + unsafe { + let apartment = CoInitializeEx(None, COINIT_APARTMENTTHREADED); + let result = (|| -> windows::core::Result> { + let dialog: IFileOpenDialog = CoCreateInstance(&FileOpenDialog, None, CLSCTX_INPROC_SERVER)?; + let options = dialog.GetOptions()? | FOS_FORCEFILESYSTEM; + match kind { + PickKind::Folder => dialog.SetOptions(options | FOS_PICKFOLDERS)?, + PickKind::File => { + dialog.SetOptions(options)?; + let types = HSTRING::from(SOURCE_TYPES); + let filters = [ + COMDLG_FILTERSPEC { + pszName: w!("Notes and documents"), + pszSpec: PCWSTR(types.as_ptr()), + }, + COMDLG_FILTERSPEC { + pszName: w!("All files"), + pszSpec: w!("*.*"), + }, + ]; + dialog.SetFileTypes(&filters)?; + } + } + if let Some(folder) = initial.filter(|folder| Path::new(folder).is_dir()) { + if let Ok(item) = SHCreateItemFromParsingName::<_, _, IShellItem>(&HSTRING::from(folder), None) { + dialog.SetFolder(&item)?; + } + } + let parent = (owner != 0).then_some(HWND(owner as *mut _)); + if let Err(error) = dialog.Show(parent) { + return if error.code() == ERROR_CANCELLED.to_hresult() { + Ok(None) + } else { + Err(error) + }; + } + let name = dialog.GetResult()?.GetDisplayName(SIGDN_FILESYSPATH)?; + let path = name.to_string().ok(); + CoTaskMemFree(Some(name.0.cast_const().cast())); + Ok(path) + })(); + if apartment.is_ok() { + CoUninitialize(); + } + Ok(result?) + } +} + +#[cfg(not(windows))] +pub fn pick(_owner: isize, _kind: PickKind, _initial: Option) -> IpcResult> { + Err(crate::ipc::IpcError::not_implemented("interop_pick")) +} diff --git a/app/src-tauri/src/interop/restore.rs b/app/src-tauri/src/interop/restore.rs new file mode 100644 index 00000000..0acacdb4 --- /dev/null +++ b/app/src-tauri/src/interop/restore.rs @@ -0,0 +1,88 @@ +//! What an import made, as the interface shows it. +//! +//! An import writes a whole notebook folder into the notes folder that setup chose, in the note format, and the +//! core opens it like any notebook there. Its node IDs are the core's, so the notes service, the tree, and the +//! page commands reach it with no map between the interface's IDs and the core's. This module answers with the +//! new notebook's tree so the dialog can offer to open it, and cleans up what a crashed import left behind. + +use std::{fs, path::Path}; + +use opennote_core::session::notebook::NotebookHandle; +use serde::Serialize; + +/// The hidden folder an import works in, inside the notes folder, until it finishes (`DiskSink`). +const STAGING_PREFIX: &str = ".importing-"; + +/// Deletes what a crashed import left in its hidden working folder in the notes folder. +pub fn clean_staging(folder: &Path) { + let Ok(entries) = fs::read_dir(folder) else { + return; + }; + for entry in entries.flatten() { + if entry.file_name().to_string_lossy().starts_with(STAGING_PREFIX) { + let _ = fs::remove_dir_all(entry.path()); + } + } +} + +/// An imported notebook as the interface shows it. Every ID is a node ID of the notes tree. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportedTree { + /// The notebook's node ID. + pub notebook_id: String, + /// The notebook folder. + pub dir: String, + pub title: String, + pub color: Option, + pub sections: Vec, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportedSection { + /// The section's node ID. + pub id: String, + pub title: String, + pub color: Option, + pub pages: Vec, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportedPageNode { + /// The page's node ID, which is the core's page ID. + pub core: String, + pub title: String, + /// 0 for a page, 1 and 2 for subpages. + pub level: u8, +} + +/// The navigation tree of an imported notebook, in display order. +pub fn tree_of(handle: &NotebookHandle) -> ImportedTree { + let tree = handle.tree(); + ImportedTree { + notebook_id: tree.notebook.to_string(), + dir: handle.path().to_string_lossy().into_owned(), + title: tree.title, + color: tree.color.map(|color| color.to_text()), + sections: tree + .sections + .into_iter() + .map(|section| ImportedSection { + id: section.id.to_string(), + title: section.title, + color: section.color.map(|color| color.to_text()), + pages: section + .pages + .into_iter() + .map(|page| ImportedPageNode { + core: page.id.to_string(), + title: page.title, + level: page.level, + }) + .collect(), + }) + .collect(), + } +} diff --git a/app/src-tauri/src/interop/tests.rs b/app/src-tauri/src/interop/tests.rs new file mode 100644 index 00000000..820c7009 --- /dev/null +++ b/app/src-tauri/src/interop/tests.rs @@ -0,0 +1,461 @@ +use std::{ + fs, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; + +use base64::Engine as _; +use opennote_interop::{CancelToken, Control, Event, ImportOptions, InteropError, NoPdfRenderer, Phase, Unit}; +use serde_json::{json, Value}; + +use super::{ + commands::{import_alone, run_import}, + export::{self, ExportFormat, ExportPage, ExportRequest, ExportScope, ExportSection, TreeSource}, + jobs, + restore::{self, ImportedTree}, +}; +use crate::core_bridge::CoreBridge; + +/// A small Markdown vault: two notes in a folder and one at the top. +fn vault(base: &Path) -> PathBuf { + let vault = base.join("Vault"); + fs::create_dir_all(vault.join("Biology")).expect("a folder"); + fs::write( + vault.join("Biology").join("Cells.md"), + "# Cells\n\nCells divide by mitosis.\n", + ) + .expect("a note"); + fs::write( + vault.join("Biology").join("Plants.md"), + "# Plants\n\nPlants make sugar from light.\n", + ) + .expect("a note"); + fs::write(vault.join("Welcome.md"), "# Welcome\n\nStart here.\n").expect("a note"); + vault +} + +struct Imported { + notes: PathBuf, + notebook: PathBuf, + tree: ImportedTree, +} + +/// Imports the vault through the same steps as the command: write into the notes folder, open it in the core as a +/// notes command does, and read the tree. +fn import_vault(bridge: &CoreBridge, base: &Path) -> Imported { + let notes = base.join("Notes"); + // The notes folder already holds a notebook, as it does after setup. + bridge + .notes(Some(notes.clone()), |bridge| { + let input = + json!({ "kind": "notebook", "placement": { "parentId": null, "beforeId": null }, "title": "Mine" }); + bridge.dispatch("notes_create", &json!({ "input": input })).map(|_| ()) + }) + .expect("a first notebook"); + let device = bridge.with(|bridge| Ok(bridge.core.device())).expect("a device"); + restore::clean_staging(¬es); + let (report, notebook) = run_import(¬es, &vault(base), &ImportOptions::default(), device, Control::none()) + .expect("the vault imports"); + assert_eq!(report.pages.len(), 3); + let tree = bridge + .notes(Some(notes.clone()), |bridge| { + let handle = bridge.core.open_notebook(¬ebook).expect("the notebook opens"); + Ok(restore::tree_of(&handle)) + }) + .expect("the tree"); + Imported { notes, notebook, tree } +} + +/// The core page IDs of the tree, in display order. +fn pages_of(tree: &ImportedTree) -> Vec { + tree.sections + .iter() + .flat_map(|section| §ion.pages) + .map(|page| page.core.clone()) + .collect() +} + +fn page_json(bridge: &CoreBridge, notes: &Path, page: &str) -> String { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + let handle = bridge.handle(page, "main-1")?; + let envelope = handle.envelope(None).expect("an envelope"); + let decoded = opennote_core::wire::envelope::decode(&envelope.bytes).expect("decodes"); + Ok(String::from_utf8_lossy(decoded.page_json).into_owned()) + }) + .expect("the page opens") +} + +#[test] +fn an_imported_notebook_is_a_real_notebook_folder_in_the_notes_tree_and_survives_a_restart() { + let dir = tempfile::tempdir().expect("a temp folder"); + let local = dir.path().join("app"); + let bridge = CoreBridge::at(local.clone()); + let imported = import_vault(&bridge, dir.path()); + assert_eq!(imported.notebook.parent(), Some(imported.notes.as_path())); + assert!(imported.notebook.join("notebook.json").is_file()); + // The notes commands list it beside the notebook that was there, with the IDs the tree gave. + let listed = bridge + .notes(Some(imported.notes.clone()), |bridge| { + bridge.dispatch("notes_list_notebooks", &json!({})) + }) + .expect("the notebooks"); + let ids: Vec<&str> = listed + .as_array() + .expect("a list") + .iter() + .filter_map(|node| node["id"].as_str()) + .collect(); + assert_eq!(ids.len(), 2, "{listed}"); + assert!(ids.contains(&imported.tree.notebook_id.as_str())); + let pages = pages_of(&imported.tree); + assert_eq!(pages.len(), 3); + let node = bridge + .notes(Some(imported.notes.clone()), |bridge| { + bridge.dispatch("notes_get", &json!({ "id": pages[0] })) + }) + .expect("a page node"); + assert_eq!(node["kind"], "page"); + let all: String = pages + .iter() + .map(|page| page_json(&bridge, &imported.notes, page)) + .collect(); + assert!(all.contains("mitosis") && all.contains("sugar from light") && all.contains("Start here")); + bridge.shutdown(); + + // Another start opens the imported notebook from the notes folder like any other. + let again = CoreBridge::at(local); + let all: String = pages + .iter() + .map(|page| page_json(&again, &imported.notes, page)) + .collect(); + let after = again + .notes(Some(imported.notes.clone()), |bridge| { + bridge.dispatch("notes_list_notebooks", &json!({})) + }) + .expect("the notebooks"); + again.shutdown(); + assert!(all.contains("mitosis") && all.contains("Start here")); + assert_eq!(after.as_array().expect("a list").len(), 2); +} + +#[test] +fn an_import_leaves_the_notes_folder_with_only_notebook_folders() { + let dir = tempfile::tempdir().expect("a temp folder"); + let bridge = CoreBridge::at(dir.path().join("app")); + let imported = import_vault(&bridge, dir.path()); + bridge.shutdown(); + let names: Vec = fs::read_dir(&imported.notes) + .expect("the notes folder") + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + assert!(names.iter().all(|name| !name.starts_with(".importing-")), "{names:?}"); + assert_eq!(names.len(), 2, "{names:?}"); +} + +#[test] +fn a_crashed_imports_working_folder_is_cleaned_up() { + let dir = tempfile::tempdir().expect("a temp folder"); + let stale = dir.path().join(".importing-1234"); + fs::create_dir_all(stale.join("half")).expect("a leftover"); + fs::create_dir_all(dir.path().join("Biology")).expect("a notebook folder"); + restore::clean_staging(dir.path()); + assert!(!stale.exists()); + assert!(dir.path().join("Biology").is_dir()); +} + +#[test] +fn a_one_file_export_reveals_the_file_so_it_can_be_updated_later() { + let dir = tempfile::tempdir().expect("a temp folder"); + let file = dir.path().join("table.docx"); + fs::write(&file, b"docx").expect("a file"); + let done = super::commands::ExportDone::from(opennote_interop::Exported { + root: dir.path().to_path_buf(), + files: vec![file.clone()], + report: opennote_interop::Report::new(opennote_interop::ReportKind::Export, "table"), + }); + assert_eq!(done.reveal, file.to_string_lossy()); +} + +#[test] +fn a_second_import_does_not_remove_the_working_folder_of_one_still_running() { + let dir = tempfile::tempdir().expect("a temp folder"); + let folder = dir.path().to_path_buf(); + let (started, wait) = std::sync::mpsc::channel(); + let first = { + let folder = folder.clone(); + std::thread::spawn(move || { + import_alone(&folder, || { + let staging = folder.join(".importing-first"); + fs::create_dir_all(&staging).expect("a working folder"); + started.send(()).expect("send"); + std::thread::sleep(std::time::Duration::from_millis(300)); + staging.is_dir() + }) + }) + }; + wait.recv().expect("the first import started"); + import_alone(&folder, || ()); + assert!( + first.join().expect("the first import"), + "the running import kept its folder" + ); +} + +fn request(format: ExportFormat, scope: ExportScope, folder: &Path, pages: &[String]) -> ExportRequest { + let page = |ui: &str, title: &str, level: u8| ExportPage { + ui: ui.to_owned(), + title: title.to_owned(), + level, + }; + ExportRequest { + format, + scope, + title: "Biology".to_owned(), + sections: vec![ExportSection { + title: "Unit one".to_owned(), + pages: vec![ + page(&pages[0], "Cells", 0), + page(&pages[1], "Plants", 1), + // A page the core doesn't know has no files to read. + page("ui-never", "Fresh page", 0), + ], + }], + folder: folder.to_string_lossy().into_owned(), + replace: None, + password: None, + history: false, + } +} + +fn imported_bridge(dir: &Path) -> (CoreBridge, Imported) { + let bridge = CoreBridge::at(dir.join("app")); + let imported = import_vault(&bridge, dir); + (bridge, imported) +} + +fn files_under(dir: &Path) -> Vec { + let mut found = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(next) = stack.pop() { + for entry in fs::read_dir(&next).expect("a folder").flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else { + found.push(path); + } + } + } + found.sort(); + found +} + +#[test] +fn a_markdown_export_writes_the_interface_tree_with_its_pages() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (bridge, imported) = imported_bridge(dir.path()); + let pages = pages_of(&imported.tree); + let out = dir.path().join("out"); + fs::create_dir_all(&out).expect("an output folder"); + let request = request(ExportFormat::Markdown, ExportScope::Notebook, &out, &pages); + let source = bridge + .with(|bridge| Ok(TreeSource::build(bridge, &request))) + .expect("the bridge") + .expect("the source"); + let exported = export::run(&source, &request, &NoPdfRenderer, &Control::none()).expect("the export"); + bridge.shutdown(); + assert_eq!(exported.files.len(), 3); + let text: String = files_under(&exported.root) + .iter() + .filter(|path| path.extension().is_some_and(|ext| ext == "md")) + .map(|path| fs::read_to_string(path).expect("a page")) + .collect(); + assert!(text.contains("mitosis"), "{text}"); + assert!(text.contains("Fresh page"), "{text}"); +} + +#[test] +fn a_section_export_and_a_word_export_each_write_their_files() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (bridge, imported) = imported_bridge(dir.path()); + let pages = pages_of(&imported.tree); + let out = dir.path().join("out"); + fs::create_dir_all(&out).expect("an output folder"); + let word = request(ExportFormat::Docx, ExportScope::Section, &out, &pages); + let single = request(ExportFormat::HtmlSingle, ExportScope::Notebook, &out, &pages); + let (word_done, single_done) = bridge + .with(|bridge| { + let word_source = TreeSource::build(bridge, &word).expect("the source"); + let single_source = TreeSource::build(bridge, &single).expect("the source"); + Ok(( + export::run(&word_source, &word, &NoPdfRenderer, &Control::none()), + export::run(&single_source, &single, &NoPdfRenderer, &Control::none()), + )) + }) + .expect("the bridge"); + bridge.shutdown(); + let word_done = word_done.expect("the Word export"); + let single_done = single_done.expect("the single page export"); + assert!(word_done + .files + .iter() + .any(|path| path.extension().is_some_and(|ext| ext == "docx"))); + assert!(single_done + .files + .iter() + .any(|path| path.extension().is_some_and(|ext| ext == "html"))); +} + +#[test] +fn pdf_export_writes_the_printed_pages_and_an_index() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (bridge, imported) = imported_bridge(dir.path()); + let pages = pages_of(&imported.tree); + let out = dir.path().join("out"); + fs::create_dir_all(&out).expect("an output folder"); + let request = request(ExportFormat::Pdf, ExportScope::Notebook, &out, &pages); + // The interface prints each page in the hidden window and stages the file under the job's name. + let pdf = b"%PDF-1.7 +1 0 obj<<>>endobj +%%EOF +"; + for page in &pages { + let args = + json!({ "job": "j-pdf-test", "page": page, "data": base64::engine::general_purpose::STANDARD.encode(pdf) }); + super::pdf_stage::stage(serde_json::from_value(args).expect("arguments")).expect("staged"); + } + let renderer = super::pdf_stage::take("j-pdf-test"); + let result = bridge + .with(|bridge| { + let source = TreeSource::build(bridge, &request).expect("the source"); + Ok(export::run(&source, &request, &renderer, &Control::none())) + }) + .expect("the bridge"); + bridge.shutdown(); + let exported = result.expect("the export"); + let files = files_under(&out); + let pdfs = files + .iter() + .filter(|p| p.extension().is_some_and(|e| e == "pdf")) + .count(); + // The request names two pages the core knows (Cells and Plants) and one it does not, which is skipped. + assert_eq!(pdfs, 2, "{files:?}"); + assert!(files.iter().any(|p| p.ends_with("index.html"))); + assert_eq!(exported.report.pages.len(), 2); +} + +#[test] +fn a_pdf_export_with_nothing_printed_reports_every_page() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (bridge, imported) = imported_bridge(dir.path()); + let pages = pages_of(&imported.tree); + let out = dir.path().join("out"); + fs::create_dir_all(&out).expect("an output folder"); + let request = request(ExportFormat::Pdf, ExportScope::Notebook, &out, &pages); + let renderer = super::pdf_stage::take("j-nothing-staged"); + let result = bridge + .with(|bridge| { + let source = TreeSource::build(bridge, &request).expect("the source"); + Ok(export::run(&source, &request, &renderer, &Control::none())) + }) + .expect("the bridge"); + bridge.shutdown(); + let exported = result.expect("the export"); + let (_, skipped) = exported.report.loss_counts(); + assert_eq!(skipped, pages.len()); +} + +#[test] +fn a_notebook_shares_as_one_locked_file_that_opens_with_its_password() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (bridge, imported) = imported_bridge(dir.path()); + let pages = pages_of(&imported.tree); + let out = dir.path().join("out"); + fs::create_dir_all(&out).expect("an output folder"); + let mut request = request(ExportFormat::Share, ExportScope::Notebook, &out, &pages); + let secret = ["blue", "pencil", "case"].join(" "); + request.password = serde_json::from_value(json!(secret)).expect("a password"); + request.history = true; + assert!(!format!("{request:?}").contains("pencil"), "the password never prints"); + let device = bridge.with(|bridge| Ok(bridge.core.device())).expect("a device"); + let result = bridge + .with(|bridge| { + let source = TreeSource::build(bridge, &request).expect("the source"); + Ok(export::run(&source, &request, &NoPdfRenderer, &Control::none())) + }) + .expect("the bridge"); + bridge.shutdown(); + let exported = result.expect("the share"); + assert_eq!(exported.files.len(), 1); + let file = &exported.files[0]; + assert!(file.extension().is_some_and(|e| e == "opennote")); + let found = opennote_interop::detect(file).expect("known"); + assert!(found.needs_password); + let choices: super::commands::ImportChoices = + serde_json::from_value(json!({ "reportPage": false, "password": secret })).expect("choices"); + assert!(!format!("{choices:?}").contains("pencil")); + let parent = dir.path().join("opened"); + fs::create_dir_all(&parent).expect("a folder"); + let (report, _) = run_import(&parent, file, &choices.options(), device, Control::none()).expect("opens"); + assert_eq!(report.pages.len(), 3); +} + +#[test] +fn a_canceled_import_leaves_no_notebook_behind() { + let dir = tempfile::tempdir().expect("a temp folder"); + let bridge = CoreBridge::at(dir.path().join("app")); + let device = bridge.with(|bridge| Ok(bridge.core.device())).expect("a device"); + let token = CancelToken::new(); + token.cancel(); + let parent = dir.path().join("imports"); + let result = run_import( + &parent, + &vault(dir.path()), + &ImportOptions::default(), + device, + Control::with_cancel(token), + ); + bridge.shutdown(); + assert!(matches!(result, Err(InteropError::Canceled))); + assert_eq!(fs::read_dir(&parent).expect("the parent").count(), 0); +} + +#[test] +fn a_running_job_cancels_by_name_and_forgets_itself_when_done() { + let emit: jobs::Emit = Arc::new(|_| {}); + let (job, control) = jobs::Job::start("job-a", emit); + assert!(!control.token().is_canceled()); + assert!(jobs::cancel("job-a")); + assert!(control.token().is_canceled()); + drop(job); + assert!(!jobs::cancel("job-a")); +} + +#[test] +fn progress_events_are_thinned_but_the_ends_of_a_job_always_arrive() { + let seen = Arc::new(Mutex::new(Vec::::new())); + let sink = seen.clone(); + let emit: jobs::Emit = Arc::new(move |payload| sink.lock().expect("the log").push(payload)); + let (job, control) = jobs::Job::start("job-b", emit); + control.emit(&Event::Started { + what: "Import".to_owned(), + }); + control.begin(Phase::Converting, Unit::Items, Some(500)); + for _ in 0..500 { + control.step(Phase::Converting, 1, "page"); + } + control.emit(&Event::Finished { pages: 500 }); + drop(job); + let seen = seen.lock().expect("the log"); + let kinds: Vec<&str> = seen.iter().filter_map(|payload| payload["event"].as_str()).collect(); + assert_eq!(kinds.first(), Some(&"started")); + assert_eq!(kinds.last(), Some(&"finished")); + assert!( + kinds.iter().filter(|kind| **kind == "progress").count() < 50, + "{}", + kinds.len() + ); + assert!(seen.iter().all(|payload| payload["job"] == "job-b")); +} diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index a5e38f7e..87571c5e 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -1,29 +1,50 @@ //! The OpenNote desktop shell: the app's managed state, its commands, and the run loop (ARCHITECTURE.md section //! 8.1). `main.rs` runs the early start-up steps, then calls [`run`]. +pub mod api; pub mod appearance; pub mod args; +pub mod audio; +pub mod audio_more; pub mod boot; +#[cfg(test)] +mod build_rules; pub mod clipboard; pub mod command_list; +pub mod connectors; pub mod core_bridge; +pub mod deeplink; pub mod early; pub mod events; +pub mod hardening; pub mod images; +pub mod ink_bridge; pub mod install; pub mod instance; +pub mod intel; +pub mod interop; pub mod ipc; pub mod lifecycle; pub mod log; +pub mod notes; pub mod notes_snapshot; +pub mod page_docx; +pub mod page_export; +pub mod page_extras; pub mod paths; pub mod perf; +pub mod platform_flags; +pub mod print_job; pub mod settings; +pub mod share; pub mod shell; +pub mod shellqol; pub mod speech; pub mod spelling; pub mod state; +pub mod study; pub mod theme_tokens; +pub mod tool_windows; pub mod updater; pub mod window; pub mod zoom; @@ -51,11 +72,17 @@ pub fn run(context: EarlyContext) { guard, webview2_version, } = context; + // The crash hooks go first, so a crash while the rest starts is saved too (once the person said yes). + let hardening = hardening::Hardening::start(&paths); install::set_app_user_model_id(); + deeplink::register_protocol(); let loaded = SettingsStore::load(&paths); let (state, state_notice) = DeviceStateStore::load(&paths); perf::mark("settingsLoaded", None); let mut launch = boot::Launch { args, guard }; + deeplink::remember_launch(&launch.args.rest); + // A Markdown or text file the app was opened with, from Explorer's "Open with" (A3-18). + interop::open_files::remember_launch(&launch.args.rest); // A moved-from path that isn't the copy this one was made from is dropped, so it also gets no "Moved" notice. if let Some(old) = launch.args.moved_from.take() { if install::delete_moved_from(&paths, old.clone()) { @@ -63,6 +90,8 @@ pub fn run(context: EarlyContext) { } } let settings = loaded.store.get(); + install::uninstall::refresh_if_installed(&paths, &settings.experimental.flags); + install::cli::refresh(&paths, &settings.experimental.flags); let mut notices = loaded.notices; notices.extend(state_notice); notices.extend(launch.notices(env!("CARGO_PKG_VERSION"))); @@ -77,6 +106,14 @@ pub fn run(context: EarlyContext) { }; let hooks = lifecycle::Hooks(updater::hooks(&paths)); let pages = core_bridge::CoreBridge::new(&paths); + pages.set_retention(settings.editing.history.keep); + let audio = audio::AudioState::new(&paths); + let intel = intel::IntelState::new(&paths); + let connectors = connectors::Connectors::new(&paths); + let local_api = api::ApiState::new(&paths); + let share = share::ShareState { + inbox: share::inbox(&paths), + }; let builder = images::register_renditions(tauri::Builder::default()); let app = builder .manage(loaded.store) @@ -87,8 +124,15 @@ pub fn run(context: EarlyContext) { .manage(paths) .manage(ExitState::default()) .manage(pages) + .manage(hardening) .manage(clipboard::ClipTokens::default()) .manage(spelling::SpellService) + .manage(page_export::ExportGrants::default()) + .manage(audio) + .manage(intel) + .manage(connectors) + .manage(local_api) + .manage(share) // The instance guard holds the profile's lock, so it lives in managed state until the process exits. .manage(instance) .setup(|app| { @@ -97,6 +141,9 @@ pub fn run(context: EarlyContext) { .on_forwarded(move |forwarded| window::receive_forwarded(&handle, forwarded.args)); window::caption::init(app.handle()); window::create(app.handle())?; + shellqol::start(app.handle()); + page_extras::attach::sweep_old_copies(); + api::start(app.handle()); Ok(()) }) .invoke_handler(commands()) @@ -104,8 +151,12 @@ pub fn run(context: EarlyContext) { .expect("OpenNote failed to start"); app.run(|app, event| { if let tauri::RunEvent::Exit = event { + app.state::().stop(); + audio::shutdown(app); + page_extras::attach::shutdown(); flush_files(app); app.state::().shutdown(); + app.state::().end_clean(); } }); } @@ -121,6 +172,7 @@ pub fn flush_files(app: &tauri::AppHandle) { } /// Every command in `command_list::APP_COMMANDS`. A test keeps the two lists in step. +// checks-disable-next-line modifiability: Tauri's handler macro takes the whole list of commands in one place. fn commands() -> impl Fn(Invoke) -> bool + Send + Sync + 'static { tauri::generate_handler![ settings::commands::settings_update, @@ -137,8 +189,60 @@ fn commands() -> impl Fn(Invoke) -> bool + Send + Sync + 'static { lifecycle::app_first_paint, lifecycle::app_ready, lifecycle::app_exit_ready, + lifecycle::app_close_anyway, perf::perf_mark, crate::log::log_write, + audio::audio_assets_dir, + audio::audio_devices, + audio::audio_clock, + audio::audio_prepare, + audio::audio_begin, + audio::audio_recording_status, + audio::audio_pause_recording, + audio::audio_resume_recording, + audio::audio_switch_microphone, + audio::audio_switch_system_audio, + audio::audio_stop, + audio::audio_recover, + audio::audio_open_playback, + audio::audio_play, + audio::audio_pause_playback, + audio::audio_seek, + audio::audio_skip, + audio::audio_set_speed, + audio::audio_set_skip_silence, + audio::audio_playback_status, + audio::audio_close_playback, + audio::audio_trim_silence, + audio::audio_remove_part, + audio::audio_delete_files, + audio::audio_adopt_tracks, + audio_more::audio_split, + audio_more::audio_enhance, + audio_more::audio_compress, + audio_more::audio_storage_scan, + audio_more::audio_purge_history, + audio_more::audio_export, + audio_more::audio_export_video, + audio_more::audio_import_file, + audio_more::meeting::audio_meeting_poll, + audio_more::snap::audio_snap, + hardening::crash_consent_get, + hardening::crash_consent_set, + hardening::crash_example, + hardening::crash_list, + hardening::crash_prepare, + hardening::crash_send, + hardening::crash_delete, + hardening::crash_delete_all, + hardening::diagnostics_self_check, + hardening::diagnostics_build_feedback, + hardening::diagnostics_save_feedback, + hardening::diagnostics_startup, + hardening::diagnostics_enter_safe_mode, + hardening::diagnostics_restart, + hardening::privacy_get, + hardening::privacy_set_offline, install::install_status, install::install_pick_folder, install::install_check_folder, @@ -153,6 +257,22 @@ fn commands() -> impl Fn(Invoke) -> bool + Send + Sync + 'static { updater::updater_go_back, notes_snapshot::notes_snapshot_load, notes_snapshot::notes_snapshot_save, + notes::notes_load_initial, + notes::notes_list_notebooks, + notes::notes_list_children, + notes::notes_get, + notes::notes_create, + notes::notes_rename, + notes::notes_set_color, + notes::notes_move, + notes::notes_set_page_level, + notes::notes_trash, + notes::notes_restore, + notes::notes_list_trash, + notes::notes_restore_from_trash, + notes::notes_purge, + notes::notes_status, + notes::notes_flush, core_bridge::page_open, core_bridge::page_apply, core_bridge::page_undo, @@ -164,6 +284,11 @@ fn commands() -> impl Fn(Invoke) -> bool + Send + Sync + 'static { core_bridge::history_restore, core_bridge::history_restore_blocks, core_bridge::history_name, + core_bridge::admin::history_delete, + ink_bridge::page_add_strokes, + ink_bridge::page_read_strokes, + core_bridge::search::search_call, + shellqol::shellqol_call, clipboard::clipboard_facts, clipboard::clipboard_read, images::import::image_import, @@ -176,5 +301,70 @@ fn commands() -> impl Fn(Invoke) -> bool + Send + Sync + 'static { spelling::spell_remove_word, speech::speech_voices, speech::speech_synthesize, + page_export::print_prepare, + page_export::print_render, + page_export::print_close, + page_export::export_pick_save, + page_export::export_write, + page_export::export_open, + print_job::print_printers, + print_job::print_submit, + page_docx::export_selection_docx, + tool_windows::tool_window_open, + tool_windows::tool_windows_reset, + study::study_anki_read, + study::study_anki_write, + study::study_zotero_items, + study::study_cite_lookup, + study::study_zotero_run, + study::study_zotero_reply, + study::study_zotero_cancel, + interop::commands::interop_pick, + interop::commands::interop_detect, + interop::commands::interop_local_sources, + interop::commands::interop_preview, + interop::commands::interop_import, + interop::commands::interop_cancel, + interop::commands::interop_export, + interop::commands::interop_reveal, + interop::more::interop_more, + intel::intel_settings_get, + intel::intel_settings_set, + intel::intel_status, + intel::intel_ocr_languages, + intel::intel_ocr_recognize, + intel::intel_ink_recognize, + intel::intel_ink_tidy, + intel::intel_summarize, + intel::intel_keywords, + intel::intel_action_items, + intel::intel_chapters, + intel::intel_vocabulary_offer, + intel::intel_speech_voices, + intel::intel_speech_synthesize, + intel::intel_read_aloud_start, + intel::intel_read_aloud_next, + intel::intel_read_aloud_cancel, + intel::intel_clip_audio, + page_extras::caret::page_extras_caret, + page_extras::title::page_extras_link_title, + page_extras::attach::attachment_import, + page_extras::attach::attachment_open, + page_extras::attach::attachment_applied, + page_extras::thumbnail::attachment_thumbnail, + intel::intel_ext_call, + intel::transcribe::intel_transcribe, + intel::transcribe::intel_transcribe_cancel, + intel::stream::intel_stream_start, + intel::stream::intel_stream_stop, + intel::caption_style::intel_caption_style, + connectors::connectors_list, + connectors::connectors_connect, + connectors::connectors_cancel, + connectors::connectors_disconnect, + connectors::connectors_request, + api::api_call, + share::share_take, + connectors::accounts_call, ] } diff --git a/app/src-tauri/src/lifecycle.rs b/app/src-tauri/src/lifecycle.rs index e762c7b0..a0953453 100644 --- a/app/src-tauri/src/lifecycle.rs +++ b/app/src-tauri/src/lifecycle.rs @@ -1,18 +1,18 @@ +// checks-disable-file modifiability: one module whose parts share its state; split it when it grows again + //! The app's lifecycle after start-up (ARCHITECTURE.md sections 8.2 and 8.7): the show strategy, first paint, //! `app_ready`, and the exit handshake that every way of closing goes through (ADR 0015). use std::{ + collections::{BTreeMap, BTreeSet}, path::PathBuf, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Mutex, MutexGuard, PoisonError, - }, + sync::{Arc, Mutex, MutexGuard, PoisonError}, thread, time::Duration, }; use serde::{Deserialize, Serialize}; -use tauri::{AppHandle, Emitter, Manager}; +use tauri::{AppHandle, Emitter, Manager, WebviewWindow, WindowEvent}; use crate::{events, ipc::IpcResult, window}; @@ -98,18 +98,80 @@ pub fn timeout_for(reason: ExitReason) -> Duration { } } -/// The handshake in progress, in managed state. Only one runs at a time. +/// The handshakes in progress, in managed state: the app's exit, which asks every window whose interface listens, +/// and the closes of single windows beside the main one, which ask only that window. One exit runs at a time. #[derive(Default)] pub struct ExitState { inner: Mutex, - ui_ready: AtomicBool, } #[derive(Default)] struct ExitInner { next_id: u64, - pending: Option<(u64, ExitReason)>, + pending: Option, planned_relaunch: Option, + /// The windows whose interface has painted, so listens for the exit event. + ready: BTreeSet, + /// The windows asked to close on their own, with their handshake's id. + closing: BTreeMap, + /// What each window refused last, for its "Close anyway". + refusals: BTreeMap, + /// The windows whose own close came while the app's exit waited. The exit closes them; if it is refused, their + /// close is asked again rather than dropped. + held: BTreeSet, +} + +/// The app's exit while it waits for its windows. +struct Pending { + id: u64, + reason: ExitReason, + /// The windows that haven't answered yet. + waiting: BTreeSet, + /// A window it waited for went without answering, so not every window agreed. + lost: bool, +} + +/// What a window refused: the app's exit, or its own close. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Refusal { + App, + Window, +} + +/// An exit that began: its id and the windows to ask. No windows means no interface listens yet. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Begun { + pub id: u64, + pub windows: Vec, +} + +/// What a window's answer to `app://before-exit` does. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Answer { + /// Every window answered and none refused: the app exits. `answered` is false when a window it waited for went + /// without answering, so its pages may not have saved, and an update mustn't apply. + Exit { reason: ExitReason, answered: bool }, + /// The window agreed, and others still have to. + Wait, + /// The window refused, so the app stays open. + Refused, + /// The window agreed to close on its own. + CloseWindow, + /// The window refused to close on its own. + KeepWindow, + /// Nothing asked this window, or the question ran out of time. + Stale, +} + +/// What a close request for a window beside the main one does. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WindowClose { + /// Its interface isn't listening, so it has nothing to save: it closes now. + Now, + /// Ask it with `app://before-exit` and wait for handshake `id`. + Ask(u64), + /// It is already being asked, or the app is exiting: the request waits for that. + Hold, } impl ExitState { @@ -117,38 +179,85 @@ impl ExitState { self.inner.lock().unwrap_or_else(PoisonError::into_inner) } - /// Starts a handshake, and returns its id. Returns `None` when one is already running, except that Windows - /// ending the session takes over from a slower one. - pub fn begin(&self, reason: ExitReason) -> Option { + /// Starts the app's exit, asking those of the `open` windows whose interface listens. Returns `None` when an + /// exit is already running, except that Windows ending the session takes over from a slower one. + pub fn begin(&self, reason: ExitReason, open: impl IntoIterator) -> Option { let mut inner = self.inner(); - let taken = match inner.pending { + let taken = match &inner.pending { None => false, - Some((_, current)) => current == ExitReason::SessionEnd || reason != ExitReason::SessionEnd, + Some(current) => current.reason == ExitReason::SessionEnd || reason != ExitReason::SessionEnd, }; if taken { return None; } inner.next_id += 1; let id = inner.next_id; - inner.pending = Some((id, reason)); - Some(id) + let waiting: BTreeSet = open.into_iter().filter(|label| inner.ready.contains(label)).collect(); + let windows = waiting.iter().cloned().collect(); + inner.pending = Some(Pending { + id, + reason, + waiting, + lost: false, + }); + Some(Begun { id, windows }) + } + + /// A window answered `app://before-exit`. While the app exits, a window's answer is its answer to the exit, + /// even when it was asked to close on its own first: the exit closes it too. + pub fn answer(&self, window: &str, ok: bool) -> Answer { + let mut inner = self.inner(); + let for_exit = inner + .pending + .as_ref() + .is_some_and(|pending| pending.waiting.contains(window)); + if !for_exit { + if inner.closing.remove(window).is_none() { + return Answer::Stale; + } + if ok { + return Answer::CloseWindow; + } + inner.refusals.insert(window.to_owned(), Refusal::Window); + return Answer::KeepWindow; + } + inner.closing.remove(window); + let Some(pending) = inner.pending.as_mut() else { + return Answer::Stale; + }; + pending.waiting.remove(window); + if !ok { + inner.pending = None; + inner.refusals.insert(window.to_owned(), Refusal::App); + return Answer::Refused; + } + if !pending.waiting.is_empty() { + return Answer::Wait; + } + let (reason, answered) = (pending.reason, !pending.lost); + inner.pending = None; + Answer::Exit { reason, answered } } - /// The interface said `ok`: ends the handshake and returns why the app is closing. - pub fn accept(&self) -> Option { - self.inner().pending.take().map(|(_, reason)| reason) + /// The windows whose own close was held while an exit waited, which a refused exit asks again. A window that + /// refused the exit itself is left out: it refused its close too, and its "Close anyway" says so. + pub fn take_held(&self, refused_by: &str) -> Vec { + let mut inner = self.inner(); + let held = std::mem::take(&mut inner.held); + held.into_iter().filter(|window| window != refused_by).collect() } - /// The interface refused: the window stays open. - pub fn refuse(&self) -> Option { - self.accept() + /// What `window` refused last, which its "Close anyway" repeats. Each refusal is taken once. + pub fn take_refusal(&self, window: &str) -> Option { + self.inner().refusals.remove(window) } - /// The time ran out for handshake `id`. Returns the reason if it was still waiting. + /// The time ran out for exit `id`. Returns the reason if it was still waiting. pub fn expire(&self, id: u64) -> Option { let mut inner = self.inner(); - match inner.pending { - Some((pending, reason)) if pending == id => { + match &inner.pending { + Some(pending) if pending.id == id => { + let reason = pending.reason; inner.pending = None; Some(reason) } @@ -156,6 +265,57 @@ impl ExitState { } } + /// A window went. The exit stops waiting for it, and returns the reason when it was the last one waited for. + /// A window that went without answering didn't agree, so an exit it ends counts as unanswered (see + /// [`Answer::Exit`]), however the others answered. + pub fn forget_window(&self, window: &str) -> Option { + let mut inner = self.inner(); + inner.ready.remove(window); + inner.closing.remove(window); + inner.refusals.remove(window); + inner.held.remove(window); + let pending = inner.pending.as_mut()?; + if !pending.waiting.remove(window) { + return None; + } + pending.lost = true; + if !pending.waiting.is_empty() { + return None; + } + let reason = pending.reason; + inner.pending = None; + Some(reason) + } + + /// Someone asked to close a window beside the main one. + pub fn begin_window_close(&self, window: &str) -> WindowClose { + let mut inner = self.inner(); + if !inner.ready.contains(window) { + return WindowClose::Now; + } + if inner.pending.is_some() { + inner.held.insert(window.to_owned()); + return WindowClose::Hold; + } + if inner.closing.contains_key(window) { + return WindowClose::Hold; + } + inner.next_id += 1; + let id = inner.next_id; + inner.closing.insert(window.to_owned(), id); + WindowClose::Ask(id) + } + + /// The time ran out for a window's close `id`. True if it was still waiting, so the window closes anyway. + pub fn expire_window_close(&self, window: &str, id: u64) -> bool { + let mut inner = self.inner(); + if inner.closing.get(window) == Some(&id) { + inner.closing.remove(window); + return true; + } + false + } + /// Remembers a process to start when the next exit completes, such as the moved copy of the app. pub fn plan_relaunch(&self, relaunch: Relaunch) { self.inner().planned_relaunch = Some(relaunch); @@ -170,44 +330,222 @@ impl ExitState { self.inner().planned_relaunch.take() } - /// The page has painted, so the interface is listening for the exit event. - pub fn mark_ui_ready(&self) { - self.ui_ready.store(true, Ordering::Release); + /// The window's page has painted, so its interface is listening for the exit event. + pub fn mark_ui_ready(&self, window: &str) { + self.inner().ready.insert(window.to_owned()); + } +} + +/// What the handshakes do to the app's windows. The app's handle does it for real, and the tests record it, so the +/// wiring of closes, answers, and windows that go runs without a Tauri runtime. +pub(crate) trait Shell: Clone + Send + 'static { + /// Runs `work` on the handshakes' state. + fn exits(&self, work: impl FnOnce(&ExitState) -> R) -> R; + /// The labels of the open windows. + fn windows(&self) -> Vec; + /// Sends `app://before-exit` with `reason` to one window. + fn ask(&self, window: &str, reason: ExitReason); + /// Asks a window to close, which runs its close through [`window_close_requested`]. + fn close(&self, window: &str); + /// Closes a window at once, without asking it. + fn destroy(&self, window: &str); + /// Brings a window to the front. + fn focus(&self, window: &str); + /// Runs `work` after `delay`, off this thread. + fn after(&self, delay: Duration, work: impl FnOnce() + Send + 'static); + /// A window went: what it held in Rust is let go, such as the page sessions it left open. + fn release(&self, window: &str); + /// Saves, starts a planned process, and exits. See [`finish`]. + fn finish(&self, reason: ExitReason, answered: bool); +} + +impl Shell for AppHandle { + fn exits(&self, work: impl FnOnce(&ExitState) -> R) -> R { + work(&self.state::()) + } + + fn windows(&self) -> Vec { + self.webview_windows().into_keys().collect() + } + + fn ask(&self, window: &str, reason: ExitReason) { + if let Err(error) = self.emit_to(window, events::APP_BEFORE_EXIT, reason) { + log::warn!("Couldn't send {} to {window}: {error}", events::APP_BEFORE_EXIT); + } } - fn ui_ready(&self) -> bool { - self.ui_ready.load(Ordering::Acquire) + fn close(&self, window: &str) { + if let Some(open) = self.get_webview_window(window) { + if let Err(error) = open.close() { + log::warn!("Couldn't close the {window} window: {error}"); + } + } + } + + fn destroy(&self, window: &str) { + if let Some(open) = self.get_webview_window(window) { + if let Err(error) = open.destroy() { + log::warn!("Couldn't close the {window} window: {error}"); + } + } + } + + fn focus(&self, window: &str) { + if let Some(open) = self.get_webview_window(window) { + let _ = open.unminimize(); + let _ = open.show(); + let _ = open.set_focus(); + } + } + + fn after(&self, delay: Duration, work: impl FnOnce() + Send + 'static) { + thread::spawn(move || { + thread::sleep(delay); + work(); + }); + } + + fn release(&self, window: &str) { + if let Some(audio) = self.try_state::() { + audio.forget_window(window); + } + let Some(bridge) = self.try_state::() else { + return; + }; + // Retired now, so a window opened again with this label keeps its own sessions, while the old ones close as + // soon as the bridge is free. + let gone = bridge.window_went(window); + let app = self.clone(); + thread::spawn(move || { + let closed = app.state::().close_window(&gone); + if closed > 0 { + log::debug!("Closed {closed} page sessions the {} window left open.", gone.label); + } + }); + } + + fn finish(&self, reason: ExitReason, answered: bool) { + finish(self, reason, answered); } } -/// Starts the exit handshake (section 8.7): holds the close, tells the interface why, and waits for its answer -/// for up to 3 s (1 s when Windows is ending the session). Every way of closing comes through here. +/// Starts the exit handshake (section 8.7): holds the close, tells every window whose interface listens why, and +/// waits for all their answers for up to 3 s (1 s when Windows is ending the session). Every way of closing the +/// app comes through here. Each window flushes its own pages, so asking only the main window would lose what a +/// page window hadn't sent yet. pub fn request_exit(app: &AppHandle, reason: ExitReason) { - let state = app.state::(); - let Some(id) = state.begin(reason) else { + begin_exit(app, reason); +} + +/// [`request_exit`] on any shell. +pub(crate) fn begin_exit(shell: &S, reason: ExitReason) { + let open = shell.windows(); + let Some(begun) = shell.exits(|state| state.begin(reason, open)) else { log::debug!("Already closing, so {reason:?} was ignored."); return; }; // Before the first paint the interface has nothing to save and isn't listening. - if !state.ui_ready() { - if let Some(reason) = state.expire(id) { - finish(app, reason, false); + if begun.windows.is_empty() { + if let Some(reason) = shell.exits(|state| state.expire(begun.id)) { + shell.finish(reason, false); } return; } - if let Err(error) = app.emit_to(window::MAIN, events::APP_BEFORE_EXIT, reason) { - log::warn!("Couldn't send {}: {error}", events::APP_BEFORE_EXIT); + for label in &begun.windows { + shell.ask(label, reason); } - let app = app.clone(); - thread::spawn(move || { - thread::sleep(timeout_for(reason)); - if let Some(reason) = app.state::().expire(id) { + let timer = shell.clone(); + let id = begun.id; + shell.after(timeout_for(reason), move || { + if let Some(reason) = timer.exits(|state| state.expire(id)) { log::warn!("The interface didn't answer the exit request in time, so OpenNote exits without it."); - finish(&app, reason, false); + timer.finish(reason, false); + } + }); +} + +/// Keeps a window beside the main one (a page, quick capture, or a tool) in the handshake. Its close asks only +/// that window to save, as the app's exit asks every window, and waits for it. A window that went is no longer +/// waited for, and the page sessions it left open close. +pub fn watch_window(window: &WebviewWindow) { + let app = window.app_handle().clone(); + let label = window.label().to_owned(); + window.on_window_event(move |event| match event { + WindowEvent::CloseRequested { api, .. } => { + if window_close_requested(&app, &label) { + api.prevent_close(); + } } + WindowEvent::Destroyed => window_destroyed(&app, &label), + _ => {} }); } +/// A window beside the main one was asked to close. Returns whether the close waits: for the window's answer, +/// which closes it ([`exit_answered`]), or for the time to run out, which closes it anyway. +pub(crate) fn window_close_requested(shell: &S, label: &str) -> bool { + match shell.exits(|state| state.begin_window_close(label)) { + WindowClose::Now => false, + WindowClose::Hold => true, + WindowClose::Ask(id) => { + shell.ask(label, ExitReason::Close); + let timer = shell.clone(); + let label = label.to_owned(); + shell.after(EXIT_TIMEOUT, move || { + if timer.exits(|state| state.expire_window_close(&label, id)) { + log::warn!("The {label} window didn't answer its close in time, so it closes without it."); + timer.destroy(&label); + } + }); + true + } + } +} + +/// A window beside the main one went: what it held is let go, and the exit stops waiting for it. An exit it was +/// the last one waited for goes ahead as unanswered, as that window never saved. +pub(crate) fn window_destroyed(shell: &S, label: &str) { + shell.release(label); + if let Some(reason) = shell.exits(|state| state.forget_window(label)) { + shell.finish(reason, false); + } +} + +/// A window's answer to `app://before-exit`, as [`app_exit_ready`] gets it. +pub(crate) fn exit_answered(shell: &S, label: &str, result: &ExitResult) { + match shell.exits(|state| state.answer(label, result.ok)) { + Answer::Exit { reason, answered } => shell.finish(reason, answered), + Answer::Refused => { + log::info!( + "The {label} window kept the app open: {}", + result.reason.as_deref().unwrap_or("no reason given") + ); + shell.exits(ExitState::cancel_relaunch); + // A window closed while the exit waited had its close held for the exit, which won't come now. + for window in shell.exits(|state| state.take_held(label)) { + shell.close(&window); + } + // That window says why, with "Close anyway". The person closed another window, so it comes forward. + if label != window::MAIN { + shell.focus(label); + } + } + Answer::CloseWindow => shell.destroy(label), + Answer::KeepWindow | Answer::Wait | Answer::Stale => {} + } +} + +/// "Close anyway" in a window that refused repeats what it refused: the app's exit is asked again, from whichever +/// window refused it, and a window's own close closes that window. +pub(crate) fn close_anyway(shell: &S, label: &str) { + match shell.exits(|state| state.take_refusal(label)) { + Some(Refusal::App) => begin_exit(shell, ExitReason::Close), + Some(Refusal::Window) => shell.close(label), + None if label == window::MAIN => begin_exit(shell, ExitReason::Close), + None => shell.close(label), + } +} + /// Saves what Rust holds, starts the update's or the move's new process if there is one, and exits. `answered` /// says the interface agreed, which is what allows an update to apply. A relaunch that Rust planned itself, such /// as the moved copy, starts either way, except when Windows is ending the session. @@ -250,40 +588,40 @@ pub fn on_close_requested(app: &AppHandle) { } /// The page has painted its first frame. Under the hidden show strategy, this shows the window. +/// Every window's interface calls it, as its exit handshake listens from then on. Only the main window shows. #[tauri::command] -pub fn app_first_paint(app: AppHandle) -> IpcResult<()> { - app.state::().mark_ui_ready(); - window::show(&app); +pub fn app_first_paint(app: AppHandle, window: WebviewWindow) -> IpcResult<()> { + app.state::().mark_ui_ready(window.label()); + if window.label() == window::MAIN { + window::show(&app); + } Ok(()) } /// The last page is on screen. #[tauri::command] -pub fn app_ready(app: AppHandle, timings: ReadyTimings) -> IpcResult<()> { +pub fn app_ready(app: AppHandle, window: WebviewWindow, timings: ReadyTimings) -> IpcResult<()> { log::debug!( "Ready {} ms after the first paint", timings.page_ready_epoch_ms - timings.first_paint_epoch_ms ); - app.state::().mark_ui_ready(); + app.state::().mark_ui_ready(window.label()); app.state::().0.on_ready(&app); Ok(()) } -/// The interface's answer to `app://before-exit`: `ok` lets the exit finish, and a refusal keeps the window open. +/// A window's answer to `app://before-exit`. The app exits once every window it asked agreed, and a refusal keeps +/// it open. A window asked to close on its own closes when it agrees. #[tauri::command] -pub fn app_exit_ready(app: AppHandle, result: ExitResult) -> IpcResult<()> { - let state = app.state::(); - if result.ok { - if let Some(reason) = state.accept() { - finish(&app, reason, true); - } - } else if state.refuse().is_some() { - log::info!( - "The interface kept the window open: {}", - result.reason.as_deref().unwrap_or("no reason given") - ); - state.cancel_relaunch(); - } +pub fn app_exit_ready(app: AppHandle, window: WebviewWindow, result: ExitResult) -> IpcResult<()> { + exit_answered(&app, window.label(), &result); + Ok(()) +} + +/// "Close anyway" from the toast of a window that refused: repeats the close it refused. See [`close_anyway`]. +#[tauri::command] +pub fn app_close_anyway(app: AppHandle, window: WebviewWindow) -> IpcResult<()> { + close_anyway(&app, window.label()); Ok(()) } @@ -339,44 +677,184 @@ mod tests { } } + /// An exit state with these windows listening. + fn listening(windows: &[&str]) -> ExitState { + let state = ExitState::default(); + for window in windows { + state.mark_ui_ready(window); + } + state + } + + fn open(windows: &[&str]) -> Vec { + windows.iter().map(|window| (*window).to_owned()).collect() + } + #[test] fn runs_one_handshake_at_a_time() { - let state = ExitState::default(); - let first = state.begin(ExitReason::Close).expect("the first starts"); - assert_eq!(state.begin(ExitReason::Close), None); - assert_eq!(state.begin(ExitReason::RestartToUpdate), None); - assert_eq!(state.accept(), Some(ExitReason::Close)); + let state = listening(&["main"]); + let first = state + .begin(ExitReason::Close, open(&["main"])) + .expect("the first starts"); + assert_eq!(state.begin(ExitReason::Close, open(&["main"])), None); + assert_eq!(state.begin(ExitReason::RestartToUpdate, open(&["main"])), None); + assert_eq!( + state.answer("main", true), + Answer::Exit { + reason: ExitReason::Close, + answered: true + } + ); // After the answer, a later close may start again. - assert!(state.begin(ExitReason::Close).is_some_and(|second| second != first)); + let second = state.begin(ExitReason::Close, open(&["main"])).expect("starts again"); + assert_ne!(second.id, first.id); } #[test] fn a_refusal_keeps_the_window_and_the_old_timer_does_nothing() { - let state = ExitState::default(); - let id = state.begin(ExitReason::Close).expect("starts"); - assert_eq!(state.refuse(), Some(ExitReason::Close)); - assert_eq!(state.expire(id), None); - let later = state.begin(ExitReason::Close).expect("starts again"); - assert_eq!(state.expire(id), None, "a stale timer can't end a newer handshake"); - assert_eq!(state.expire(later), Some(ExitReason::Close)); + let state = listening(&["main"]); + let begun = state.begin(ExitReason::Close, open(&["main"])).expect("starts"); + assert_eq!(state.answer("main", false), Answer::Refused); + assert_eq!(state.expire(begun.id), None); + let later = state.begin(ExitReason::Close, open(&["main"])).expect("starts again"); + assert_eq!( + state.expire(begun.id), + None, + "a stale timer can't end a newer handshake" + ); + assert_eq!(state.expire(later.id), Some(ExitReason::Close)); } #[test] fn a_hung_interface_times_out_once() { - let state = ExitState::default(); - let id = state.begin(ExitReason::Close).expect("starts"); - assert_eq!(state.expire(id), Some(ExitReason::Close)); - assert_eq!(state.expire(id), None); - assert_eq!(state.accept(), None, "a late answer finds nothing to finish"); + let state = listening(&["main"]); + let begun = state.begin(ExitReason::Close, open(&["main"])).expect("starts"); + assert_eq!(state.expire(begun.id), Some(ExitReason::Close)); + assert_eq!(state.expire(begun.id), None); + assert_eq!( + state.answer("main", true), + Answer::Stale, + "a late answer finds nothing to finish" + ); } #[test] fn windows_ending_the_session_takes_over_from_a_slower_handshake() { + let state = listening(&["main"]); + state + .begin(ExitReason::RestartToUpdate, open(&["main"])) + .expect("starts"); + assert!(state.begin(ExitReason::SessionEnd, open(&["main"])).is_some()); + assert_eq!(state.begin(ExitReason::Close, open(&["main"])), None); + assert_eq!(state.begin(ExitReason::SessionEnd, open(&["main"])), None); + } + + #[test] + fn an_interface_that_has_not_painted_is_not_asked() { let state = ExitState::default(); - state.begin(ExitReason::RestartToUpdate).expect("starts"); - assert!(state.begin(ExitReason::SessionEnd).is_some()); - assert_eq!(state.begin(ExitReason::Close), None); - assert_eq!(state.begin(ExitReason::SessionEnd), None); + let begun = state.begin(ExitReason::Close, open(&["main"])).expect("starts"); + assert!(begun.windows.is_empty()); + } + + /// F3-3: the exit asked only the main window, so a page window's unsent typing died with the process. + #[test] + fn the_exit_asks_every_listening_window_and_waits_for_all_of_them() { + let state = listening(&["main", "page-p1", "capture"]); + let begun = state + .begin(ExitReason::Close, open(&["main", "page-p1", "capture", "tool-timers"])) + .expect("starts"); + assert_eq!( + begun.windows, + open(&["capture", "main", "page-p1"]), + "a tool window has nothing to save" + ); + assert_eq!(state.answer("main", true), Answer::Wait); + assert_eq!(state.answer("capture", true), Answer::Wait); + assert_eq!(state.answer("main", true), Answer::Stale, "one answer per window"); + assert_eq!( + state.answer("page-p1", true), + Answer::Exit { + reason: ExitReason::Close, + answered: true + } + ); + } + + #[test] + fn one_window_refusing_keeps_the_app_open() { + let state = listening(&["main", "page-p1"]); + state + .begin(ExitReason::Close, open(&["main", "page-p1"])) + .expect("starts"); + assert_eq!(state.answer("main", true), Answer::Wait); + assert_eq!(state.answer("page-p1", false), Answer::Refused); + assert!( + state.begin(ExitReason::Close, open(&["main"])).is_some(), + "a later close may start" + ); + } + + #[test] + fn a_window_that_goes_during_the_exit_is_not_waited_for() { + let state = listening(&["main", "page-p1"]); + state + .begin(ExitReason::Close, open(&["main", "page-p1"])) + .expect("starts"); + assert_eq!(state.answer("main", true), Answer::Wait); + assert_eq!(state.forget_window("page-p1"), Some(ExitReason::Close)); + assert_eq!(state.forget_window("page-p1"), None); + } + + /// F3-3: closing a page window tore its webview down at once, with the last typing still unsent. + #[test] + fn closing_a_page_window_asks_that_window_and_closes_it_when_it_agrees() { + let state = listening(&["main", "page-p1"]); + let WindowClose::Ask(id) = state.begin_window_close("page-p1") else { + panic!("a listening window is asked before it closes"); + }; + assert_eq!( + state.begin_window_close("page-p1"), + WindowClose::Hold, + "a second click waits for the first" + ); + assert_eq!(state.answer("page-p1", true), Answer::CloseWindow); + assert!( + !state.expire_window_close("page-p1", id), + "the timer finds nothing left to close" + ); + assert!(matches!(state.begin_window_close("page-p1"), WindowClose::Ask(_))); + assert_eq!(state.answer("page-p1", false), Answer::KeepWindow); + } + + #[test] + fn a_window_that_does_not_answer_its_close_closes_anyway_and_one_not_listening_closes_now() { + let state = listening(&["page-p1"]); + let WindowClose::Ask(id) = state.begin_window_close("page-p1") else { + panic!("asked"); + }; + assert!(state.expire_window_close("page-p1", id)); + assert_eq!(state.answer("page-p1", true), Answer::Stale); + assert_eq!(state.begin_window_close("tool-timers"), WindowClose::Now); + state.forget_window("page-p1"); + assert_eq!( + state.begin_window_close("page-p1"), + WindowClose::Now, + "a closed window stops listening" + ); + } + + #[test] + fn a_window_close_during_the_exit_waits_for_the_exit() { + let state = listening(&["main", "page-p1"]); + state + .begin(ExitReason::Close, open(&["main", "page-p1"])) + .expect("starts"); + assert_eq!(state.begin_window_close("page-p1"), WindowClose::Hold); + assert_eq!( + state.answer("page-p1", true), + Answer::Wait, + "its answer belongs to the exit" + ); } #[test] @@ -393,4 +871,247 @@ mod tests { state.cancel_relaunch(); assert_eq!(state.take_relaunch(), None); } + + /// A shell that records what the handshakes do, with timers the test runs by hand. + #[derive(Clone, Default)] + struct Fake(Arc); + + type Timer = Box; + + #[derive(Default)] + struct FakeInner { + state: ExitState, + open: Mutex>, + done: Mutex>, + timers: Mutex>, + } + + impl Fake { + /// The app with these windows open and listening. + fn with(windows: &[&str]) -> Fake { + let fake = Fake::default(); + for window in windows { + fake.0.state.mark_ui_ready(window); + fake.0.open.lock().unwrap().push((*window).to_owned()); + } + fake + } + + fn note(&self, what: String) { + self.0.done.lock().unwrap().push(what); + } + + /// What happened since the last call. + fn take(&self) -> Vec { + std::mem::take(&mut *self.0.done.lock().unwrap()) + } + + /// Lets the time run out for every timer started so far. + fn run_timers(&self) { + let timers = std::mem::take(&mut *self.0.timers.lock().unwrap()); + for timer in timers { + timer(); + } + } + + /// The window answers, as app_exit_ready does. + fn answer(&self, window: &str, ok: bool) { + let result = ExitResult { + ok, + reason: (!ok).then(|| "errors.recording".to_owned()), + }; + exit_answered(self, window, &result); + } + + /// The window's webview goes, as Tauri's Destroyed event says. + fn gone(&self, window: &str) { + self.0.open.lock().unwrap().retain(|open| open != window); + window_destroyed(self, window); + } + } + + impl Shell for Fake { + fn exits(&self, work: impl FnOnce(&ExitState) -> R) -> R { + work(&self.0.state) + } + fn windows(&self) -> Vec { + self.0.open.lock().unwrap().clone() + } + fn ask(&self, window: &str, reason: ExitReason) { + self.note(format!("ask {window} {reason:?}")); + } + fn close(&self, window: &str) { + self.note(format!("close {window}")); + } + fn destroy(&self, window: &str) { + self.note(format!("destroy {window}")); + } + fn focus(&self, window: &str) { + self.note(format!("focus {window}")); + } + fn after(&self, _delay: Duration, work: impl FnOnce() + Send + 'static) { + self.0.timers.lock().unwrap().push(Box::new(work)); + } + fn release(&self, window: &str) { + self.note(format!("release {window}")); + } + fn finish(&self, reason: ExitReason, answered: bool) { + self.note(format!("finish {reason:?} answered={answered}")); + } + } + + fn says(lines: &[&str]) -> Vec { + lines.iter().map(|line| (*line).to_owned()).collect() + } + + /// F3-3: a page window's close tore its webview down with its last typing unsent. The close is held, the window + /// is asked, it closes when it agrees, and what it held is let go when it has gone. + #[test] + fn a_page_window_close_asks_it_closes_it_when_it_agrees_and_lets_its_sessions_go() { + let fake = Fake::with(&["main", "page-p1"]); + assert!(window_close_requested(&fake, "page-p1"), "the close waits"); + assert_eq!(fake.take(), says(&["ask page-p1 Close"])); + assert!(window_close_requested(&fake, "page-p1"), "a second click waits too"); + assert_eq!(fake.take(), says(&[]), "and doesn't ask again"); + fake.answer("page-p1", true); + assert_eq!(fake.take(), says(&["destroy page-p1"])); + fake.gone("page-p1"); + assert_eq!(fake.take(), says(&["release page-p1"])); + fake.run_timers(); + assert_eq!(fake.take(), says(&[]), "the timer finds the close done"); + assert!( + !window_close_requested(&fake, "tool-timers"), + "a window not listening closes at once" + ); + } + + #[test] + fn a_page_window_that_does_not_answer_its_close_closes_when_the_time_runs_out() { + let fake = Fake::with(&["main", "page-p1"]); + assert!(window_close_requested(&fake, "page-p1")); + fake.take(); + fake.run_timers(); + assert_eq!(fake.take(), says(&["destroy page-p1"])); + fake.answer("page-p1", true); + assert_eq!(fake.take(), says(&[]), "a late answer finds nothing to do"); + } + + /// F3-3: the exit asked only the main window, so a page window's unsent typing died with the process. + #[test] + fn the_app_exit_asks_every_listening_window_and_exits_when_all_agree() { + let fake = Fake::with(&["main", "page-p1", "capture"]); + fake.0.open.lock().unwrap().push("tool-timers".into()); + begin_exit(&fake, ExitReason::Close); + assert_eq!( + fake.take(), + says(&["ask capture Close", "ask main Close", "ask page-p1 Close"]) + ); + fake.answer("main", true); + fake.answer("capture", true); + assert_eq!(fake.take(), says(&[])); + fake.answer("page-p1", true); + assert_eq!(fake.take(), says(&["finish Close answered=true"])); + fake.run_timers(); + assert_eq!(fake.take(), says(&[]), "the timer finds the exit done"); + } + + /// A window that died during the exit never saved, so the exit it ends mustn't count as agreed: an update + /// applies only on an agreed exit. + #[test] + fn a_window_that_dies_during_the_exit_leaves_it_unanswered() { + let fake = Fake::with(&["main", "page-p1"]); + begin_exit(&fake, ExitReason::RestartToUpdate); + fake.take(); + fake.answer("main", true); + fake.gone("page-p1"); + assert_eq!( + fake.take(), + says(&["release page-p1", "finish RestartToUpdate answered=false"]) + ); + + let fake = Fake::with(&["main", "page-p1"]); + begin_exit(&fake, ExitReason::RestartToUpdate); + fake.gone("page-p1"); + fake.take(); + fake.answer("main", true); + assert_eq!( + fake.take(), + says(&["finish RestartToUpdate answered=false"]), + "however the others answer after it" + ); + } + + /// A page window that refuses the app's exit says why in its own toast, so it comes to the front, and its + /// "Close anyway" asks for the app's exit again, not for that window alone. + #[test] + fn a_page_window_refusing_the_exit_comes_forward_and_its_close_anyway_retries_the_exit() { + let fake = Fake::with(&["main", "page-p1"]); + begin_exit(&fake, ExitReason::Close); + fake.take(); + fake.answer("main", true); + fake.answer("page-p1", false); + assert_eq!(fake.take(), says(&["focus page-p1"])); + fake.run_timers(); + assert_eq!(fake.take(), says(&[]), "the refusal ended the exit"); + close_anyway(&fake, "page-p1"); + assert_eq!(fake.take(), says(&["ask main Close", "ask page-p1 Close"])); + fake.answer("main", true); + fake.answer("page-p1", true); + assert_eq!(fake.take(), says(&["finish Close answered=true"])); + } + + /// A page window closed while the app's exit waited had its close held for the exit; when the exit was refused, + /// that close was dropped and the window stayed open. + #[test] + fn a_window_closed_while_a_refused_exit_waited_is_asked_to_close_again() { + let fake = Fake::with(&["main", "page-p1", "page-p2"]); + begin_exit(&fake, ExitReason::Close); + assert!(window_close_requested(&fake, "page-p1"), "held for the exit"); + assert!(window_close_requested(&fake, "page-p2"), "held for the exit"); + fake.take(); + fake.answer("page-p1", true); + fake.answer("page-p2", false); + assert_eq!( + fake.take(), + says(&["close page-p1", "focus page-p2"]), + "the window that refused keeps its close refused, the other is asked again" + ); + assert!(window_close_requested(&fake, "page-p1")); + assert_eq!(fake.take(), says(&["ask page-p1 Close"])); + fake.answer("page-p1", true); + assert_eq!(fake.take(), says(&["destroy page-p1"])); + } + + #[test] + fn close_anyway_after_a_window_refused_its_own_close_closes_that_window() { + let fake = Fake::with(&["main", "page-p1"]); + window_close_requested(&fake, "page-p1"); + fake.answer("page-p1", false); + assert_eq!( + fake.take(), + says(&["ask page-p1 Close"]), + "its own close: nothing comes forward" + ); + close_anyway(&fake, "page-p1"); + assert_eq!(fake.take(), says(&["close page-p1"])); + // The main window's own refusal retries the app's exit, as its caption button does. + begin_exit(&fake, ExitReason::Close); + fake.answer("main", false); + fake.take(); + close_anyway(&fake, "main"); + assert_eq!(fake.take(), says(&["ask main Close", "ask page-p1 Close"])); + } + + #[test] + fn a_window_asked_to_close_on_its_own_answers_the_exit_that_starts_meanwhile() { + let fake = Fake::with(&["main", "page-p1"]); + window_close_requested(&fake, "page-p1"); + begin_exit(&fake, ExitReason::Close); + fake.take(); + fake.answer("page-p1", true); + fake.answer("main", true); + assert_eq!(fake.take(), says(&["finish Close answered=true"])); + fake.answer("page-p1", true); + assert_eq!(fake.take(), says(&[]), "its second answer is stale"); + } } diff --git a/app/src-tauri/src/log.rs b/app/src-tauri/src/log.rs index 11c9493b..d498ea21 100644 --- a/app/src-tauri/src/log.rs +++ b/app/src-tauri/src/log.rs @@ -61,9 +61,17 @@ pub fn truncate(message: &str, max_bytes: usize) -> &str { &message[..end] } -/// Replaces the person's profile folder in `message` with `%USERPROFILE%`, so a log they share doesn't carry -/// their user name. Comparison ignores case, as Windows paths do. +/// Keeps the person's profile folder and every secret out of `message`. The profile folder becomes +/// `%USERPROFILE%`, so a log they share doesn't carry their user name. A token, an authorization code, and the +/// value after a word such as `password` or `client_secret` become ``, so a message that holds one by +/// mistake (a failed request that echoes its body, say) still leaves nothing in the file. pub fn redact(message: &str, profile: &Path) -> String { + opennote_crashreport::redact_secrets(&hide_profile(message, profile)) +} + +/// Replaces the person's profile folder in `message` with `%USERPROFILE%`. Comparison ignores case, as Windows +/// paths do. +fn hide_profile(message: &str, profile: &Path) -> String { let profile = profile.to_string_lossy(); if profile.is_empty() { return message.to_owned(); @@ -198,7 +206,27 @@ impl log::Log for FileSink { } } +/// What a panic's log line says in place of a message built at run time. +const RUN_TIME_MESSAGE: &str = "a message built at run time, left out"; + +/// The log line for a panic: where it started and, when it is a string literal in the program, its message. A +/// message built at run time can hold note content, so it is left out, as the crash report leaves it out. +fn panic_line(location: Option<&std::panic::Location<'_>>, payload: &(dyn std::any::Any + Send)) -> String { + let location = location.map_or_else( + || "an unknown place".to_owned(), + |l| format!("{}:{}:{}", l.file(), l.line(), l.column()), + ); + let message = payload + .downcast_ref::<&'static str>() + .copied() + .unwrap_or(RUN_TIME_MESSAGE); + format!("Panic: panicked at {location}: {message}") +} + /// Starts logging to `dir`, and logs panics too. Safe to call once; later calls are ignored. +/// +/// It replaces the panic hook and doesn't call the one before it. `opennote_crashreport::install` calls the hook +/// it replaces. So install it after this, or panics stop reaching the log. pub fn init(dir: &Path, profile: &Path) { let sink = FileSink { dir: dir.to_owned(), @@ -212,7 +240,9 @@ pub fn init(dir: &Path, profile: &Path) { } else { log::LevelFilter::Info }); - std::panic::set_hook(Box::new(|info| log::error!("Panic: {info}"))); + std::panic::set_hook(Box::new(|info| { + log::error!("{}", panic_line(info.location(), info.payload())); + })); } } @@ -262,6 +292,51 @@ mod tests { assert_eq!(redact("Nothing to hide", Path::new("")), "Nothing to hide"); } + #[test] + fn keeps_tokens_out_of_messages() { + // Built in pieces so secret scanners do not take the test values for real ones. + let access = format!( + "{}.{}", + concat!("ya", "29"), + "a0AfB-byCdEfGhIjKlMnOpQrStUvWxYz0123456789" + ); + let refresh = concat!("1//0g", "Zx9Qw3Er5Ty7Ui1Op4As6Df8Gh2Jk0LmN"); + let profile = Path::new(""); + for message in [ + format!("Renewing the sign-in failed: access_token={access}"), + format!("The service answered {{\"refresh_token\":\"{refresh}\"}}"), + format!("Request refused (Authorization: Bearer {access})"), + format!("Got {refresh} back"), + "Sent client_secret=abc123 to the token endpoint".to_owned(), + ] { + let logged = redact(&message, profile); + assert!( + !logged.contains("a0AfB") && !logged.contains("Zx9Qw3") && !logged.contains("abc123"), + "{logged}" + ); + assert!(logged.contains(""), "{logged}"); + } + assert_eq!( + redact("Connected the Google connector.", profile), + "Connected the Google connector." + ); + } + + #[test] + fn a_panic_line_keeps_only_a_literal_message() { + let here = std::panic::Location::caller(); + let at = format!("{}:{}:{}", here.file(), here.line(), here.column()); + let literal: Box = Box::new("index out of bounds"); + assert_eq!( + panic_line(Some(here), literal.as_ref()), + format!("Panic: panicked at {at}: index out of bounds") + ); + let built: Box = Box::new(String::from("no section named Holiday plans")); + let line = panic_line(None, built.as_ref()); + assert_eq!(line, format!("Panic: panicked at an unknown place: {RUN_TIME_MESSAGE}")); + assert!(!line.contains("Holiday")); + } + #[test] fn lets_twenty_messages_a_second_through() { let start = Instant::now(); diff --git a/app/src-tauri/src/notes/change.rs b/app/src-tauri/src/notes/change.rs new file mode 100644 index 00000000..4b43a1bf --- /dev/null +++ b/app/src-tauri/src/notes/change.rs @@ -0,0 +1,563 @@ +//! Changing the tree: create, rename, colors, moves, and page levels, with the contract's rules checked before +//! anything changes, so a call that rejects changes nothing (ARCHITECTURE.md section 12.2). + +use std::collections::HashSet; + +use opennote_core::{ + model::{Color, NotebookTree, PEN_NAMES}, + session::{ + notebook::{NodePlacement, NodeProps, NodeRef, NotebookHandle, ParentRef}, + notes::{check_title, NodeKind}, + }, + GroupId, Id, PageId, SectionId, +}; + +use super::{ + from_core, invalid_move, + tree::{block_at, container_children, depth_of, flat_pages, group_chain, height_of, Found, GROUP_DEPTH}, + CreateInput, NodeSummary, Placement, +}; +use crate::{core_bridge::Bridge, ipc::IpcError, ipc::IpcResult}; + +/// The highest page level: a sub-subpage. +const MAX_LEVEL: u8 = 2; + +fn default_title(kind: NodeKind) -> &'static str { + match kind { + NodeKind::Notebook => "Untitled notebook", + NodeKind::SectionGroup => "Untitled section group", + NodeKind::Section => "Untitled section", + NodeKind::Page => "Untitled page", + } +} + +fn parse_kind(kind: &str) -> IpcResult { + match kind { + "notebook" => Ok(NodeKind::Notebook), + "sectionGroup" => Ok(NodeKind::SectionGroup), + "section" => Ok(NodeKind::Section), + "page" => Ok(NodeKind::Page), + _ => Err(IpcError::invalid("kind", "isn't a kind of node")), + } +} + +/// A pen name as a color; anything else is no color. +pub(crate) fn pen(color: Option<&str>) -> Option { + color + .filter(|name| PEN_NAMES.contains(name)) + .map(|name| Color::Palette(name.into())) +} + +fn title_of(text: &str) -> IpcResult { + check_title(text).map_err(from_core) +} + +/// Whether a parent of this kind holds children of that kind. +fn can_hold(parent: Option, child: NodeKind) -> bool { + matches!( + (parent, child), + (None, NodeKind::Notebook) + | ( + Some(NodeKind::Notebook | NodeKind::SectionGroup), + NodeKind::SectionGroup | NodeKind::Section + ) + | (Some(NodeKind::Section), NodeKind::Page) + ) +} + +/// The containers above a node, innermost first, ending with its notebook. +pub(crate) fn ancestors(found: &Found) -> Vec { + let notebook = found.notebook(); + let tree = notebook.tree(); + let mut out = Vec::new(); + let group = match found { + Found::Notebook(_) => return out, + Found::Node(_, NodeRef::Group(g)) => tree.groups.iter().find(|x| x.id == *g).and_then(|x| x.parent), + Found::Node(_, NodeRef::Section(s)) => tree.section(*s).and_then(|x| x.group), + Found::Node(_, NodeRef::Page(p)) => { + let Some((section, _)) = tree.find_page(*p) else { + return out; + }; + out.push(section.id.to_string()); + section.group + } + }; + out.extend(group_chain(&tree, group).iter().map(ToString::to_string)); + out.push(notebook.id().to_string()); + out +} + +/// The index in `reduced` where moved nodes go: before `before`, or after the moved ones when `before` moves too. +fn insertion_index(target: &[String], reduced: &[String], before: Option<&str>, moved: &HashSet) -> usize { + let Some(before) = before else { + return reduced.len(); + }; + let anchor = if moved.contains(before) { + let at = target.iter().position(|id| id == before).unwrap_or(target.len()); + target.iter().skip(at).find(|id| !moved.contains(*id)).cloned() + } else { + Some(before.to_owned()) + }; + anchor + .and_then(|anchor| reduced.iter().position(|id| *id == anchor)) + .unwrap_or(reduced.len()) +} + +/// Fits page blocks inserted at `index` of `reduced`: each block's first page rises to at most one level below +/// the page before it. Returns each root's new level, or `None` when the page after them would be too deep. +pub(crate) fn fit_blocks(reduced: &[(String, u8)], index: usize, blocks: &[Vec<(String, u8)>]) -> Option> { + let mut previous: i32 = match index.checked_sub(1).and_then(|i| reduced.get(i)) { + Some((_, level)) => i32::from(*level), + None => -1, + }; + let mut roots = Vec::with_capacity(blocks.len()); + for block in blocks { + let (Some((_, first)), Some((_, last))) = (block.first(), block.last()) else { + continue; + }; + let first = i32::from(*first); + let delta = first.min(previous + 1) - first; + roots.push(u8::try_from(first + delta).unwrap_or(0)); + previous = i32::from(*last) + delta; + } + match reduced.get(index) { + Some((_, next)) if i32::from(*next) > previous + 1 => None, + _ => Some(roots), + } +} + +/// Whether a flat list of levels keeps the rules: it starts at 0, and each page is at most one deeper than the one +/// before it. +pub(crate) fn levels_valid(levels: &[u8]) -> bool { + let mut previous: i32 = -1; + levels.iter().all(|&level| { + let ok = level <= MAX_LEVEL && i32::from(level) <= previous + 1; + previous = i32::from(level); + ok + }) +} + +/// The pages of `flat` that start a block, with the block's pages and their levels. +pub(crate) fn blocks_of(flat: &[(String, u8)], roots: &[String]) -> Vec> { + roots + .iter() + .filter_map(|root| flat.iter().position(|(id, _)| id == root)) + .map(|index| { + block_at(flat, index) + .into_iter() + .filter_map(|id| flat.iter().find(|(x, _)| *x == id).cloned()) + .collect() + }) + .collect() +} + +/// A section's place in a notebook, as a parent for its pages. +fn section_of(found: &Found) -> Option { + match found { + Found::Node(_, NodeRef::Section(section)) => Some(*section), + _ => None, + } +} + +impl Bridge { + /// The NodeRef of a group or section of `notebook` by ID, if it is a child of `group` there. + fn container_ref(tree: &NotebookTree, id: &str) -> Option { + let parsed = Id::parse(id).ok()?; + if tree.groups.iter().any(|g| g.id.0 == parsed) { + return Some(NodeRef::Group(GroupId(parsed))); + } + tree.section(SectionId(parsed)).map(|s| NodeRef::Section(s.id)) + } + + /// Checks a `beforeId` among the children `siblings`: unknown is `not-found`, elsewhere is `invalid-move`. + fn check_before(&self, before: Option<&str>, siblings: &[String]) -> IpcResult<()> { + if let Some(before) = before { + self.find(before)?; + if !siblings.iter().any(|id| id == before) { + return Err(invalid_move( + "The node to place before isn't a child of the new parent.", + )); + } + } + Ok(()) + } + + pub(crate) fn create(&mut self, input: CreateInput) -> IpcResult { + let kind = parse_kind(&input.kind)?; + let title = title_of(input.title.as_deref().unwrap_or(default_title(kind)))?; + let parent = match input.placement.parent_id.as_deref() { + Some(id) => Some(self.find(id)?), + None => None, + }; + if !can_hold(parent.as_ref().map(Found::kind), kind) { + return Err(invalid_move("That kind of node can't go there.")); + } + let before = input.placement.before_id.as_deref(); + let color = pen(input.color.as_deref()); + match (kind, parent) { + (NodeKind::Notebook, _) => self.create_notebook(&title, before, color), + (NodeKind::Page, Some(found)) => { + let section = section_of(&found).ok_or_else(|| invalid_move("Pages go into sections."))?; + self.create_page(found.notebook(), section, &title, before, input.page_level.unwrap_or(0)) + } + (_, Some(found)) => self.create_container(&found, kind, &title, before, color), + (_, None) => Err(invalid_move("That kind of node can't go there.")), + } + } + + fn create_notebook(&mut self, title: &str, before: Option<&str>, color: Option) -> IpcResult { + let siblings: Vec = self.notebooks().iter().map(|n| n.id().to_string()).collect(); + self.check_before(before, &siblings)?; + let folder = self + .notes + .folder + .clone() + .ok_or_else(|| super::error("unavailable", "Choose a notes folder before making a notebook."))?; + std::fs::create_dir_all(&folder).map_err(IpcError::from)?; + let notebook = self.core.create_notebook(&folder, title).map_err(from_core)?; + if let Some(before) = before.and_then(|id| self.find(id).ok()) { + let path = before.notebook().path().to_path_buf(); + self.core + .move_notebook(notebook.path(), Some(&path)) + .map_err(from_core)?; + } + if color.is_some() { + notebook.set_notebook_color(color).map_err(from_core)?; + } + self.summary_of(&Found::Notebook(notebook)) + } + + fn create_container( + &mut self, + parent: &Found, + kind: NodeKind, + title: &str, + before: Option<&str>, + color: Option, + ) -> IpcResult { + let notebook = parent.notebook().clone(); + let tree = notebook.tree(); + let (at, group) = match parent { + Found::Notebook(_) => (ParentRef::Notebook, None), + Found::Node(_, NodeRef::Group(g)) => (ParentRef::Group(*g), Some(*g)), + _ => { + return Err(invalid_move( + "Groups and sections go into notebooks and section groups.", + )) + } + }; + if kind == NodeKind::SectionGroup && depth_of(&tree, group) + 1 > GROUP_DEPTH { + return Err(invalid_move("Section groups nest at most 4 deep.")); + } + self.check_before(before, &container_children(&tree, group))?; + let placement = NodePlacement { + parent: at, + before: before.and_then(|id| Bridge::container_ref(&tree, id)), + }; + let node = if kind == NodeKind::SectionGroup { + NodeRef::Group(notebook.create_group(title, placement).map_err(from_core)?) + } else { + NodeRef::Section(notebook.create_section(title, placement).map_err(from_core)?) + }; + if color.is_some() { + let props = NodeProps { + color: Some(color), + ..NodeProps::default() + }; + notebook.set_props(node, props).map_err(from_core)?; + } + self.summary_of(&Found::Node(notebook, node)) + } + + fn create_page( + &mut self, + notebook: &NotebookHandle, + section: SectionId, + title: &str, + before: Option<&str>, + level: u8, + ) -> IpcResult { + let flat = flat_pages(¬ebook.tree(), section); + let ids: Vec = flat.iter().map(|(id, _)| id.clone()).collect(); + self.check_before(before, &ids)?; + let index = before + .and_then(|b| ids.iter().position(|id| id == b)) + .unwrap_or(ids.len()); + let previous = index + .checked_sub(1) + .and_then(|i| flat.get(i)) + .map_or(-1, |(_, l)| i32::from(*l)); + let next_too_deep = flat.get(index).is_some_and(|(_, next)| *next > level.saturating_add(1)); + if level > MAX_LEVEL || i32::from(level) > previous + 1 || next_too_deep { + return Err(invalid_move("The page level breaks the rules.")); + } + let at_end = NodePlacement { + parent: ParentRef::Section(section), + before: None, + }; + let page = notebook.create_page_titled(section, at_end, title).map_err(from_core)?; + if index < flat.len() || level > 0 { + notebook + .move_page_blocks(&[(page, level)], section, index) + .map_err(from_core)?; + } + self.summary_of(&Found::Node(notebook.clone(), NodeRef::Page(page))) + } + + pub(crate) fn rename(&mut self, id: &str, title: &str) -> IpcResult { + let found = self.find(id)?; + let title = title_of(title)?; + match &found { + Found::Notebook(notebook) => notebook.rename_notebook(&title), + Found::Node(notebook, node) => notebook.rename(*node, &title), + } + .map_err(from_core)?; + // A notebook named right after it was made gets a folder of that name, not "Untitled notebook". + let found = match found { + Found::Notebook(notebook) => { + Found::Notebook(self.core.name_new_notebook_folder(¬ebook).map_err(from_core)?) + } + other => other, + }; + self.summary_of(&found) + } + + pub(crate) fn set_color(&mut self, id: &str, color: Option) -> IpcResult { + let found = self.find(id)?; + let color = pen(color.as_deref()); + match &found { + Found::Notebook(notebook) => notebook.set_notebook_color(color).map_err(from_core)?, + // A page keeps its color in its entry of the section file, as a section keeps its own. + Found::Node(notebook, node) => { + let props = NodeProps { + color: Some(color), + ..NodeProps::default() + }; + notebook.set_props(*node, props).map_err(from_core)?; + } + } + self.summary_of(&found) + } + + /// The contract's move: the nodes go before `beforeId` among the new parent's children, in the given order. + pub(crate) fn move_nodes(&mut self, ids: &[String], placement: &Placement) -> IpcResult<()> { + let mut seen = HashSet::new(); + let mut found = Vec::new(); + for id in ids { + if seen.insert(id.clone()) { + found.push(self.find(id)?); + } + } + let parent = match placement.parent_id.as_deref() { + Some(id) => Some(self.find(id)?), + None => None, + }; + let parent_kind = parent.as_ref().map(Found::kind); + for node in &found { + if !can_hold(parent_kind, node.kind()) { + return Err(invalid_move("That kind of node can't go there.")); + } + if let Some(parent) = &parent { + if parent.id() == node.id() || ancestors(parent).contains(&node.id()) { + return Err(invalid_move("A node can't move into itself.")); + } + } + } + let before = placement.before_id.as_deref(); + match parent { + None => self.move_notebooks(&found, before), + Some(parent) if parent_kind == Some(NodeKind::Section) => self.move_pages(&found, &parent, before), + Some(parent) => self.move_containers(&found, &parent, before), + } + } + + fn move_notebooks(&mut self, found: &[Found], before: Option<&str>) -> IpcResult<()> { + let target: Vec = self.notebooks().iter().map(|n| n.id().to_string()).collect(); + self.check_before(before, &target)?; + let moved: HashSet = found.iter().map(Found::id).collect(); + let reduced: Vec = target.iter().filter(|id| !moved.contains(*id)).cloned().collect(); + let index = insertion_index(&target, &reduced, before, &moved); + let anchor = match reduced.get(index) { + Some(id) => Some(self.find(id)?.notebook().path().to_path_buf()), + None => None, + }; + for node in found { + self.core + .move_notebook(node.notebook().path(), anchor.as_deref()) + .map_err(from_core)?; + } + Ok(()) + } + + fn move_containers(&mut self, found: &[Found], parent: &Found, before: Option<&str>) -> IpcResult<()> { + let target_notebook = parent.notebook().clone(); + let tree = target_notebook.tree(); + let (at, group) = match parent { + Found::Notebook(_) => (ParentRef::Notebook, None), + Found::Node(_, NodeRef::Group(g)) => (ParentRef::Group(*g), Some(*g)), + _ => { + return Err(invalid_move( + "Groups and sections go into notebooks and section groups.", + )) + } + }; + let depth = depth_of(&tree, group); + for node in found { + if let Found::Node(notebook, node_ref) = node { + if depth + height_of(¬ebook.tree(), *node_ref) > GROUP_DEPTH { + return Err(invalid_move("Section groups nest at most 4 deep.")); + } + } + } + let target = container_children(&tree, group); + self.check_before(before, &target)?; + let ids: HashSet = found.iter().map(Found::id).collect(); + let roots: Vec<&Found> = found + .iter() + .filter(|node| !ancestors(node).iter().any(|a| ids.contains(a))) + .collect(); + let moved: HashSet = roots.iter().map(|node| node.id()).collect(); + let reduced: Vec = target.iter().filter(|id| !moved.contains(*id)).cloned().collect(); + let index = insertion_index(&target, &reduced, before, &moved); + let anchor = reduced.get(index).and_then(|id| Bridge::container_ref(&tree, id)); + let placement = NodePlacement { + parent: at, + before: anchor, + }; + for root in roots { + let Found::Node(source, node) = root else { + continue; + }; + if source.id() == target_notebook.id() { + source.move_node(*node, placement).map_err(from_core)?; + } else { + source + .move_section_to_notebook(*node, &target_notebook, placement) + .map_err(from_core)?; + } + } + Ok(()) + } + + fn move_pages(&mut self, found: &[Found], parent: &Found, before: Option<&str>) -> IpcResult<()> { + let target_notebook = parent.notebook().clone(); + let section = section_of(parent).ok_or_else(|| invalid_move("Pages go into sections."))?; + let target = flat_pages(&target_notebook.tree(), section); + let target_ids: Vec = target.iter().map(|(id, _)| id.clone()).collect(); + self.check_before(before, &target_ids)?; + // Each named page with its subpages, leaving out pages inside another named page's block. + let mut blocks: Vec<(NotebookHandle, Vec<(String, u8)>)> = Vec::new(); + for node in found { + let Found::Node(notebook, NodeRef::Page(page)) = node else { + continue; + }; + let tree = notebook.tree(); + let Some((home, _)) = tree.find_page(*page) else { + continue; + }; + let flat = flat_pages(&tree, home.id); + if let Some(block) = blocks_of(&flat, &[page.to_string()]).pop() { + blocks.push((notebook.clone(), block)); + } + } + let inside = |root: &str, own: usize| { + blocks + .iter() + .enumerate() + .any(|(i, (_, block))| i != own && block.iter().skip(1).any(|(id, _)| id == root)) + }; + let blocks: Vec<(NotebookHandle, Vec<(String, u8)>)> = blocks + .iter() + .enumerate() + .filter(|(i, (_, block))| block.first().is_some_and(|(root, _)| !inside(root, *i))) + .map(|(_, block)| block.clone()) + .collect(); + let moved: HashSet = blocks + .iter() + .flat_map(|(_, b)| b.iter().map(|(id, _)| id.clone())) + .collect(); + let reduced: Vec<(String, u8)> = target.iter().filter(|(id, _)| !moved.contains(id)).cloned().collect(); + let reduced_ids: Vec = reduced.iter().map(|(id, _)| id.clone()).collect(); + let index = insertion_index(&target_ids, &reduced_ids, before, &moved); + let shapes: Vec> = blocks.iter().map(|(_, b)| b.clone()).collect(); + let levels = fit_blocks(&reduced, index, &shapes) + .ok_or_else(|| invalid_move("The page after them would be too deep."))?; + let mut roots = Vec::with_capacity(blocks.len()); + for ((source, block), level) in blocks.iter().zip(levels) { + let Some(root) = block.first().and_then(|(id, _)| PageId::parse(id).ok()) else { + continue; + }; + if source.id() != target_notebook.id() { + let at_end = NodePlacement { + parent: ParentRef::Section(section), + before: None, + }; + source + .move_to_notebook(root, &target_notebook, at_end) + .map_err(from_core)?; + } + roots.push((root, level)); + } + target_notebook + .move_page_blocks(&roots, section, index) + .map_err(from_core) + } + + /// The contract's setPageLevel: each page, with its subpages, moves to `level` in place. + pub(crate) fn set_page_level(&mut self, ids: &[String], level: u8) -> IpcResult<()> { + if level > MAX_LEVEL { + return Err(invalid_move("Pages nest at most 2 levels deep.")); + } + let mut pages: Vec<(NotebookHandle, PageId)> = Vec::new(); + let mut seen = HashSet::new(); + for id in ids { + if !seen.insert(id.clone()) { + continue; + } + match self.find(id)? { + Found::Node(notebook, NodeRef::Page(page)) => pages.push((notebook, page)), + _ => return Err(invalid_move("Only pages have levels.")), + } + } + // Check every section first, as the reference model does, so a rejection changes nothing. + let mut sections: Vec<(NotebookHandle, SectionId, Vec)> = Vec::new(); + for (notebook, page) in &pages { + let tree = notebook.tree(); + let Some((home, _)) = tree.find_page(*page) else { + continue; + }; + match sections.iter_mut().find(|(_, s, _)| *s == home.id) { + Some((_, _, list)) => list.push(*page), + None => sections.push((notebook.clone(), home.id, vec![*page])), + } + } + for (notebook, section, list) in §ions { + let flat = flat_pages(¬ebook.tree(), *section); + let mut levels: Vec = flat.iter().map(|(_, l)| *l).collect(); + let mut targets: Vec = list + .iter() + .filter_map(|p| flat.iter().position(|(id, _)| *id == p.to_string())) + .collect(); + targets.sort_unstable(); + for index in targets { + let shaped: Vec<(String, u8)> = flat.iter().zip(&levels).map(|((id, _), l)| (id.clone(), *l)).collect(); + let block = block_at(&shaped, index); + let delta = i32::from(level) - i32::from(levels.get(index).copied().unwrap_or(0)); + for id in block { + if let Some(i) = flat.iter().position(|(x, _)| *x == id) { + if let Some(slot) = levels.get_mut(i) { + *slot = u8::try_from(i32::from(*slot) + delta).unwrap_or(u8::MAX); + } + } + } + } + if !levels_valid(&levels) { + return Err(invalid_move("The page levels would break the rules.")); + } + } + for (notebook, _, list) in sections { + notebook.set_page_level(&list, level).map_err(from_core)?; + } + Ok(()) + } +} diff --git a/app/src-tauri/src/notes/events.rs b/app/src-tauri/src/notes/events.rs new file mode 100644 index 00000000..0a2e7486 --- /dev/null +++ b/app/src-tauri/src/notes/events.rs @@ -0,0 +1,176 @@ +//! The notes events: after each command, and after tree changes the core makes on its own, the bridge compares +//! the library and each notebook's tree with what it last sent and sends `upserted`, `removed`, and +//! `childrenChanged` events (crates/core/src/session/tree_events.rs). Applying them is idempotent. + +use std::collections::HashMap; + +use opennote_core::{ + model::NotebookTree, + session::{ + notes::{all_nodes, notebook_node}, + tree_events::{diff, TreeEvent}, + }, +}; +use serde_json::{json, Value}; + +use super::{tree::summary, NOTES_EVENT}; +use crate::core_bridge::Bridge; + +/// What the interface last heard: the notebooks in order and their trees. +pub(crate) struct Seen { + order: Vec, + trees: HashMap, +} + +fn event_json(event: TreeEvent) -> Value { + match event { + TreeEvent::Upserted { nodes } => { + let nodes: Vec<_> = nodes.into_iter().map(summary).collect(); + json!({ "type": "upserted", "nodes": nodes }) + } + TreeEvent::Removed { ids } => json!({ "type": "removed", "ids": ids }), + TreeEvent::ChildrenChanged { parent_id } => json!({ "type": "childrenChanged", "parentId": parent_id }), + TreeEvent::Reset => json!({ "type": "reset" }), + } +} + +/// A page moved between notebooks is removed from the old one and upserted in the new one. Which comes first +/// depends on library order, and a removal after the upsert would delete the node just added, so ids that +/// this pass also upserts are not reported as removed. +fn drop_moved_removals(events: &mut Vec) { + let upserted: std::collections::HashSet = events + .iter() + .filter_map(|event| match event { + TreeEvent::Upserted { nodes } => Some(nodes.iter().map(|n| n.id.clone())), + _ => None, + }) + .flatten() + .collect(); + if upserted.is_empty() { + return; + } + for event in events.iter_mut() { + if let TreeEvent::Removed { ids } = event { + ids.retain(|id| !upserted.contains(id)); + } + } + events.retain(|event| !matches!(event, TreeEvent::Removed { ids } if ids.is_empty())); +} + +impl Bridge { + /// Sends the events that take the interface from what it last heard to the tree as it is now. + pub(crate) fn send_tree_events(&mut self) { + let notebooks = self.notebooks(); + let now = Seen { + order: notebooks.iter().map(|n| n.id().to_string()).collect(), + trees: notebooks.iter().map(|n| (n.id().to_string(), n.tree())).collect(), + }; + let Some(old) = self.notes.seen.replace(now) else { + return; + }; + let Some(now) = self.notes.seen.as_ref() else { + return; + }; + let mut events = Vec::new(); + for (id, tree) in &old.trees { + if !now.trees.contains_key(id) { + let ids: Vec = all_nodes(tree).into_iter().map(|n| n.id).collect(); + events.push(TreeEvent::Removed { ids }); + } + } + for id in &now.order { + let Some(tree) = now.trees.get(id) else { + continue; + }; + match old.trees.get(id) { + Some(before) => events.extend(diff(before, tree)), + None => events.push(TreeEvent::Upserted { + nodes: vec![notebook_node(tree)], + }), + } + } + if old.order != now.order { + events.push(TreeEvent::ChildrenChanged { parent_id: None }); + } + drop_moved_removals(&mut events); + for event in events { + self.relay.send(NOTES_EVENT, event_json(event)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use opennote_core::session::notes::{NodeInfo, NodeKind}; + + fn page(id: &str) -> NodeInfo { + NodeInfo { + id: id.into(), + kind: NodeKind::Page, + parent_id: None, + title: String::new(), + color: None, + page_level: 0, + child_count: 0, + created: Default::default(), + modified: Default::default(), + read_only: false, + pinned: false, + archived: false, + encrypted: false, + protection: opennote_core::session::notes::Protection::None, + } + } + + fn removed(ids: &[&str]) -> TreeEvent { + TreeEvent::Removed { + ids: ids.iter().map(|id| id.to_string()).collect(), + } + } + + #[test] + fn a_move_to_an_earlier_notebook_keeps_the_upsert() { + // The new notebook's upsert comes first, the old notebook's removal after it. + let mut events = vec![ + TreeEvent::Upserted { + nodes: vec![page("p1")], + }, + removed(&["p1", "p2"]), + ]; + drop_moved_removals(&mut events); + assert_eq!( + events, + vec![ + TreeEvent::Upserted { + nodes: vec![page("p1")] + }, + removed(&["p2"]) + ] + ); + } + + #[test] + fn a_move_to_a_later_notebook_drops_the_whole_removal() { + let mut events = vec![ + removed(&["p1"]), + TreeEvent::Upserted { + nodes: vec![page("p1")], + }, + ]; + drop_moved_removals(&mut events); + assert_eq!( + events, + vec![TreeEvent::Upserted { + nodes: vec![page("p1")] + }] + ); + } + + #[test] + fn a_plain_removal_is_untouched() { + let mut events = vec![removed(&["p1"])]; + drop_moved_removals(&mut events); + assert_eq!(events, vec![removed(&["p1"])]); + } +} diff --git a/app/src-tauri/src/notes/folders.rs b/app/src-tauri/src/notes/folders.rs new file mode 100644 index 00000000..19fb958f --- /dev/null +++ b/app/src-tauri/src/notes/folders.rs @@ -0,0 +1,34 @@ +//! Renaming a notebook renames its folder (A4-4), when the person keeps "Rename folders with notebooks" on. The core +//! closes the notebook, renames its folder to a safe, free name, and opens it again (library_api.rs). Pages of the +//! notebook that were open must open again, so the interface is told which. + +use opennote_core::PageId; + +use super::{from_core, tree::Found}; +use crate::{core_bridge::Bridge, ipc::IpcResult}; + +/// The key in the shell's own choices (qol.json) for "Rename folders with notebooks". Absent means on. +pub const PREF_KEY: &str = "renameNotebookFolders"; + +/// Whether the choice, as read from qol.json, keeps folder renames on. +pub fn wanted(prefs: &serde_json::Map) -> bool { + prefs.get(PREF_KEY).and_then(serde_json::Value::as_bool).unwrap_or(true) +} + +impl Bridge { + /// Renames the folder of the notebook `id` after its title. Returns the pages that were open in it, which the + /// interface opens again. Anything that isn't a notebook is left alone. + pub(crate) fn rename_notebook_folder(&mut self, id: &str) -> IpcResult> { + let Found::Notebook(notebook) = self.find(id)? else { + return Ok(Vec::new()); + }; + let path = notebook.path().to_path_buf(); + let book = notebook.id(); + drop(notebook); + let moved = self.core.rename_notebook_folder(&path).map_err(from_core)?; + if moved == path { + return Ok(Vec::new()); + } + Ok(self.forget_notebook_pages(book)) + } +} diff --git a/app/src-tauri/src/notes/migrate.rs b/app/src-tauri/src/notes/migrate.rs new file mode 100644 index 00000000..07d7b00f --- /dev/null +++ b/app/src-tauri/src/notes/migrate.rs @@ -0,0 +1,580 @@ +//! Moves a beta 1 profile into real notebooks, once, at the first start of a later build. +//! +//! Beta 1 kept the tree in the Phase 2 snapshot, `%LOCALAPPDATA%\OpenNote\phase2-notes.json`, and page content in +//! one notebook of its own, `Pages`, in the notes folder (or under `phase4` for an older profile), with a map from +//! the tree's page IDs to that notebook's pages in `phase4\pages.json`. The migration makes a notebook folder in +//! the notes folder for each notebook of the snapshot, with its section groups, sections, and pages in order, moves +//! each page's content, history, and images from `Pages` into its new place, and rebuilds the snapshot's Trash. +//! +//! First it copies the old files to `%LOCALAPPDATA%\OpenNote\beta-1-backup` and moves the snapshot and the map +//! there, and writes a marker, `beta-1-migration.json`, that names the backup and every notebook made so far. A +//! migration stopped part way (a crash, a power cut, the Pages notebook failing to open) runs again from the +//! backup at the next start: it removes the notebooks the stopped run made, puts the Pages notebook back from the +//! backup, and starts over. The marker goes once the migration finished. Then it removes the old `Pages` notebook. + +use std::{ + collections::{BTreeMap, HashMap}, + fs, io, + path::{Path, PathBuf}, +}; + +use opennote_core::{ + session::{ + notebook::{NodePlacement, NodeProps, NodeRef, NotebookHandle, ParentRef}, + notes::check_title, + }, + CoreError, PageId, SectionId, +}; +use serde::{Deserialize, Serialize}; + +use super::{change::pen, NOTEBOOK_FILE}; +use crate::core_bridge::Bridge; + +/// The Phase 2 snapshot's file, in this device's files. +pub const SNAPSHOT_FILE: &str = "phase2-notes.json"; + +/// Where the migration keeps the old files. +pub const BACKUP_DIR: &str = "beta-1-backup"; + +/// The marker of a migration that started and hasn't finished, in this device's files. +pub const MARKER_FILE: &str = "beta-1-migration.json"; + +/// The marker's content: where the backup is, where the notebooks go, where the Pages notebook was, and the +/// notebooks made so far. +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Marker { + backup: PathBuf, + folder: PathBuf, + old_dir: Option, + #[serde(default)] + made: Vec, +} + +impl Marker { + fn read(root: &Path) -> Option { + let text = fs::read_to_string(root.join(MARKER_FILE)).ok()?; + serde_json::from_str(&text).ok() + } + + fn write(&self, root: &Path) -> io::Result<()> { + let json = serde_json::to_vec_pretty(self).map_err(io::Error::other)?; + let temp = root.join(format!("{MARKER_FILE}.tmp")); + fs::write(&temp, json)?; + fs::rename(&temp, root.join(MARKER_FILE)) + } +} + +#[cfg(test)] +thread_local! { + /// Tests stop a migration after this many notebooks, as a crash would. + pub(crate) static STOP_AFTER: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// The bridge notebook of beta 1. +const OLD_NOTEBOOK: &str = "Pages"; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Snapshot { + #[serde(default)] + folder: String, + #[serde(default)] + notebooks: Vec, + #[serde(default)] + trash: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct SnapNode { + id: String, + kind: String, + #[serde(default)] + title: String, + #[serde(default)] + color: Option, + #[serde(default)] + page_level: Option, + #[serde(default)] + children: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct SnapTrash { + #[serde(default)] + parent_id: Option, + #[serde(default)] + nodes: Vec, +} + +/// What a migration did, written beside the backup. +#[derive(Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Report { + pub notebooks: u32, + pub sections: u32, + pub pages: u32, + /// Pages whose content moved from the old `Pages` notebook. + pub pages_with_content: u32, + pub trash_items: u32, + pub problems: Vec, +} + +/// Where a node of the snapshot went: its notebook, and what its children go into. +#[derive(Clone)] +struct Made { + notebook: NotebookHandle, + parent: ParentRef, +} + +struct Migration<'a> { + bridge: &'a mut Bridge, + /// The marker of a real migration, which records each notebook made; none for the sample library. + marker: Option<(PathBuf, Marker)>, + folder: PathBuf, + old: Option, + map: BTreeMap, + made: HashMap, + report: Report, +} + +fn count(n: &mut u32) { + *n = n.saturating_add(1); +} + +fn clean_title(title: &str, fallback: &str) -> String { + check_title(title).unwrap_or_else(|_| fallback.to_owned()) +} + +/// Copies a folder with everything in it. +pub(crate) fn copy_dir(from: &Path, to: &Path) -> io::Result<()> { + fs::create_dir_all(to)?; + for entry in fs::read_dir(from)? { + let entry = entry?; + let target = to.join(entry.file_name()); + if entry.file_type()?.is_dir() { + copy_dir(&entry.path(), &target)?; + } else { + fs::copy(entry.path(), target)?; + } + } + Ok(()) +} + +/// A backup folder that isn't there yet. +fn fresh_backup(root: &Path) -> PathBuf { + let first = root.join(BACKUP_DIR); + if !first.exists() { + return first; + } + (2..1000) + .map(|n| root.join(format!("{BACKUP_DIR} ({n})"))) + .find(|path| !path.exists()) + .unwrap_or_else(|| root.join(format!("{BACKUP_DIR} (last)"))) +} + +/// Migrates a beta 1 profile under `bridge.root`, if there is one, into notebooks in `folder` (or the snapshot's +/// folder). Problems go to the log and the report; the notes that could move have moved. +pub(crate) fn run(bridge: &mut Bridge, folder: Option<&Path>) -> Option { + let root = bridge.root.clone(); + let result = if let Some(marker) = Marker::read(&root) { + ::log::warn!("A beta 1 migration stopped part way; running it again from the backup"); + resume(bridge, &root, marker) + } else if root.join(SNAPSHOT_FILE).is_file() { + migrate(bridge, &root, folder) + } else { + return None; + }; + match result { + Ok(report) => { + ::log::info!("Moved the beta 1 notes into notebooks: {report:?}"); + Some(report) + } + Err(error) => { + ::log::error!("Couldn't move the beta 1 notes into notebooks: {error}"); + None + } + } +} + +fn migrate(bridge: &mut Bridge, root: &Path, folder: Option<&Path>) -> Result { + let snapshot_file = root.join(SNAPSHOT_FILE); + let text = fs::read_to_string(&snapshot_file).map_err(|e| format!("reading the snapshot: {e}"))?; + let snapshot: Snapshot = serde_json::from_str(&text).map_err(|e| format!("reading the snapshot: {e}"))?; + let folder = folder + .map(Path::to_path_buf) + .or_else(|| (!snapshot.folder.is_empty()).then(|| PathBuf::from(&snapshot.folder))) + .ok_or("there is no notes folder yet")?; + let map_file = root.join("phase4").join("pages.json"); + let map: BTreeMap = match fs::read_to_string(&map_file) { + Ok(text) => serde_json::from_str(&text).map_err(|e| format!("reading the page map: {e}"))?, + Err(error) if error.kind() == io::ErrorKind::NotFound => BTreeMap::new(), + Err(error) => return Err(format!("reading the page map: {error}")), + }; + let old_dir = [folder.join(OLD_NOTEBOOK), root.join("phase4").join(OLD_NOTEBOOK)] + .into_iter() + .find(|dir| dir.join(NOTEBOOK_FILE).is_file()); + + // The backup first. The snapshot and the map move into it, and the marker says to finish from there. + let backup = fresh_backup(root); + fs::create_dir_all(&backup).map_err(|e| format!("making the backup folder: {e}"))?; + if let Some(dir) = &old_dir { + copy_dir(dir, &backup.join(OLD_NOTEBOOK)).map_err(|e| format!("backing up the Pages notebook: {e}"))?; + } + if map_file.is_file() { + fs::copy(&map_file, backup.join("pages.json")).map_err(|e| format!("backing up the page map: {e}"))?; + } + fs::copy(&snapshot_file, backup.join(SNAPSHOT_FILE)).map_err(|e| format!("backing up the snapshot: {e}"))?; + let marker = Marker { + backup, + folder, + old_dir, + made: Vec::new(), + }; + marker + .write(root) + .map_err(|e| format!("writing the migration marker: {e}"))?; + let _ = fs::remove_file(&snapshot_file); + let _ = fs::remove_file(&map_file); + rebuild(bridge, root, &snapshot, map, marker) +} + +/// Runs a stopped migration again from its backup: the notebooks it made go, and the Pages notebook comes back +/// as it was. +fn resume(bridge: &mut Bridge, root: &Path, mut marker: Marker) -> Result { + for path in std::mem::take(&mut marker.made) { + let _ = bridge.core.remove_notebook(&path); + let _ = bridge.core.forget_notebook(&path); + if path.exists() { + fs::remove_dir_all(&path).map_err(|e| format!("removing a notebook of the stopped run: {e}"))?; + } + } + let kept = marker.backup.join(OLD_NOTEBOOK); + if let Some(dir) = marker.old_dir.as_ref().filter(|_| kept.join(NOTEBOOK_FILE).is_file()) { + if dir.exists() { + fs::remove_dir_all(dir).map_err(|e| format!("removing the moved Pages notebook: {e}"))?; + } + copy_dir(&kept, dir).map_err(|e| format!("putting the Pages notebook back: {e}"))?; + } + marker + .write(root) + .map_err(|e| format!("writing the migration marker: {e}"))?; + let text = + fs::read_to_string(marker.backup.join(SNAPSHOT_FILE)).map_err(|e| format!("reading the snapshot: {e}"))?; + let snapshot: Snapshot = serde_json::from_str(&text).map_err(|e| format!("reading the snapshot: {e}"))?; + let map: BTreeMap = match fs::read_to_string(marker.backup.join("pages.json")) { + Ok(text) => serde_json::from_str(&text).map_err(|e| format!("reading the page map: {e}"))?, + Err(_) => BTreeMap::new(), + }; + rebuild(bridge, root, &snapshot, map, marker) +} + +/// Makes the notebooks of the snapshot. The marker stays until this finished, so a stop part way runs it again. +fn rebuild( + bridge: &mut Bridge, + root: &Path, + snapshot: &Snapshot, + map: BTreeMap, + marker: Marker, +) -> Result { + let (folder, old_dir, backup) = (marker.folder.clone(), marker.old_dir.clone(), marker.backup.clone()); + // A run that can't start says why beside the backup, and runs again at the next start. + let failed = |problem: String| { + let report = Report { + problems: vec![problem.clone()], + ..Report::default() + }; + if let Ok(json) = serde_json::to_vec_pretty(&report) { + let _ = fs::write(backup.join("migration.json"), json); + } + problem + }; + fs::create_dir_all(&folder).map_err(|e| failed(format!("making the notes folder: {e}")))?; + let old = match &old_dir { + Some(dir) => Some( + bridge + .core + .open_notebook(dir) + .map_err(|e| failed(format!("opening the Pages notebook: {e}")))?, + ), + None => None, + }; + let mut migration = Migration { + bridge, + marker: Some((root.to_path_buf(), marker)), + folder, + old, + map, + made: HashMap::new(), + report: Report::default(), + }; + for notebook in &snapshot.notebooks { + #[cfg(test)] + if STOP_AFTER.get().is_some_and(|stop| migration.report.notebooks >= stop) { + return Err("stopped by the test".into()); + } + if let Err(error) = migration.notebook(notebook) { + migration + .report + .problems + .push(format!("notebook {}: {error}", notebook.title)); + } + } + for item in &snapshot.trash { + if let Err(error) = migration.trash_item(item) { + migration.report.problems.push(format!("a Trash item: {error}")); + } + } + migration.remove_old(); + let report = migration.report; + if let Ok(json) = serde_json::to_vec_pretty(&report) { + let _ = fs::write(backup.join("migration.json"), json); + } + let _ = fs::remove_file(root.join(MARKER_FILE)); + Ok(report) +} + +/// The sample library of development and test builds (app/src/services/notes/fixtures.ts, `sample`). +pub(crate) const SAMPLE: &str = include_str!("sample.json"); + +/// The environment variable that asks a development or test build to start an empty library with the samples. +pub const SEED_VARIABLE: &str = "OPENNOTE_NOTES_SEED"; + +/// Makes the sample library in `folder` when a development or end-to-end test build asks for it with +/// `OPENNOTE_NOTES_SEED=sample` and the library is empty, as the in-memory service's dev channel did. +pub(crate) fn seed_sample(bridge: &mut Bridge, folder: &Path) { + if !seeding() || !bridge.core.library().notebooks.is_empty() { + return; + } + if let Err(error) = make_library(bridge, folder, SAMPLE) { + ::log::warn!("Couldn't make the sample library: {error}"); + } +} + +/// Whether this is a development or test build that was asked for the sample library. +pub(crate) fn seeding() -> bool { + let asked = std::env::var(SEED_VARIABLE).is_ok_and(|seed| seed == "sample"); + asked && (cfg!(debug_assertions) || cfg!(feature = "test-endpoints")) +} + +/// Makes the notebooks of snapshot-shaped JSON in `folder`. +pub(crate) fn make_library(bridge: &mut Bridge, folder: &Path, json: &str) -> Result { + let snapshot: Snapshot = serde_json::from_str(json).map_err(|e| e.to_string())?; + fs::create_dir_all(folder).map_err(|e| e.to_string())?; + let mut migration = Migration { + bridge, + marker: None, + folder: folder.to_path_buf(), + old: None, + map: BTreeMap::new(), + made: HashMap::new(), + report: Report::default(), + }; + for notebook in &snapshot.notebooks { + migration.notebook(notebook).map_err(|e| e.to_string())?; + } + Ok(migration.report) +} + +impl Migration<'_> { + fn notebook(&mut self, node: &SnapNode) -> Result { + let title = clean_title(&node.title, "Untitled notebook"); + let notebook = self.bridge.core.create_notebook(&self.folder, &title)?; + if let Some((root, marker)) = &mut self.marker { + marker.made.push(notebook.path().to_path_buf()); + if let Err(error) = marker.write(root) { + self.report + .problems + .push(format!("writing the migration marker: {error}")); + } + } + if let Some(color) = pen(node.color.as_deref()) { + notebook.set_notebook_color(Some(color))?; + } + count(&mut self.report.notebooks); + self.made.insert( + node.id.clone(), + Made { + notebook: notebook.clone(), + parent: ParentRef::Notebook, + }, + ); + for child in &node.children { + self.child(¬ebook, ParentRef::Notebook, child)?; + } + Ok(notebook) + } + + /// A section group or section, at the end of `parent`. + fn child(&mut self, notebook: &NotebookHandle, parent: ParentRef, node: &SnapNode) -> Result { + let at = NodePlacement { parent, before: None }; + let made = if node.kind == "sectionGroup" { + let group = notebook.create_group(&clean_title(&node.title, "Untitled section group"), at)?; + for child in &node.children { + self.child(notebook, ParentRef::Group(group), child)?; + } + NodeRef::Group(group) + } else { + let section = notebook.create_section(&clean_title(&node.title, "Untitled section"), at)?; + count(&mut self.report.sections); + self.pages(notebook, section, &node.children)?; + NodeRef::Section(section) + }; + if let Some(color) = pen(node.color.as_deref()) { + let props = NodeProps { + color: Some(Some(color)), + ..NodeProps::default() + }; + notebook.set_props(made, props)?; + } + let parent = match made { + NodeRef::Group(group) => ParentRef::Group(group), + NodeRef::Section(section) => ParentRef::Section(section), + NodeRef::Page(page) => ParentRef::Page(page), + }; + self.made.insert( + node.id.clone(), + Made { + notebook: notebook.clone(), + parent, + }, + ); + Ok(made) + } + + /// Pages at the end of a section, in order, then their levels. Returns the first page. + fn pages( + &mut self, + notebook: &NotebookHandle, + section: SectionId, + pages: &[SnapNode], + ) -> Result, CoreError> { + let mut made = Vec::new(); + for page in pages.iter().filter(|p| p.kind == "page") { + made.push((self.page(notebook, section, page)?, page.page_level.unwrap_or(0))); + } + // Each page in order: the pages after it are still at level 0, so none moves with it. + for (page, level) in &made { + if *level > 0 { + if let Err(error) = notebook.set_page_level(&[*page], *level) { + self.report.problems.push(format!("a page level: {error}")); + } + } + } + Ok(made.first().map(|(page, _)| *page)) + } + + /// One page at the end of a section: the old page with its content when beta 1 kept one, else a new page. + /// The heading typed on the page is the title, else the name in the list. + fn page(&mut self, notebook: &NotebookHandle, section: SectionId, node: &SnapNode) -> Result { + let title = clean_title(&node.title, "Untitled page"); + let at = NodePlacement { + parent: ParentRef::Section(section), + before: None, + }; + count(&mut self.report.pages); + let old_page = self.map.get(&node.id).and_then(|id| PageId::parse(id).ok()); + if let (Some(old), Some(page)) = (&self.old, old_page) { + let moved = match old.tree().find_page(page) { + Some(_) => old.move_to_notebook(page, notebook, at).map(|_| true), + None => Ok(false), + }; + if let Err(error) = &moved { + // The content stays in the backup; the page still gets its place and name. + self.report + .problems + .push(format!("the content of {}: {error}", node.title)); + } + if moved.unwrap_or(false) { + count(&mut self.report.pages_with_content); + let heading = notebook + .tree() + .find_page(page) + .map(|(_, p)| p.title.clone()) + .unwrap_or_default(); + if heading.trim().is_empty() { + notebook.rename(NodeRef::Page(page), &title)?; + } + self.made.insert( + node.id.clone(), + Made { + notebook: notebook.clone(), + parent: ParentRef::Page(page), + }, + ); + return Ok(page); + } + } + let page = notebook.create_page_titled(section, at, &title)?; + self.made.insert( + node.id.clone(), + Made { + notebook: notebook.clone(), + parent: ParentRef::Page(page), + }, + ); + Ok(page) + } + + /// A Trash item of the snapshot: made again where it was, then moved to Trash. + fn trash_item(&mut self, item: &SnapTrash) -> Result<(), String> { + let Some(root) = item.nodes.first() else { + return Ok(()); + }; + if root.kind == "notebook" { + let notebook = self.notebook(root).map_err(|e| e.to_string())?; + let path = notebook.path().to_path_buf(); + self.bridge.core.remove_notebook(&path).map_err(|e| e.to_string())?; + count(&mut self.report.trash_items); + return Ok(()); + } + let parent = item.parent_id.as_ref().and_then(|id| self.made.get(id)).cloned(); + let Some(Made { notebook, parent }) = parent else { + self.report.problems.push(format!("{}: its place is gone", root.title)); + return Ok(()); + }; + let made = match (root.kind.as_str(), parent) { + ("page", ParentRef::Section(section)) => { + let pages = self.pages(¬ebook, section, &item.nodes).map_err(|e| e.to_string())?; + pages.map(NodeRef::Page) + } + ("page", _) => None, + (_, parent) => Some(self.child(¬ebook, parent, root).map_err(|e| e.to_string())?), + }; + if let Some(made) = made { + notebook.delete(&[made]).map_err(|e| e.to_string())?; + count(&mut self.report.trash_items); + } + Ok(()) + } + + /// Takes the old `Pages` notebook out of the library and deletes its folder, which the backup keeps. + fn remove_old(&mut self) { + let Some(old) = self.old.take() else { + return; + }; + let path = old.path().to_path_buf(); + if let Err(error) = self.bridge.core.remove_notebook(&path) { + self.report + .problems + .push(format!("closing the Pages notebook: {error}")); + return; + } + match fs::remove_dir_all(&path) { + Ok(()) => { + if let Err(error) = self.bridge.core.forget_notebook(&path) { + self.report + .problems + .push(format!("forgetting the Pages notebook: {error}")); + } + } + // It stays in Trash on this device, so it doesn't show as a notebook. + Err(error) => self + .report + .problems + .push(format!("deleting the Pages notebook: {error}")), + } + } +} diff --git a/app/src-tauri/src/notes/mod.rs b/app/src-tauri/src/notes/mod.rs new file mode 100644 index 00000000..7d3bc3d2 --- /dev/null +++ b/app/src-tauri/src/notes/mod.rs @@ -0,0 +1,495 @@ +//! The notes commands: the Phase 2 notes contract (app/src/services/notes/types.ts, ADR 0014) served by the core, +//! in the notes folder that setup chose. Each notebook is a folder there in the note format, with its sections, +//! pages, Trash, and history inside it, so the notes folder holds everything that is part of the notes. Only the +//! library's order of notebooks and the core's journals stay on this device. +//! +//! Node IDs are the core's IDs. Errors carry the contract's codes (`not-found`, `invalid-name`, `invalid-move`, +//! `read-only`, `conflict`, `unavailable`, `io`); an `invalid-name` error names its reason in `field`. + +pub mod change; +pub mod events; +pub mod folders; +pub mod migrate; +pub mod purged; +pub mod trash; +pub mod tree; + +#[cfg(test)] +mod tests; + +use std::{ + collections::{BTreeMap, HashMap}, + fs, + path::{Path, PathBuf}, +}; + +use opennote_core::{ + session::notes::{error_code, invalid_name_reason, SaveStatus}, + CoreError, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tauri::{AppHandle, State}; + +use crate::{ + core_bridge::{run_notes, Bridge, CoreBridge}, + ipc::{IpcError, IpcResult}, +}; + +/// The event that carries one notes event to the interface. +pub const NOTES_EVENT: &str = "notes:event"; + +/// The file in a folder that makes it a notebook (docs/format/README.md section 3.2). +pub const NOTEBOOK_FILE: &str = "notebook.json"; + +/// A node as the contract's `NodeSummary` describes it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NodeSummary { + pub id: String, + pub kind: &'static str, + pub parent_id: Option, + pub title: String, + /// A pen name, or none. Always none for pages. + pub color: Option, + pub page_level: u8, + pub child_count: u32, + pub created: String, + pub modified: String, + pub read_only: bool, + /// A pinned page. Always false for the rest. + pub pinned: bool, + /// Hidden from the tree until "Show archived". + pub archived: bool, + /// An encrypted section, or a page in one. The smart features index and cache nothing of it. + pub encrypted: bool, + /// A protected section, or a page of one: "locked" or "unlocked" (ADR 0042). Left out for the rest. + #[serde(skip_serializing_if = "Option::is_none")] + pub protected: Option<&'static str>, +} + +/// The contract's `LibraryInfo`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LibraryOut { + pub folder: String, + pub read_only: bool, +} + +/// The contract's `InitialTree`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct InitialTree { + pub library: LibraryOut, + pub notebooks: Vec, + pub children: BTreeMap>, + pub resolved_path: Vec, + pub page: Option, +} + +/// Where nodes go: before `before_id` among the children of `parent_id`, or at the end. +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Placement { + #[serde(default)] + pub parent_id: Option, + #[serde(default)] + pub before_id: Option, +} + +/// The contract's `CreateInput`. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateInput { + pub kind: String, + pub placement: Placement, + #[serde(default)] + pub title: Option, + #[serde(default)] + pub color: Option, + #[serde(default)] + pub page_level: Option, +} + +/// The contract's `TrashReceipt`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ReceiptOut { + pub id: String, + pub node_ids: Vec, +} + +/// The contract's `TrashedItem`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TrashedItem { + pub receipt_id: String, + pub node: NodeSummary, + pub trashed_at: String, + pub original_parent_id: Option, + pub original_parent_title: String, + pub page_count: u32, +} + +/// The save status and whether anything waits to be saved. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StatusOut { + pub status: SaveStatus, + pub unsaved: bool, +} + +/// What a trash call left for its undo: the receipt its items share, and where each root was. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Pending { + pub receipt: String, + pub root: String, + pub parent: Option, + /// The sibling after the root when it went to Trash, which the root goes back before. + pub before: Option, + /// The root's place in the trash call's display order. + pub order: usize, +} + +/// What the notes commands keep between calls. +#[derive(Default)] +pub struct NotesState { + /// The notes folder in use, where new notebooks go. + pub(crate) folder: Option, + /// Whether this start has opened the library's notebooks and looked for a beta 1 profile. + pub(crate) started: bool, + /// The trees the interface last heard about, for the notes events. + pub(crate) seen: Option, + /// Trash items of this session's trash calls, by item key. + pub(crate) pending: HashMap, +} + +pub(crate) fn error(code: &str, message: impl Into) -> IpcError { + IpcError::new(code, message) +} + +pub(crate) fn not_found(what: &str) -> IpcError { + error("not-found", format!("{what} isn't there.")) +} + +pub(crate) fn invalid_move(why: &str) -> IpcError { + error("invalid-move", why) +} + +pub(crate) fn invalid_name(reason: &str) -> IpcError { + IpcError { + field: Some(reason.to_owned()), + ..error("invalid-name", format!("The name is {reason}.")) + } +} + +/// A core error with the contract's code. +pub(crate) fn from_core(error: CoreError) -> IpcError { + let code = error_code(&error); + if code == "invalid-name" { + return invalid_name(invalid_name_reason(&error).unwrap_or("empty")); + } + IpcError::new(code, error.to_string()) +} + +/// Whether two paths name the same folder, ignoring case on Windows. +pub(crate) fn same_path(a: &Path, b: &Path) -> bool { + opennote_core::session::library::same_path(a, b) +} + +impl Bridge { + /// Uses `folder` as the notes folder. The first call of a start migrates a beta 1 profile and opens the + /// library's notebooks; a new folder is searched for notebooks, so a notes folder copied from another computer + /// shows its notebooks. + pub(crate) fn use_folder(&mut self, folder: Option) { + if !self.notes.started { + self.notes.started = true; + migrate::run(self, folder.as_deref()); + self.open_library(); + } + let Some(folder) = folder else { + return; + }; + if self + .notes + .folder + .as_deref() + .is_some_and(|known| same_path(known, &folder)) + { + return; + } + self.notes.folder = Some(folder.clone()); + if let Err(error) = fs::create_dir_all(&folder) { + ::log::warn!("Couldn't create the notes folder: {error}"); + } + if let Err(error) = self.core.set_library_folder(&folder) { + ::log::warn!("Couldn't keep the notes folder in the library: {error}"); + } + self.open_folder_notebooks(&folder); + migrate::seed_sample(self, &folder); + } + + /// Opens every notebook of the library whose folder is there. + fn open_library(&mut self) { + for notebook in self.core.library().notebooks { + if notebook.open || !notebook.available { + continue; + } + if let Err(error) = self.core.open_notebook(¬ebook.path) { + ::log::warn!("Couldn't open a notebook of the library: {error}"); + } + } + if let Err(error) = self.core.recover_pending(None) { + ::log::warn!("Couldn't recover the waiting journals: {error}"); + } + } + + /// Opens the notebooks in `folder` that the library doesn't list yet, such as notebooks another computer made, + /// in name order. Notebooks this device moved to Trash stay there. + fn open_folder_notebooks(&mut self, folder: &Path) { + let Ok(entries) = fs::read_dir(folder) else { + return; + }; + let library = self.core.library(); + let mut found: Vec = entries + .filter_map(Result::ok) + .filter(|entry| !entry.file_name().to_string_lossy().starts_with('.')) + .map(|entry| entry.path()) + .filter(|path| path.join(NOTEBOOK_FILE).is_file()) + .filter(|path| !library.notebooks.iter().any(|known| same_path(&known.path, path))) + .filter(|path| !library.removed.iter().any(|gone| same_path(&gone.entry.path, path))) + .collect(); + found.sort(); + for path in found { + if let Err(error) = self.core.open_notebook(&path) { + ::log::warn!("Couldn't open the notebook at {}: {error}", path.display()); + } + } + } + + /// The save status, and whether any page has changes that aren't saved yet. + pub(crate) fn notes_status(&self) -> StatusOut { + StatusOut { + status: self.core.save_status(), + unsaved: self.core.has_unsaved(), + } + } + + /// Saves every open page and the journals. + pub(crate) fn notes_flush(&self) -> IpcResult<()> { + self.core + .flush_all(std::time::Duration::from_secs(10)) + .map(|_| ()) + .map_err(from_core) + } + + /// One notes command by name, with its arguments as the interface sends them, for the notes harness that runs + /// the contract suite against this bridge. + pub fn dispatch(&mut self, command: &str, args: &Value) -> IpcResult { + fn arg(args: &Value, name: &str) -> IpcResult { + serde_json::from_value(args.get(name).cloned().unwrap_or(Value::Null)) + .map_err(|error| IpcError::invalid(name, &error.to_string())) + } + fn out(value: T) -> IpcResult { + serde_json::to_value(value).map_err(|error| IpcError::new("io", error.to_string())) + } + match command { + "notes_load_initial" => out(self.load_initial(&arg::>(args, "path")?)?), + "notes_list_notebooks" => out(self.list_notebooks()), + "notes_list_children" => out(self.list_children(&arg::(args, "parentId")?)?), + "notes_get" => out(self.get_node(&arg::(args, "id")?)), + "notes_create" => out(self.create(arg(args, "input")?)?), + "notes_rename" => out(self.rename(&arg::(args, "id")?, &arg::(args, "title")?)?), + "notes_set_color" => out(self.set_color(&arg::(args, "id")?, arg(args, "color")?)?), + "notes_move" => out(self.move_nodes(&arg::>(args, "ids")?, &arg(args, "placement")?)?), + "notes_set_page_level" => out(self.set_page_level(&arg::>(args, "ids")?, arg(args, "level")?)?), + "notes_trash" => out(self.trash(&arg::>(args, "ids")?)?), + "notes_restore" => out(self.restore(&arg::(args, "receiptId")?)?), + "notes_list_trash" => out(self.list_trash()?), + "notes_restore_from_trash" => out(self.restore_from_trash(&arg::>(args, "ids")?)?), + "notes_purge" => out(self.purge(&arg::>(args, "ids")?).map(|_| ())?), + "notes_status" => out(self.notes_status()), + "notes_flush" => out(self.notes_flush()?), + _ => Err(IpcError::invalid("command", "isn't a notes command")), + } + } +} + +/// The library, the notebooks, the children along `path`, and the page at its end, in one call. +#[tauri::command] +pub async fn notes_load_initial( + app: AppHandle, + bridge: State<'_, CoreBridge>, + path: Vec, +) -> IpcResult { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.load_initial(&path))) + .await +} + +#[tauri::command] +pub async fn notes_list_notebooks(app: AppHandle, bridge: State<'_, CoreBridge>) -> IpcResult> { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| Ok(bridge.list_notebooks()))) + .await +} + +#[tauri::command] +pub async fn notes_list_children( + app: AppHandle, + bridge: State<'_, CoreBridge>, + parent_id: String, +) -> IpcResult> { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.list_children(&parent_id))) + .await +} + +#[tauri::command] +pub async fn notes_get(app: AppHandle, bridge: State<'_, CoreBridge>, id: String) -> IpcResult> { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| Ok(bridge.get_node(&id)))) + .await +} + +#[tauri::command] +pub async fn notes_create(app: AppHandle, bridge: State<'_, CoreBridge>, input: CreateInput) -> IpcResult { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.create(input))) + .await +} + +/// Renames a node. A page's title is the one in its `page.json`, which its heading shows too. +#[tauri::command] +pub async fn notes_rename( + app: AppHandle, + bridge: State<'_, CoreBridge>, + id: String, + title: String, + rename_folder: Option, +) -> IpcResult { + bridge + .run(move |bridge| { + let summary = run_notes(&app, bridge, |bridge| bridge.rename(&id, &title))?; + // A4-4: the notebook's folder follows its title when the interface's flag and the person's choice allow + // it. A folder that can't be renamed, as when another app holds a file in it, keeps its name; the title + // changed. + if summary.kind == "notebook" + && rename_folder == Some(true) + && folders::wanted(&crate::shellqol::prefs::read(&app)) + { + match run_notes(&app, bridge, |bridge| bridge.rename_notebook_folder(&id)) { + Ok(pages) if !pages.is_empty() => { + let pages: Vec = pages.iter().map(ToString::to_string).collect(); + crate::shellqol::emit(&app, "reopenPages", serde_json::json!({ "pages": pages })); + } + Ok(_) => {} + Err(error) => { + ::log::warn!("The notebook folder kept its name: {}", error.code); + crate::shellqol::emit(&app, "folderRenameFailed", serde_json::json!({ "id": id })); + } + } + } + Ok(summary) + }) + .await +} + +#[tauri::command] +pub async fn notes_set_color( + app: AppHandle, + bridge: State<'_, CoreBridge>, + id: String, + color: Option, +) -> IpcResult { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.set_color(&id, color))) + .await +} + +#[tauri::command] +pub async fn notes_move( + app: AppHandle, + bridge: State<'_, CoreBridge>, + ids: Vec, + placement: Placement, +) -> IpcResult<()> { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.move_nodes(&ids, &placement))) + .await +} + +#[tauri::command] +pub async fn notes_set_page_level( + app: AppHandle, + bridge: State<'_, CoreBridge>, + ids: Vec, + level: u8, +) -> IpcResult<()> { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.set_page_level(&ids, level))) + .await +} + +#[tauri::command] +pub async fn notes_trash(app: AppHandle, bridge: State<'_, CoreBridge>, ids: Vec) -> IpcResult { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.trash(&ids))) + .await +} + +#[tauri::command] +pub async fn notes_restore( + app: AppHandle, + bridge: State<'_, CoreBridge>, + receipt_id: String, +) -> IpcResult> { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.restore(&receipt_id))) + .await +} + +#[tauri::command] +pub async fn notes_list_trash(app: AppHandle, bridge: State<'_, CoreBridge>) -> IpcResult> { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.list_trash())) + .await +} + +#[tauri::command] +pub async fn notes_restore_from_trash( + app: AppHandle, + bridge: State<'_, CoreBridge>, + ids: Vec, +) -> IpcResult> { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.restore_from_trash(&ids))) + .await +} + +/// Deletes Trash items for good. A notebook in Trash keeps its folder, which File Explorer can delete. +#[tauri::command] +pub async fn notes_purge(app: AppHandle, bridge: State<'_, CoreBridge>, ids: Vec) -> IpcResult<()> { + bridge + .run(move |bridge| { + let pages = run_notes(&app, bridge, |bridge| bridge.purge(&ids))?; + // The pages' history went with their folders; their search entries and transcripts go now. + bridge.forget_pages(&pages); + Ok(()) + }) + .await +} + +#[tauri::command] +pub async fn notes_status(app: AppHandle, bridge: State<'_, CoreBridge>) -> IpcResult { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| Ok(bridge.notes_status()))) + .await +} + +#[tauri::command] +pub async fn notes_flush(app: AppHandle, bridge: State<'_, CoreBridge>) -> IpcResult<()> { + bridge + .run(move |bridge| run_notes(&app, bridge, |bridge| bridge.notes_flush())) + .await +} diff --git a/app/src-tauri/src/notes/purged.rs b/app/src-tauri/src/notes/purged.rs new file mode 100644 index 00000000..b0e8c60a --- /dev/null +++ b/app/src-tauri/src/notes/purged.rs @@ -0,0 +1,135 @@ +//! What deleting for good touches beyond the core's own folders (A2-4, A4-3): the pages inside a Trash item, so +//! their search entries and transcripts can be forgotten, and the Recycle Bin for a notebook's folder. OpenNote never +//! deletes a notebook folder outright. Delete folder moves it to the Recycle Bin, and Windows warns before a drive +//! without one would delete it for good. + +use std::path::{Path, PathBuf}; + +use opennote_core::PageId; + +use super::NOTEBOOK_FILE; +use crate::ipc::{IpcError, IpcResult}; + +/// How deep a page folder can sit inside a Trash item or a notebook: groups, a section, and nested pages. +const MAX_DEPTH: usize = 12; + +/// The pages under `dir`: each folder named by a page ID that holds a `page.json`. Links are not followed. +pub fn pages_under(dir: &Path) -> Vec { + let mut pages = Vec::new(); + let mut pending: Vec<(PathBuf, usize)> = vec![(dir.to_path_buf(), 0)]; + while let Some((folder, depth)) = pending.pop() { + let Ok(entries) = std::fs::read_dir(&folder) else { + continue; + }; + for entry in entries.flatten() { + let Ok(kind) = entry.file_type() else { + continue; + }; + if !kind.is_dir() || kind.is_symlink() { + continue; + } + let path = entry.path(); + let name = entry.file_name(); + if let Some(page) = name.to_str().and_then(|name| PageId::parse(name).ok()) { + if path.join("page.json").is_file() { + pages.push(page); + } + } + if depth < MAX_DEPTH { + pending.push((path, depth + 1)); + } + } + } + pages.sort_unstable_by_key(ToString::to_string); + pages.dedup(); + pages +} + +/// Checks that `path` is a notebook folder that may go to the Recycle Bin: absolute, local, with its +/// `notebook.json`. A network folder has no Recycle Bin, so it is refused and File Explorer can delete it. +pub fn check_recyclable(path: &Path) -> IpcResult<()> { + let text = path.to_string_lossy(); + if !path.is_absolute() || text.starts_with(r"\\") || text.starts_with("//") { + return Err(IpcError::new( + "unavailable", + "Only a folder on this PC can go to the Recycle Bin.", + )); + } + if !path.join(NOTEBOOK_FILE).is_file() { + return Err(IpcError::new("not-found", "That folder isn't a notebook.")); + } + Ok(()) +} + +/// Moves a notebook's folder to the Recycle Bin. +pub fn recycle(path: &Path) -> IpcResult<()> { + check_recyclable(path)?; + recycle_folder(path) +} + +#[cfg(windows)] +fn recycle_folder(path: &Path) -> IpcResult<()> { + use std::os::windows::ffi::OsStrExt; + + use windows::{ + core::PCWSTR, + Win32::UI::Shell::{ + SHFileOperationW, FOF_ALLOWUNDO, FOF_NOCONFIRMATION, FOF_NOERRORUI, FOF_SILENT, FOF_WANTNUKEWARNING, + FO_DELETE, SHFILEOPSTRUCTW, + }, + }; + + // The list of paths ends with two zeros. + let mut from: Vec = path.as_os_str().encode_wide().collect(); + from.extend([0, 0]); + let flags = FOF_ALLOWUNDO.0 | FOF_NOCONFIRMATION.0 | FOF_SILENT.0 | FOF_NOERRORUI.0 | FOF_WANTNUKEWARNING.0; + let mut operation = SHFILEOPSTRUCTW { + wFunc: FO_DELETE, + pFrom: PCWSTR(from.as_ptr()), + fFlags: u16::try_from(flags).unwrap_or(u16::MAX), + ..Default::default() + }; + // SAFETY: `operation` points at `from`, a list of one path ending with two zeros, which outlives the call. + let result = unsafe { SHFileOperationW(&mut operation) }; + if result != 0 || operation.fAnyOperationsAborted.as_bool() || path.exists() { + ::log::warn!("The notebook folder didn't go to the Recycle Bin (code {result})"); + return Err(IpcError::new("io", "The folder couldn't go to the Recycle Bin.")); + } + Ok(()) +} + +#[cfg(not(windows))] +fn recycle_folder(_path: &Path) -> IpcResult<()> { + Err(IpcError::new("unavailable", "Delete the folder in your file manager.")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn finds_the_pages_inside_a_folder_at_any_depth() { + let dir = tempfile::tempdir().expect("a temp folder"); + let page = "01m3sa12426sg32pmtyffjaqcf"; + let nested = "01m3sa8yf8bryf28a7sjgb7mmc"; + let section = dir.path().join("01m3s9q9xbpmxwz4cz4ht6twg9"); + std::fs::create_dir_all(section.join(page).join(nested)).expect("folders"); + std::fs::write(section.join(page).join("page.json"), b"{}").expect("a page"); + std::fs::write(section.join(page).join(nested).join("page.json"), b"{}").expect("a page"); + // A folder named like a page without a page.json is not a page. + std::fs::create_dir_all(dir.path().join("01m3sa43z1tp9rdr5e8df2jbxy")).expect("a folder"); + let found: Vec = pages_under(dir.path()).iter().map(ToString::to_string).collect(); + assert_eq!(found, [page, nested]); + assert!(pages_under(&dir.path().join("missing")).is_empty()); + } + + #[test] + fn only_a_local_notebook_folder_can_go_to_the_recycle_bin() { + let dir = tempfile::tempdir().expect("a temp folder"); + assert!(check_recyclable(dir.path()).is_err(), "no notebook.json"); + std::fs::write(dir.path().join(NOTEBOOK_FILE), b"{}").expect("a notebook file"); + assert!(check_recyclable(dir.path()).is_ok()); + assert!(check_recyclable(Path::new(r"\\server\share\Notes")).is_err()); + assert!(check_recyclable(Path::new("Notes")).is_err()); + } +} diff --git a/app/src-tauri/src/notes/sample.json b/app/src-tauri/src/notes/sample.json new file mode 100644 index 00000000..c0c113e1 --- /dev/null +++ b/app/src-tauri/src/notes/sample.json @@ -0,0 +1,40 @@ +{ + "folder": "", + "notebooks": [ + { "id": "n-biology", "kind": "notebook", "title": "Biology 101", "color": "fern", "children": [ + { "id": "s-lectures", "kind": "section", "title": "Lectures", "color": "fern", "children": [ + { "id": "p-cell-structure", "kind": "page", "title": "Cell structure", "pageLevel": 0 }, + { "id": "p-membranes", "kind": "page", "title": "Membranes", "pageLevel": 1 }, + { "id": "p-mitosis", "kind": "page", "title": "Mitosis", "pageLevel": 0 }, + { "id": "p-meiosis", "kind": "page", "title": "Meiosis", "pageLevel": 0 }, + { "id": "p-photosynthesis", "kind": "page", "title": "Photosynthesis", "pageLevel": 0 } + ]}, + { "id": "s-labs", "kind": "section", "title": "Labs", "color": "amber", "children": [ + { "id": "p-photosynthesis-lab", "kind": "page", "title": "Photosynthesis lab", "pageLevel": 0 } + ]}, + { "id": "g-exam-prep", "kind": "sectionGroup", "title": "Exam prep", "children": [ + { "id": "s-midterm", "kind": "section", "title": "Midterm", "color": "plum", "children": [ + { "id": "p-midterm-topics", "kind": "page", "title": "Topics to review", "pageLevel": 0 } + ]}, + { "id": "s-final", "kind": "section", "title": "Final", "color": "indigo", "children": [] } + ]} + ]}, + { "id": "n-work", "kind": "notebook", "title": "Work", "color": "brick", "children": [ + { "id": "s-meetings", "kind": "section", "title": "Meetings", "color": "brick", "children": [ + { "id": "p-weekly-sync", "kind": "page", "title": "Weekly sync", "pageLevel": 0 } + ]}, + { "id": "s-projects", "kind": "section", "title": "Projects", "color": "walnut", "children": [ + { "id": "p-roadmap", "kind": "page", "title": "Roadmap", "pageLevel": 0 } + ]} + ]}, + { "id": "n-personal", "kind": "notebook", "title": "Personal", "color": "plum", "children": [ + { "id": "s-journal", "kind": "section", "title": "Journal", "color": "plum", "children": [] } + ]}, + { "id": "n-recipes", "kind": "notebook", "title": "Recipes", "color": "amber", "children": [ + { "id": "s-baking", "kind": "section", "title": "Baking", "color": "amber", "children": [] } + ]}, + { "id": "n-travel", "kind": "notebook", "title": "Travel", "color": "indigo", "children": [ + { "id": "s-trips", "kind": "section", "title": "Trips", "color": "indigo", "children": [] } + ]} + ] +} diff --git a/app/src-tauri/src/notes/tests.rs b/app/src-tauri/src/notes/tests.rs new file mode 100644 index 00000000..7161dfef --- /dev/null +++ b/app/src-tauri/src/notes/tests.rs @@ -0,0 +1,837 @@ +// checks-disable-file modifiability: tests that share one set of helpers; split them when they grow again + +//! Tests of the notes commands against a real core in a temporary folder. The contract suite also runs against +//! these commands through the notes harness (app/src/services/notes/core/contract.test.ts). + +use std::{ + fs, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; + +use opennote_core::{ + ops::resolve::{Edit, TxnRequest}, + session::page::PageHandle, +}; +use serde_json::{json, Value}; + +use crate::{core_bridge::CoreBridge, ipc::IpcResult}; + +/// A profile and a notes folder in a temporary folder, with the bridge over them. +struct Lib { + notes: PathBuf, + bridge: CoreBridge, + events: Arc>>, + _dir: Option, +} + +impl Lib { + fn new() -> Lib { + let dir = tempfile::tempdir().expect("a temp folder"); + let mut lib = Lib::at( + &dir.path().join("local"), + &dir.path().join("Documents").join("OpenNote"), + ); + lib._dir = Some(dir); + lib + } + + fn at(local: &Path, notes: &Path) -> Lib { + let bridge = CoreBridge::at(local.to_path_buf()); + let events = Arc::new(Mutex::new(Vec::new())); + let sink = events.clone(); + bridge.listen(Box::new(move |name, payload| { + if name == super::NOTES_EVENT { + sink.lock().expect("the events").push(payload); + } + })); + Lib { + notes: notes.to_path_buf(), + bridge, + events, + _dir: None, + } + } + + fn call(&self, command: &str, args: Value) -> IpcResult { + self.bridge + .notes(Some(self.notes.clone()), |bridge| bridge.dispatch(command, &args)) + } + + fn ok(&self, command: &str, args: Value) -> Value { + self.call(command, args) + .unwrap_or_else(|error| panic!("{command}: {error}")) + } + + fn add(&self, parent: Option<&str>, kind: &str, title: &str) -> String { + let input = json!({ "kind": kind, "title": title, "placement": { "parentId": parent, "beforeId": null } }); + let node = self.ok("notes_create", json!({ "input": input })); + node["id"].as_str().expect("an id").to_owned() + } + + fn add_page(&self, section: &str, title: &str, level: u8) -> String { + let input = json!({ + "kind": "page", "title": title, "pageLevel": level, + "placement": { "parentId": section, "beforeId": null }, + }); + self.ok("notes_create", json!({ "input": input }))["id"] + .as_str() + .expect("an id") + .to_owned() + } + + fn titles(&self, parent: Option<&str>) -> Vec { + let list = match parent { + None => self.ok("notes_list_notebooks", json!({})), + Some(id) => self.ok("notes_list_children", json!({ "parentId": id })), + }; + list.as_array() + .expect("a list") + .iter() + .map(|node| { + let level = node["pageLevel"].as_u64().unwrap_or(0); + let title = node["title"].as_str().unwrap_or_default(); + if node["kind"] == "page" { + format!("{title}:{level}") + } else { + title.to_owned() + } + }) + .collect() + } + + fn code(&self, command: &str, args: Value) -> (String, Option) { + let error = self.call(command, args).expect_err("the call fails"); + (error.code, error.field) + } + + fn page(&self, page: &str) -> PageHandle { + self.bridge + .notes(Some(self.notes.clone()), |bridge| bridge.handle(page, "main-1")) + .expect("the page opens") + } +} + +impl Drop for Lib { + fn drop(&mut self) { + self.bridge.shutdown(); + } +} + +fn type_text(handle: &PageHandle, seq: u64, markdown: &str) { + let block: opennote_core::ops::resolve::NewBlock = serde_json::from_value(json!({ + "id": "01k6f00000000000000000b001", "type": "text", "data": { "markdown": markdown } + })) + .expect("a new block"); + let request = TxnRequest { + page: handle.id(), + client: handle.client().clone(), + client_seq: seq, + coalesce: None, + ui: None, + edits: vec![Edit::InsertBlock { + block, + after: None, + before: None, + }], + }; + handle.apply(request).expect("applies"); +} + +fn page_json(handle: &PageHandle) -> Value { + let envelope = handle.envelope(None).expect("an envelope"); + let decoded = opennote_core::wire::envelope::decode(&envelope.bytes).expect("decodes"); + serde_json::from_slice(decoded.page_json).expect("page JSON") +} + +/// The folders directly inside `dir`, by name. +fn folders(dir: &Path) -> Vec { + let mut names: Vec = fs::read_dir(dir) + .map(|entries| { + entries + .filter_map(Result::ok) + .filter(|e| e.path().is_dir()) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default(); + names.sort(); + names +} + +#[test] +fn the_tree_lives_in_notebook_folders_in_the_notes_folder() { + let lib = Lib::new(); + let biology = lib.add(None, "notebook", "Biology"); + let exams = lib.add(Some(&biology), "sectionGroup", "Exam prep"); + let lectures = lib.add(Some(&biology), "section", "Lectures"); + let midterm = lib.add(Some(&exams), "section", "Midterm"); + let cell = lib.add_page(&lectures, "Cell", 0); + lib.add_page(&lectures, "Membranes", 1); + lib.add_page(&midterm, "Topics", 0); + assert_eq!(lib.titles(None), ["Biology"]); + assert_eq!(lib.titles(Some(&biology)), ["Exam prep", "Lectures"]); + assert_eq!(lib.titles(Some(&lectures)), ["Cell:0", "Membranes:1"]); + assert_eq!(lib.titles(Some(&exams)), ["Midterm"]); + let folder = lib.notes.join("Biology"); + assert!(folder.join("notebook.json").is_file()); + let page = folder.join(&lectures).join(&cell); + assert!( + page.join("page.json").is_file(), + "the page lives in its section in the notebook" + ); + assert!(!lib.notes.join("Pages").exists()); + let got = lib.ok("notes_get", json!({ "id": cell })); + assert_eq!(got["parentId"], json!(lectures)); + assert_eq!( + lib.ok("notes_get", json!({ "id": "01m3s9q9xbpmxwz4cz4ht6twg9" })), + Value::Null + ); + let tree = lib.ok("notes_load_initial", json!({ "path": [biology, lectures, cell] })); + assert_eq!(tree["resolvedPath"].as_array().map(Vec::len), Some(3)); + assert_eq!(tree["page"]["title"], "Cell"); + assert_eq!(tree["library"]["folder"], json!(lib.notes.display().to_string())); +} + +#[test] +fn hostile_notebook_names_make_safe_folders_and_keep_their_titles() { + let lib = Lib::new(); + let names = [ + "a/b\\c:d", + "CON", + "nul.txt", + "..", + " Spaced ", + "Biology", + "Biology", + "what?*<>|\"", + "trailing dots...", + ]; + for name in names { + lib.add(None, "notebook", name); + } + let long = "N".repeat(200); + lib.add(None, "notebook", &long); + let titles = lib.titles(None); + assert_eq!(titles.len(), names.len() + 1); + assert!(titles.contains(&"a/b\\c:d".to_owned())); + assert!(titles.contains(&"Spaced".to_owned())); + assert_eq!(titles.iter().filter(|t| *t == "Biology").count(), 2); + let made = folders(&lib.notes); + assert_eq!(made.len(), names.len() + 1, "one folder each: {made:?}"); + for folder in &made { + assert!( + !folder.contains(['/', '\\', ':', '?', '*', '<', '>', '|', '"']), + "{folder}" + ); + assert!(!folder.ends_with('.') && !folder.ends_with(' '), "{folder}"); + assert!(folder.encode_utf16().count() <= 64, "{folder}"); + assert!( + !folder.eq_ignore_ascii_case("con") && !folder.eq_ignore_ascii_case("nul.txt"), + "{folder}" + ); + assert!(lib.notes.join(folder).join("notebook.json").is_file()); + } + assert!(made.contains(&"Biology".to_owned()) && made.contains(&"Biology (2)".to_owned())); + // Titles past 200 characters, and empty ones, are refused with their reason. + let too_long = json!({ "input": { "kind": "notebook", "title": "N".repeat(201), "placement": {} } }); + assert_eq!( + lib.code("notes_create", too_long), + ("invalid-name".into(), Some("too-long".into())) + ); + let empty = json!({ "input": { "kind": "notebook", "title": " ", "placement": {} } }); + assert_eq!( + lib.code("notes_create", empty), + ("invalid-name".into(), Some("empty".into())) + ); +} + +#[test] +fn hostile_section_and_page_titles_stay_titles_and_folders_use_ids() { + let lib = Lib::new(); + let notebook = lib.add(None, "notebook", "Work"); + let section = lib.add(Some(¬ebook), "section", "..\\..\\escape"); + let page = lib.add_page(§ion, "CON", 0); + lib.add_page(§ion, "a/b:c*?", 0); + lib.add_page(§ion, "a/b:c*?", 0); + assert_eq!(lib.titles(Some(§ion)), ["CON:0", "a/b:c*?:0", "a/b:c*?:0"]); + let folder = lib.notes.join("Work"); + assert_eq!(folders(&folder).iter().filter(|f| **f == section).count(), 1); + assert!(folder.join(§ion).join(&page).join("page.json").is_file()); + assert!(!lib.notes.join("escape").exists() && !folder.join("CON").exists()); +} + +#[test] +fn renames_colors_and_the_page_title_is_one_field() { + let lib = Lib::new(); + let notebook = lib.add(None, "notebook", "Biology"); + let section = lib.add(Some(¬ebook), "section", "Lectures"); + let page = lib.add_page(§ion, "Cell", 0); + lib.ok("notes_rename", json!({ "id": notebook, "title": " Biology 101 " })); + assert_eq!(lib.titles(None), ["Biology 101"]); + // The folder keeps its name. + assert!(lib.notes.join("Biology").join("notebook.json").is_file()); + // Renaming the page in the list changes the page's own title, which its heading shows. + lib.ok("notes_rename", json!({ "id": page, "title": "Cell structure" })); + let handle = lib.page(&page); + assert_eq!(page_json(&handle)["title"], "Cell structure"); + // Typing in the heading renames the page in the list once the page saves. + let edit: Edit = serde_json::from_value(json!({ "edit": "setPage", "title": "Cells" })).expect("an edit"); + let request = TxnRequest { + page: handle.id(), + client: handle.client().clone(), + client_seq: 1, + coalesce: None, + ui: None, + edits: vec![edit], + }; + handle.apply(request).expect("applies"); + handle.save_now().expect("saves"); + // The core refreshes the list's copy of the title right after the save. + let mut titles = lib.titles(Some(§ion)); + for _ in 0..100 { + if titles == ["Cells:0"] { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + titles = lib.titles(Some(§ion)); + } + assert_eq!(titles, ["Cells:0"]); + let renamed_empty = lib.code("notes_rename", json!({ "id": page, "title": "" })); + assert_eq!(renamed_empty, ("invalid-name".into(), Some("empty".into()))); + assert_eq!( + lib.code("notes_rename", json!({ "id": "nope", "title": "x" })).0, + "not-found" + ); + // Colors: pen names only, and never on pages. + let colored = lib.ok("notes_set_color", json!({ "id": section, "color": "fern" })); + assert_eq!(colored["color"], "fern"); + let odd = lib.ok("notes_set_color", json!({ "id": section, "color": "#ff0000" })); + assert_eq!(odd["color"], Value::Null); + let on_page = lib.ok("notes_set_color", json!({ "id": page, "color": "fern" })); + assert_eq!(on_page["color"], Value::Null); + let notebook_color = lib.ok("notes_set_color", json!({ "id": notebook, "color": "plum" })); + assert_eq!(notebook_color["color"], "plum"); +} + +#[test] +fn moves_reorder_and_cross_notebooks_with_the_rules_checked() { + let lib = Lib::new(); + let biology = lib.add(None, "notebook", "Biology"); + let work = lib.add(None, "notebook", "Work"); + let lectures = lib.add(Some(&biology), "section", "Lectures"); + let labs = lib.add(Some(&biology), "section", "Labs"); + let meetings = lib.add(Some(&work), "section", "Meetings"); + let cell = lib.add_page(&lectures, "Cell", 0); + lib.add_page(&lectures, "Membranes", 1); + let mitosis = lib.add_page(&lectures, "Mitosis", 0); + let content = lib.page(&cell); + type_text(&content, 1, "Kept through moves"); + content.save_now().expect("saves"); + // Notebooks reorder. + lib.ok( + "notes_move", + json!({ "ids": [work], "placement": { "parentId": null, "beforeId": biology } }), + ); + assert_eq!(lib.titles(None), ["Work", "Biology"]); + // A section moves to another notebook, with its pages. + lib.ok( + "notes_move", + json!({ "ids": [lectures], "placement": { "parentId": work, "beforeId": meetings } }), + ); + assert_eq!(lib.titles(Some(&work)), ["Lectures", "Meetings"]); + assert_eq!(lib.titles(Some(&biology)), ["Labs"]); + assert_eq!(lib.titles(Some(&lectures)), ["Cell:0", "Membranes:1", "Mitosis:0"]); + // A page moves with its subpage to another notebook's section, and keeps its content. + lib.ok( + "notes_move", + json!({ "ids": [cell], "placement": { "parentId": labs, "beforeId": null } }), + ); + assert_eq!(lib.titles(Some(&labs)), ["Cell:0", "Membranes:1"]); + assert_eq!(lib.titles(Some(&lectures)), ["Mitosis:0"]); + let moved = lib.page(&cell); + assert_eq!(page_json(&moved)["blocks"][0]["data"]["markdown"], "Kept through moves"); + // The rules: kinds, cycles, unknown nodes, and siblings. + assert_eq!( + lib.code( + "notes_move", + json!({ "ids": [mitosis], "placement": { "parentId": biology } }) + ) + .0, + "invalid-move" + ); + let group = lib.add(Some(&biology), "sectionGroup", "Group"); + let inner = lib.add(Some(&group), "sectionGroup", "Inner"); + assert_eq!( + lib.code( + "notes_move", + json!({ "ids": [group], "placement": { "parentId": inner } }) + ) + .0, + "invalid-move" + ); + assert_eq!( + lib.code( + "notes_move", + json!({ "ids": ["nope"], "placement": { "parentId": biology } }) + ) + .0, + "not-found" + ); + let elsewhere = json!({ "ids": [labs], "placement": { "parentId": biology, "beforeId": meetings } }); + assert_eq!(lib.code("notes_move", elsewhere).0, "invalid-move"); + // Groups nest at most 4 deep. + let three = lib.add(Some(&inner), "sectionGroup", "Three"); + let four = lib.add(Some(&three), "sectionGroup", "Four"); + let five = json!({ "input": { "kind": "sectionGroup", "title": "Five", "placement": { "parentId": four } } }); + assert_eq!(lib.code("notes_create", five).0, "invalid-move"); + // Levels. + lib.ok("notes_set_page_level", json!({ "ids": [mitosis], "level": 0 })); + assert_eq!( + lib.code("notes_set_page_level", json!({ "ids": [mitosis], "level": 1 })) + .0, + "invalid-move" + ); +} + +#[test] +fn trash_restore_and_purge_keep_trash_in_the_notebook() { + let lib = Lib::new(); + let biology = lib.add(None, "notebook", "Biology"); + let lectures = lib.add(Some(&biology), "section", "Lectures"); + let cell = lib.add_page(&lectures, "Cell", 0); + lib.add_page(&lectures, "Membranes", 1); + let mitosis = lib.add_page(&lectures, "Mitosis", 0); + let receipt = lib.ok("notes_trash", json!({ "ids": [cell] })); + assert_eq!(receipt["nodeIds"], json!([cell])); + assert_eq!(lib.titles(Some(&lectures)), ["Mitosis:0"]); + let trash = lib.ok("notes_list_trash", json!({})); + assert_eq!(trash[0]["node"]["id"], json!(cell)); + assert_eq!(trash[0]["pageCount"], 2); + assert_eq!(trash[0]["originalParentTitle"], "Lectures"); + assert_eq!( + folders(&lib.notes.join("Biology").join(".opennote").join("trash")).len(), + 1 + ); + // Restore from the list puts it back where it was. + lib.ok("notes_restore_from_trash", json!({ "ids": [cell] })); + assert_eq!(lib.titles(Some(&lectures)), ["Cell:0", "Membranes:1", "Mitosis:0"]); + // Undo of a trash call. + let receipt = lib.ok("notes_trash", json!({ "ids": [mitosis] })); + lib.ok("notes_restore", json!({ "receiptId": receipt["id"] })); + assert_eq!(lib.titles(Some(&lectures)), ["Cell:0", "Membranes:1", "Mitosis:0"]); + assert_eq!( + lib.code("notes_restore", json!({ "receiptId": receipt["id"] })).0, + "not-found" + ); + assert_eq!(lib.code("notes_restore", json!({ "receiptId": "nope" })).0, "not-found"); + // Purge deletes the item for good. + lib.ok("notes_trash", json!({ "ids": [mitosis] })); + lib.ok("notes_purge", json!({ "ids": [mitosis] })); + assert_eq!(lib.ok("notes_list_trash", json!({})), json!([])); + assert_eq!( + lib.code("notes_restore_from_trash", json!({ "ids": [mitosis] })).0, + "not-found" + ); + // A notebook in Trash leaves the list and keeps its folder, and comes back. + lib.ok("notes_trash", json!({ "ids": [biology] })); + assert!(lib.titles(None).is_empty()); + assert!(lib.notes.join("Biology").join("notebook.json").is_file()); + assert_eq!(lib.ok("notes_list_trash", json!({}))[0]["node"]["kind"], "notebook"); + lib.ok("notes_restore_from_trash", json!({ "ids": [biology] })); + assert_eq!(lib.titles(None), ["Biology"]); +} + +/// Marks every `section.json` of the given sections under `dir` encrypted, as the section-protect feature leaves it. +fn mark_encrypted(dir: &Path, sections: &[&str]) -> usize { + let mut marked = 0; + for entry in fs::read_dir(dir).expect("a folder").filter_map(Result::ok) { + let path = entry.path(); + if !path.is_dir() { + continue; + } + let file = path.join("section.json"); + let name = entry.file_name().to_string_lossy().into_owned(); + if sections.contains(&name.as_str()) && file.is_file() { + let mut json: Value = serde_json::from_str(&fs::read_to_string(&file).expect("reads")).expect("JSON"); + json["encryption"] = json!({ "scheme": "test" }); + fs::write(&file, serde_json::to_string(&json).expect("JSON")).expect("writes"); + marked += 1; + } + marked += mark_encrypted(&path, sections); + } + marked +} + +#[test] +fn trash_keeps_the_encrypted_flag_of_the_section_its_pages_came_from() { + // Trash listed the pages and sections of an encrypted section as unencrypted, so the smart features would have + // taken them for unprotected pages and kept their text. + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let (lectures, labs) = { + let lib = Lib::at(&dir.path().join("first"), ¬es); + let biology = lib.add(None, "notebook", "Biology"); + let lectures = lib.add(Some(&biology), "section", "Lectures"); + let cell = lib.add_page(&lectures, "Cell", 0); + lib.add_page(&lectures, "Mitosis", 0); + let labs = lib.add(Some(&biology), "section", "Labs"); + lib.add_page(&labs, "Titration", 0); + let plain = lib.add(Some(&biology), "section", "Plain"); + let open = lib.add_page(&plain, "Open", 0); + lib.ok("notes_trash", json!({ "ids": [cell] })); + lib.ok("notes_trash", json!({ "ids": [labs] })); + lib.ok("notes_trash", json!({ "ids": [open] })); + (lectures, labs) + }; + assert_eq!(mark_encrypted(¬es, &[&lectures, &labs]), 2); + let lib = Lib::at(&dir.path().join("second"), ¬es); + let trash = lib.ok("notes_list_trash", json!({})); + let flags: Vec<(String, bool)> = trash + .as_array() + .expect("a list") + .iter() + .map(|item| { + let node = &item["node"]; + ( + node["title"].as_str().unwrap_or_default().to_owned(), + node["encrypted"] == json!(true), + ) + }) + .collect(); + assert_eq!(flags.len(), 3); + assert!(flags.contains(&("Cell".to_owned(), true)), "{flags:?}"); + assert!(flags.contains(&("Labs".to_owned(), true)), "{flags:?}"); + assert!(flags.contains(&("Open".to_owned(), false)), "{flags:?}"); +} + +#[test] +fn a_copied_notes_folder_opens_with_its_notebooks_on_another_profile() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + let (biology, page) = { + let lib = Lib::at(&dir.path().join("first"), ¬es); + let biology = lib.add(None, "notebook", "Biology"); + let section = lib.add(Some(&biology), "section", "Lectures"); + let page = lib.add_page(§ion, "Cell", 0); + lib.add(None, "notebook", "Work"); + type_text(&lib.page(&page), 1, "Copied with the folder"); + (biology, page) + }; + let copy = dir.path().join("Elsewhere").join("Notes"); + crate::notes::migrate::copy_dir(¬es, ©).expect("a copy"); + let other = Lib::at(&dir.path().join("second"), ©); + let mut found = other.titles(None); + found.sort(); + assert_eq!(found, ["Biology", "Work"]); + assert_eq!(other.titles(Some(&biology)), ["Lectures"]); + assert_eq!( + page_json(&other.page(&page))["blocks"][0]["data"]["markdown"], + "Copied with the folder" + ); +} + +#[test] +fn changes_send_notes_events() { + let lib = Lib::new(); + let notebook = lib.add(None, "notebook", "Biology"); + lib.add(Some(¬ebook), "section", "Lectures"); + let events = lib.events.lock().expect("the events").clone(); + assert!(events.iter().any(|e| e["type"] == "upserted" + && e["nodes"] + .as_array() + .is_some_and(|n| n.iter().any(|n| n["title"] == "Lectures")))); + assert!(events + .iter() + .any(|e| e["type"] == "childrenChanged" && e["parentId"] == json!(notebook))); + let (code, _) = lib.code("notes_nothing", json!({})); + assert_eq!(code, "invalid"); +} + +#[test] +fn a_notes_folder_used_before_with_unknown_files_still_opens() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Notes"); + fs::create_dir_all(notes.join("Holiday photos")).expect("a folder"); + fs::create_dir_all(notes.join(".hidden")).expect("a folder"); + fs::write(notes.join("readme.txt"), "not a notebook").expect("a file"); + let broken = notes.join("Broken"); + fs::create_dir_all(&broken).expect("a folder"); + fs::write(broken.join("notebook.json"), "{ not json").expect("a damaged notebook"); + let lib = Lib::at(&dir.path().join("local"), ¬es); + assert!(lib.titles(None).is_empty()); + lib.add(None, "notebook", "Fresh"); + assert_eq!(lib.titles(None), ["Fresh"]); +} + +/// A beta 1 profile: the Phase 2 snapshot, the page map, and the `Pages` notebook that held the page content. +fn beta_one_profile(local: &Path, notes: &Path) -> (String, String) { + use opennote_core::session::{ + core::{Core, CoreConfig}, + events::{CoreEvent, EventSink}, + notebook::{NodePlacement, ParentRef}, + }; + struct Quiet; + impl EventSink for Quiet { + fn emit(&self, _event: CoreEvent) {} + } + let data = local.join("phase4").join("core"); + let config = CoreConfig::production(data, "0.1.0-beta.1".into()).expect("a config"); + let core = Core::start(config, Arc::new(Quiet), None).expect("a core"); + let pages = core.create_notebook(notes, "Pages").expect("the Pages notebook"); + let top = NodePlacement { + parent: ParentRef::Notebook, + before: None, + }; + let section = pages.create_section("Pages", top).expect("a section"); + let at = NodePlacement { + parent: ParentRef::Section(section), + before: None, + }; + let cell = pages.create_page(section, at).expect("a page"); + let typed = pages.create_page(section, at).expect("a page"); + let client = opennote_core::ClientId::parse("main-1").expect("a client"); + let handle = pages.open_page(cell, client.clone()).expect("opens"); + type_text(&handle, 1, "Cells divide"); + handle.close(&client).expect("closes"); + let handle = pages.open_page(typed, client.clone()).expect("opens"); + let edit: Edit = serde_json::from_value(json!({ "edit": "setPage", "title": "Heading typed" })).expect("an edit"); + let request = TxnRequest { + page: handle.id(), + client: client.clone(), + client_seq: 1, + coalesce: None, + ui: None, + edits: vec![edit], + }; + handle.apply(request).expect("applies"); + handle.close(&client).expect("closes"); + pages.close().expect("the notebook closes"); + core.flush_all(std::time::Duration::from_secs(5)).expect("flushed"); + core.shutdown(std::time::Duration::from_secs(5)); + let map = json!({ "p-cell": cell.to_string(), "p-typed": typed.to_string() }); + fs::write(local.join("phase4").join("pages.json"), map.to_string()).expect("the map"); + let snapshot = json!({ + "folder": notes.display().to_string(), + "notebooks": [{ + "id": "n-bio", "kind": "notebook", "title": "Biology", "color": "fern", + "children": [ + { "id": "g-exams", "kind": "sectionGroup", "title": "Exam prep", "children": [ + { "id": "s-mid", "kind": "section", "title": "Midterm", "children": [] } + ]}, + { "id": "s-lec", "kind": "section", "title": "Lectures", "color": "amber", "children": [ + { "id": "p-cell", "kind": "page", "title": "Cell", "pageLevel": 0 }, + { "id": "p-mem", "kind": "page", "title": "Membranes", "pageLevel": 1 }, + { "id": "p-typed", "kind": "page", "title": "Untitled page", "pageLevel": 0 } + ]} + ] + }, { "id": "n-work", "kind": "notebook", "title": "Work", "children": [] }], + "trash": [{ + "receiptId": "t-1", "trashedAt": "2026-10-01T10:00:00.000Z", "parentId": "s-lec", "beforeId": null, + "parentTitle": "Lectures", "notebookId": "n-bio", + "nodes": [{ "id": "p-old", "kind": "page", "title": "Old page", "pageLevel": 0 }] + }] + }); + fs::write(local.join("phase2-notes.json"), snapshot.to_string()).expect("the snapshot"); + (cell.to_string(), typed.to_string()) +} + +#[test] +fn a_beta_one_profile_moves_into_notebooks_once_with_a_backup() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (local, notes) = (dir.path().join("local"), dir.path().join("Documents").join("OpenNote")); + let (cell, typed) = beta_one_profile(&local, ¬es); + { + let lib = Lib::at(&local, ¬es); + assert_eq!(lib.titles(None), ["Biology", "Work"]); + let biology = lib.ok("notes_list_notebooks", json!({}))[0]["id"] + .as_str() + .expect("an id") + .to_owned(); + let children = lib.ok("notes_list_children", json!({ "parentId": biology })); + assert_eq!(lib.titles(Some(&biology)), ["Exam prep", "Lectures"]); + assert_eq!(children[1]["color"], "amber"); + let lectures = children[1]["id"].as_str().expect("an id").to_owned(); + // The heading typed on the page wins over the list's default name. + assert_eq!( + lib.titles(Some(&lectures)), + ["Cell:0", "Membranes:1", "Heading typed:0"] + ); + assert_eq!( + page_json(&lib.page(&cell))["blocks"][0]["data"]["markdown"], + "Cells divide" + ); + assert_eq!(page_json(&lib.page(&typed))["title"], "Heading typed"); + let page_dir = notes.join("Biology").join(&lectures).join(&cell); + assert!( + page_dir.join("page.json").is_file(), + "the content lives in its notebook" + ); + let trash = lib.ok("notes_list_trash", json!({})); + assert_eq!(trash[0]["node"]["title"], "Old page"); + } + assert!(!notes.join("Pages").exists()); + let backup = local.join(crate::notes::migrate::BACKUP_DIR); + assert!(backup.join("Pages").join("notebook.json").is_file()); + assert!(backup.join("phase2-notes.json").is_file() && backup.join("pages.json").is_file()); + assert!(backup.join("migration.json").is_file()); + assert!(!local.join("phase2-notes.json").exists() && !local.join("phase4").join("pages.json").exists()); + assert!(local.join("core").is_dir()); + // A second start finds nothing to migrate. + let again = Lib::at(&local, ¬es); + assert_eq!(again.titles(None), ["Biology", "Work"]); + assert!(!local + .join(format!("{} (2)", crate::notes::migrate::BACKUP_DIR)) + .exists()); +} + +#[test] +fn the_sample_library_of_test_builds_matches_the_fixture() { + let lib = Lib::new(); + let folder = lib.notes.clone(); + lib.bridge + .notes(Some(folder.clone()), |bridge| { + crate::notes::migrate::make_library(bridge, &folder, crate::notes::migrate::SAMPLE) + .map_err(|error| crate::ipc::IpcError::new("io", error)) + }) + .expect("the samples"); + assert_eq!( + lib.titles(None), + ["Biology 101", "Work", "Personal", "Recipes", "Travel"] + ); + let biology = lib.ok("notes_list_notebooks", json!({}))[0]["id"] + .as_str() + .expect("an id") + .to_owned(); + assert_eq!(lib.titles(Some(&biology)), ["Lectures", "Labs", "Exam prep"]); + let lectures = lib.ok("notes_list_children", json!({ "parentId": biology }))[0]["id"] + .as_str() + .expect("an id") + .to_owned(); + assert_eq!( + lib.titles(Some(&lectures)), + [ + "Cell structure:0", + "Membranes:1", + "Mitosis:0", + "Meiosis:0", + "Photosynthesis:0" + ] + ); +} + +#[test] +fn a_notes_folder_chosen_in_setup_opens_its_notebooks_and_tells_the_tree() { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes = dir.path().join("Synced"); + { + let earlier = Lib::at(&dir.path().join("other-computer"), ¬es); + earlier.add(None, "notebook", "From the other computer"); + } + let lib = Lib::at(&dir.path().join("local"), ¬es); + // Before setup chooses the folder, the library is empty. + let empty = lib + .bridge + .notes(None, |bridge| { + bridge.dispatch("notes_load_initial", &json!({ "path": [] })) + }) + .expect("the first load"); + assert_eq!(empty["notebooks"], json!([])); + assert_eq!(lib.titles(None), ["From the other computer"]); + let events = lib.events.lock().expect("the events").clone(); + assert!(events + .iter() + .any(|e| e["type"] == "childrenChanged" && e["parentId"].is_null())); +} + +#[test] +fn a_restored_notebook_goes_back_before_its_old_neighbor_or_to_the_end() { + let lib = Lib::new(); + let alpha = lib.add(None, "notebook", "Alpha"); + let beta = lib.add(None, "notebook", "Beta"); + let receipt = lib.ok("notes_trash", json!({ "ids": [alpha] })); + lib.add(None, "notebook", "Gamma"); + lib.ok("notes_restore", json!({ "receiptId": receipt["id"] })); + assert_eq!(lib.titles(None), ["Alpha", "Beta", "Gamma"]); + // Without its old neighbor, it goes to the end, as the reference model has it. + let receipt = lib.ok("notes_trash", json!({ "ids": [alpha] })); + lib.ok("notes_trash", json!({ "ids": [beta] })); + lib.ok("notes_restore", json!({ "receiptId": receipt["id"] })); + assert_eq!(lib.titles(None), ["Gamma", "Alpha"]); +} + +#[test] +fn a_beta_one_migration_stopped_part_way_runs_again_from_the_backup() { + let dir = tempfile::tempdir().expect("a temp folder"); + let (local, notes) = (dir.path().join("local"), dir.path().join("Documents").join("OpenNote")); + let (cell, _) = beta_one_profile(&local, ¬es); + crate::notes::migrate::STOP_AFTER.set(Some(1)); + { + let stopped = Lib::at(&local, ¬es); + // Only the first notebook was made, and its pages left the Pages notebook. + assert!(stopped.titles(None).contains(&"Biology".to_owned())); + } + crate::notes::migrate::STOP_AFTER.set(None); + assert!(local.join(crate::notes::migrate::MARKER_FILE).is_file()); + let lib = Lib::at(&local, ¬es); + assert_eq!(lib.titles(None), ["Biology", "Work"]); + let biology = lib.ok("notes_list_notebooks", json!({}))[0]["id"] + .as_str() + .expect("an id") + .to_owned(); + let lectures = lib.ok("notes_list_children", json!({ "parentId": biology }))[1]["id"] + .as_str() + .expect("an id") + .to_owned(); + assert_eq!( + lib.titles(Some(&lectures)), + ["Cell:0", "Membranes:1", "Heading typed:0"] + ); + assert_eq!( + page_json(&lib.page(&cell))["blocks"][0]["data"]["markdown"], + "Cells divide" + ); + assert!(!local.join(crate::notes::migrate::MARKER_FILE).exists()); + assert!(!notes.join("Pages").exists()); + assert!(local + .join(crate::notes::migrate::BACKUP_DIR) + .join("migration.json") + .is_file()); +} + +#[test] +fn renaming_a_notebook_folder_follows_its_title_and_reopens_its_pages() { + let lib = Lib::new(); + let notebook = lib.add(None, "notebook", "Biology"); + let section = lib.add(Some(¬ebook), "section", "Lectures"); + let page = lib.add_page(§ion, "Cell", 0); + let _open = lib.page(&page); + lib.ok("notes_rename", json!({ "id": notebook, "title": "Biology 101" })); + let reopen = lib + .bridge + .notes(Some(lib.notes.clone()), |bridge| { + bridge.rename_notebook_folder(¬ebook) + }) + .expect("renames the folder"); + assert_eq!( + reopen.iter().map(ToString::to_string).collect::>(), + std::slice::from_ref(&page) + ); + assert!(lib.notes.join("Biology 101").join("notebook.json").is_file()); + assert!(!lib.notes.join("Biology").exists()); + // The page opens again from the new folder, and the tree still lists everything. + let handle = lib.page(&page); + assert_eq!(page_json(&handle)["title"], "Cell"); + assert_eq!(lib.titles(None), ["Biology 101"]); + // A section or page ID renames nothing. + let none = lib + .bridge + .notes(Some(lib.notes.clone()), |bridge| { + bridge.rename_notebook_folder(§ion) + }) + .expect("a section"); + assert!(none.is_empty()); + assert!(super::folders::wanted(&serde_json::Map::new())); + let off = serde_json::json!({ "renameNotebookFolders": false }); + assert!(!super::folders::wanted(off.as_object().expect("an object"))); +} diff --git a/app/src-tauri/src/notes/trash.rs b/app/src-tauri/src/notes/trash.rs new file mode 100644 index 00000000..0b905aa7 --- /dev/null +++ b/app/src-tauri/src/notes/trash.rs @@ -0,0 +1,652 @@ +//! Trash: moving nodes to Trash with one receipt per call, listing what is there, restoring, and purging. +//! +//! Each notebook keeps its own Trash in its folder (docs/format/README.md section 12), so Trash travels with the +//! notebook. A notebook moved to Trash leaves the library and keeps its folder. The receipt of a trash call and +//! the sibling each root went before are kept for this session only: after a restart, each item is its own +//! receipt and goes back to its place by its order key. + +use std::{collections::HashSet, fs, path::Path}; + +use opennote_core::{ + model::{ + trash::{TrashItemFile, TrashOrigin, TrashReason}, + Named, + }, + session::{ + library::RemovedEntry, + notebook::{NodePlacement, NodeRef, NotebookHandle, ParentRef}, + notes::{ReceiptPart, TrashReceipt, TrashTarget}, + }, + GroupId, Id, PageId, SectionId, TrashItemId, +}; + +use super::{ + change::{ancestors, blocks_of, fit_blocks}, + from_core, not_found, same_path, + tree::{container_children, depth_of, flat_pages, height_of, Found, GROUP_DEPTH}, + NodeSummary, Pending, ReceiptOut, TrashedItem, +}; +use crate::{core_bridge::Bridge, ipc::IpcResult}; + +fn item_key(notebook: &Path, item: TrashItemId) -> String { + format!("{}|{item}", notebook.display()) +} + +fn notebook_key(path: &Path) -> String { + format!("notebook|{}", path.display()) +} + +/// The folder of a Trash item (docs/format/README.md section 3.1). +fn item_dir(notebook: &Path, item: TrashItemId) -> std::path::PathBuf { + notebook.join(".opennote").join("trash").join(item.to_string()) +} + +/// The pages listed in a section folder's `section.json`. +fn pages_in(section_dir: &Path) -> u32 { + let Ok(text) = fs::read_to_string(section_dir.join("section.json")) else { + return 0; + }; + let json: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); + json.get("pages") + .and_then(serde_json::Value::as_array) + .map_or(0, |pages| u32::try_from(pages.len()).unwrap_or(u32::MAX)) +} + +/// The pages of a notebook folder that isn't open. +fn pages_in_notebook(dir: &Path) -> u32 { + let Ok(entries) = fs::read_dir(dir) else { + return 0; + }; + entries + .filter_map(Result::ok) + .map(|entry| pages_in(&entry.path())) + .sum() +} + +/// Whether a section folder's `section.json` marks it encrypted (spec 5.7), as the core reads it: the key is there. +/// A file that can't be read counts as encrypted, so nothing of its pages is kept by mistake. +fn section_encrypted(section_dir: &Path) -> bool { + let Ok(text) = fs::read_to_string(section_dir.join("section.json")) else { + return true; + }; + serde_json::from_str::(&text).map_or(true, |json| json.get("encryption").is_some()) +} + +/// A trashed root as a node summary. `encrypted` carries its section's flag: the smart features would otherwise +/// take the pages of an encrypted section listed in Trash for unprotected ones. +fn trashed_node( + id: String, + kind: &'static str, + parent: Option, + title: String, + at: &str, + encrypted: bool, +) -> NodeSummary { + NodeSummary { + id, + kind, + parent_id: parent, + title, + color: None, + page_level: 0, + child_count: 0, + created: at.to_owned(), + modified: at.to_owned(), + read_only: false, + pinned: false, + archived: false, + encrypted, + protected: None, + } +} + +/// What a Trash item of a notebook lists as: its root, its old parent and that parent's title, and its pages. +fn listed_item(notebook: &NotebookHandle, file: &TrashItemFile) -> Option<(NodeSummary, Option, String, u32)> { + let at = file.deleted_at.to_rfc3339(); + let notebook_id = notebook.id().to_string(); + let notebook_title = notebook.tree().title; + let dir = item_dir(notebook.path(), file.id); + match &file.origin { + TrashOrigin::Pages { + section, + section_title, + entries, + } => { + let ids: HashSet = entries.iter().map(|e| e.id).collect(); + let root = entries + .iter() + .find(|e| e.parent.is_none_or(|p| !ids.contains(&p))) + .or(entries.first())?; + let parent = Some(section.to_string()); + // Pages keep their section's flag. A section the tree no longer has counts as encrypted. + let encrypted = notebook.tree().section(*section).is_none_or(|s| s.encrypted); + let node = trashed_node( + root.id.to_string(), + "page", + parent.clone(), + root.title.clone(), + &at, + encrypted, + ); + let count = u32::try_from(entries.len()).unwrap_or(u32::MAX); + Some((node, parent, section_title.clone(), count)) + } + TrashOrigin::Section { + group, parent_title, .. + } => { + let id = file.contents.first()?; + let parent = Some(group.map_or_else(|| notebook_id.clone(), |g| g.to_string())); + let encrypted = section_encrypted(&dir.join(id.to_string())); + let node = trashed_node( + id.to_string(), + "section", + parent.clone(), + file.title.clone(), + &at, + encrypted, + ); + let title = parent_title.clone().unwrap_or(notebook_title); + Some((node, parent, title, pages_in(&dir.join(id.to_string())))) + } + TrashOrigin::Group { groups, parent_title } => { + let root = groups.first()?; + let parent = Some(root.parent.map_or_else(|| notebook_id.clone(), |g| g.to_string())); + let node = trashed_node( + root.id.to_string(), + "sectionGroup", + parent.clone(), + file.title.clone(), + &at, + false, + ); + let count = file.contents.iter().map(|s| pages_in(&dir.join(s.to_string()))).sum(); + Some((node, parent, parent_title.clone().unwrap_or(notebook_title), count)) + } + } +} + +/// A Trash item or a notebook in Trash, as `restoreFromTrash` finds it. +enum Entry { + Item(NotebookHandle, TrashItemId), + Notebook(RemovedEntry), +} + +impl Bridge { + /// The contract's trash: each root, with what is inside it, becomes a Trash item of its notebook, and a + /// notebook leaves the library. One receipt covers them all. + pub(crate) fn trash(&mut self, ids: &[String]) -> IpcResult { + let mut seen = HashSet::new(); + let mut found = Vec::new(); + for id in ids { + if seen.insert(id.clone()) { + found.push(self.find(id)?); + } + } + // Roots: nodes not inside another named node, nor in another named page's block of subpages. + let named: HashSet = found.iter().map(Found::id).collect(); + let mut roots: Vec = found + .into_iter() + .filter(|node| !ancestors(node).iter().any(|a| named.contains(a))) + .collect(); + let mut blocks: Vec<(String, Vec)> = Vec::new(); + for node in &roots { + if let Found::Node(notebook, NodeRef::Page(page)) = node { + let tree = notebook.tree(); + if let Some((section, _)) = tree.find_page(*page) { + let flat = flat_pages(&tree, section.id); + if let Some(block) = blocks_of(&flat, &[page.to_string()]).pop() { + blocks.push((page.to_string(), block.into_iter().map(|(id, _)| id).collect())); + } + } + } + } + let inside_block = |id: &str| { + blocks + .iter() + .any(|(root, block)| root != id && block.iter().any(|x| x == id)) + }; + roots.retain(|node| !inside_block(&node.id())); + let order = self.display_order(); + roots.sort_by_key(|node| order.get(&node.id()).copied().unwrap_or(usize::MAX)); + // Where each root was: the sibling after it that isn't going to Trash too. + let mut gone: HashSet = roots.iter().map(Found::id).collect(); + gone.extend(blocks.iter().flat_map(|(_, block)| block.iter().cloned())); + let mut places = Vec::with_capacity(roots.len()); + for node in &roots { + places.push(self.place_of(node, &gone)); + } + let targets: Vec = roots + .iter() + .map(|node| match node { + Found::Notebook(notebook) => TrashTarget::Notebook(notebook.path().to_path_buf()), + Found::Node(notebook, node) => TrashTarget::Node(notebook.clone(), *node), + }) + .collect(); + let receipt = self.core.trash_nodes(&targets).map_err(from_core)?; + let text = receipt.encode(); + for part in &receipt.parts { + match part { + ReceiptPart::Notebook { path } => { + if let Some(i) = roots + .iter() + .position(|n| matches!(n, Found::Notebook(nb) if same_path(nb.path(), path))) + { + self.remember(notebook_key(path), &text, &roots, &places, i); + } + } + ReceiptPart::Items { notebook, items } => { + let mine: Vec = roots + .iter() + .enumerate() + .filter(|(_, n)| matches!(n, Found::Node(nb, _) if same_path(nb.path(), notebook))) + .map(|(i, _)| i) + .collect(); + for (item, i) in items.iter().zip(mine) { + self.remember(item_key(notebook, *item), &text, &roots, &places, i); + } + } + } + } + Ok(ReceiptOut { + id: text, + node_ids: roots.iter().map(Found::id).collect(), + }) + } + + fn remember( + &mut self, + key: String, + receipt: &str, + roots: &[Found], + places: &[(Option, Option)], + i: usize, + ) { + let (Some(root), Some((parent, before))) = (roots.get(i), places.get(i)) else { + return; + }; + let pending = Pending { + receipt: receipt.to_owned(), + root: root.id(), + parent: parent.clone(), + before: before.clone(), + order: i, + }; + self.notes.pending.insert(key, pending); + } + + /// A root's parent, and the first sibling after it (after its subpages, for a page) that isn't in `gone`. + fn place_of(&self, node: &Found, gone: &HashSet) -> (Option, Option) { + let id = node.id(); + let tree = node.notebook().tree(); + let (parent, siblings): (Option, Vec) = match node { + Found::Notebook(_) => (None, self.notebooks().iter().map(|n| n.id().to_string()).collect()), + Found::Node(notebook, NodeRef::Page(page)) => match tree.find_page(*page) { + Some((section, _)) => { + let flat = flat_pages(&tree, section.id); + let _ = notebook; + ( + Some(section.id.to_string()), + flat.into_iter().map(|(id, _)| id).collect(), + ) + } + None => (None, Vec::new()), + }, + Found::Node(notebook, NodeRef::Group(g)) => { + let parent = tree.groups.iter().find(|x| x.id == *g).and_then(|x| x.parent); + let id = parent.map_or_else(|| notebook.id().to_string(), |p| p.to_string()); + (Some(id), container_children(&tree, parent)) + } + Found::Node(notebook, NodeRef::Section(s)) => { + let group = tree.section(*s).and_then(|x| x.group); + let id = group.map_or_else(|| notebook.id().to_string(), |p| p.to_string()); + (Some(id), container_children(&tree, group)) + } + }; + let at = siblings.iter().position(|x| *x == id).unwrap_or(siblings.len()); + let before = siblings.iter().skip(at + 1).find(|x| !gone.contains(*x)).cloned(); + (parent, before) + } + + /// The listed Trash items, newest first: the items of every notebook in the library, and the notebooks in + /// Trash. Items trashed inside a notebook that is itself in Trash stay with it, unlisted. Pages that moved to + /// another notebook leave their originals in Trash, which aren't listed either. + pub(crate) fn list_trash(&self) -> IpcResult> { + let mut out: Vec<(String, usize, TrashedItem)> = Vec::new(); + for notebook in self.notebooks() { + for file in notebook.trash().map_err(from_core)? { + if file.reason != Named::Known(TrashReason::Deleted) { + continue; + } + let Some((node, parent, parent_title, page_count)) = listed_item(¬ebook, &file) else { + continue; + }; + let key = item_key(notebook.path(), file.id); + let (receipt, order) = self.receipt_of(&key, || TrashReceipt { + parts: vec![ReceiptPart::Items { + notebook: notebook.path().to_path_buf(), + items: vec![file.id], + }], + }); + let at = file.deleted_at.to_rfc3339(); + out.push(( + at.clone(), + order, + TrashedItem { + receipt_id: receipt, + node, + trashed_at: at, + original_parent_id: parent, + original_parent_title: parent_title, + page_count, + }, + )); + } + } + for removed in self.core.library().removed { + let path = removed.entry.path.clone(); + let key = notebook_key(&path); + let (receipt, order) = self.receipt_of(&key, || TrashReceipt { + parts: vec![ReceiptPart::Notebook { path: path.clone() }], + }); + let at = removed.removed_at.to_rfc3339(); + let node = trashed_node( + removed.entry.notebook.to_string(), + "notebook", + None, + removed.entry.title.clone(), + &at, + false, + ); + out.push(( + at.clone(), + order, + TrashedItem { + receipt_id: receipt, + node, + trashed_at: at, + original_parent_id: None, + original_parent_title: String::new(), + page_count: pages_in_notebook(&path), + }, + )); + } + out.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + Ok(out.into_iter().map(|(_, _, item)| item).collect()) + } + + fn receipt_of(&self, key: &str, single: impl FnOnce() -> TrashReceipt) -> (String, usize) { + match self.notes.pending.get(key) { + Some(pending) => (pending.receipt.clone(), pending.order), + None => (single().encode(), 0), + } + } + + /// The contract's restore: what is left of one trash call goes back. + pub(crate) fn restore(&mut self, receipt: &str) -> IpcResult> { + let parsed = TrashReceipt::decode(receipt).map_err(|_| not_found("That receipt"))?; + let mut entries = Vec::new(); + for part in &parsed.parts { + match part { + ReceiptPart::Notebook { path } => { + let removed = self.core.library().removed; + if let Some(entry) = removed.into_iter().find(|r| same_path(&r.entry.path, path)) { + entries.push(Entry::Notebook(entry)); + } + } + ReceiptPart::Items { notebook, items } => { + let Some(handle) = self.notebooks().into_iter().find(|n| same_path(n.path(), notebook)) else { + continue; + }; + let listed: HashSet = + handle.trash().map_err(from_core)?.iter().map(|f| f.id).collect(); + for item in items.iter().filter(|item| listed.contains(item)) { + entries.push(Entry::Item(handle.clone(), *item)); + } + } + } + } + if entries.is_empty() { + return Err(not_found("Anything of that receipt in Trash")); + } + self.restore_entries(entries) + } + + /// Restores the listed items with these root IDs. + pub(crate) fn restore_from_trash(&mut self, ids: &[String]) -> IpcResult> { + let entries = self.entries_for(ids)?; + self.restore_entries(entries) + } + + /// The Trash entries whose roots have these IDs, or `not-found` when one isn't listed. + fn entries_for(&self, ids: &[String]) -> IpcResult> { + let mut entries = Vec::new(); + let mut seen = HashSet::new(); + for id in ids { + if !seen.insert(id.clone()) { + continue; + } + let removed = self.core.library().removed; + if let Some(entry) = removed.into_iter().find(|r| r.entry.notebook.to_string() == *id) { + entries.push(Entry::Notebook(entry)); + continue; + } + let mut hit = None; + for notebook in self.notebooks() { + for file in notebook.trash().map_err(from_core)? { + if file.reason != Named::Known(TrashReason::Deleted) { + continue; + } + if listed_item(¬ebook, &file).is_some_and(|(node, ..)| node.id == *id) { + hit = Some(Entry::Item(notebook.clone(), file.id)); + } + } + } + entries.push(hit.ok_or_else(|| not_found(id))?); + } + Ok(entries) + } + + fn restore_entries(&mut self, entries: Vec) -> IpcResult> { + let mut restored: Vec<(usize, NodeSummary)> = Vec::new(); + let mut fallback: Vec<(SectionId, SectionId)> = Vec::new(); + for entry in entries { + let (order, found) = match entry { + Entry::Notebook(removed) => { + let path = removed.entry.path; + let pending = self.notes.pending.remove(¬ebook_key(&path)); + let notebook = self.core.restore_notebook(&path).map_err(from_core)?; + if let Some(pending) = &pending { + // Before the notebook it was before, or at the end when that one is gone. + let before = pending.before.as_deref().and_then(|id| match self.find(id) { + Ok(Found::Notebook(next)) => Some(next.path().to_path_buf()), + _ => None, + }); + self.core.move_notebook(&path, before.as_deref()).map_err(from_core)?; + } + (pending.map_or(0, |p| p.order), Found::Notebook(notebook)) + } + Entry::Item(notebook, item) => self.restore_item(¬ebook, item, &mut fallback)?, + }; + restored.push((order, self.summary_of(&found)?)); + } + restored.sort_by_key(|(order, _)| *order); + Ok(restored.into_iter().map(|(_, node)| node).collect()) + } + + /// Restores one Trash item: before the sibling it went before when that sibling is still there, else where its + /// order key puts it. A page whose section is gone goes into a new section named after the old one, and pages + /// of one call whose section is gone share that new section. + fn restore_item( + &mut self, + notebook: &NotebookHandle, + item: TrashItemId, + fallback: &mut Vec<(SectionId, SectionId)>, + ) -> IpcResult<(usize, Found)> { + let file = notebook + .trash() + .map_err(from_core)? + .into_iter() + .find(|f| f.id == item) + .ok_or_else(|| not_found("That Trash item"))?; + let pending = self.notes.pending.remove(&item_key(notebook.path(), item)); + let order = pending.as_ref().map_or(0, |p| p.order); + let tree = notebook.tree(); + let to = match &file.origin { + TrashOrigin::Pages { section, .. } if tree.section(*section).is_none() => fallback + .iter() + .find(|(old, _)| old == section) + .map(|(_, new)| NodePlacement { + parent: ParentRef::Section(*new), + before: None, + }), + _ => None, + }; + let nodes = notebook.restore(item, to).map_err(from_core)?; + let root = match &file.origin { + TrashOrigin::Pages { entries, .. } => { + let ids: HashSet = entries.iter().map(|e| e.id).collect(); + let root = entries + .iter() + .find(|e| e.parent.is_none_or(|p| !ids.contains(&p))) + .map(|e| NodeRef::Page(e.id)); + root.or_else(|| nodes.first().copied()) + } + TrashOrigin::Section { .. } => file.contents.first().map(|id| NodeRef::Section(SectionId(*id))), + TrashOrigin::Group { groups, .. } => groups.first().map(|g| NodeRef::Group(g.id)), + } + .ok_or_else(|| not_found("The restored node"))?; + if let (TrashOrigin::Pages { section, .. }, NodeRef::Page(page)) = (&file.origin, root) { + if let Some((home, _)) = notebook.tree().find_page(page) { + if home.id != *section && !fallback.iter().any(|(old, _)| old == section) { + fallback.push((*section, home.id)); + } + } + } + self.put_back(notebook, root, pending.as_ref()); + Ok((order, Found::Node(notebook.clone(), root))) + } + + /// Puts a restored root where the contract wants it. A group that would now nest too deep, and a group or + /// section whose old parent is gone, go to the end of the notebook. Otherwise the root goes before the sibling it + /// went to Trash before, or to the end of its old parent when that sibling is gone. Without what this session + /// remembers of the trash call, the core's order key places it. A move that doesn't fit leaves the root where + /// the core restored it. + fn put_back(&self, notebook: &NotebookHandle, root: NodeRef, pending: Option<&Pending>) { + let tree = notebook.tree(); + let result = match root { + NodeRef::Page(page) => { + let Some(pending) = pending else { + return; + }; + let Some((section, _)) = tree.find_page(page) else { + return; + }; + if pending.parent.as_deref() != Some(section.id.to_string().as_str()) { + return; + } + let flat = flat_pages(&tree, section.id); + let Some(block) = blocks_of(&flat, &[page.to_string()]).pop() else { + return; + }; + let moved: HashSet<&String> = block.iter().map(|(id, _)| id).collect(); + let reduced: Vec<(String, u8)> = flat.iter().filter(|(id, _)| !moved.contains(id)).cloned().collect(); + let before = pending.before.as_deref(); + let index = before + .and_then(|b| reduced.iter().position(|(id, _)| id == b)) + .unwrap_or(reduced.len()); + let Some(level) = fit_blocks(&reduced, index, &[block]).and_then(|l| l.first().copied()) else { + return; + }; + notebook.move_page_blocks(&[(page, level)], section.id, index) + } + NodeRef::Section(_) | NodeRef::Group(_) => { + let current = match root { + NodeRef::Section(s) => tree.section(s).and_then(|x| x.group), + NodeRef::Group(g) => tree.groups.iter().find(|x| x.id == g).and_then(|x| x.parent), + NodeRef::Page(_) => None, + }; + let to_end = |parent: Option| NodePlacement { + parent: parent.map_or(ParentRef::Notebook, ParentRef::Group), + before: None, + }; + let too_deep = depth_of(&tree, current).saturating_add(height_of(&tree, root)) > GROUP_DEPTH; + let placement = if too_deep { + to_end(None) + } else { + let Some(pending) = pending else { + return; + }; + let old_parent = pending.parent.as_deref().unwrap_or_default(); + let current_id = current.map_or_else(|| notebook.id().to_string(), |g| g.to_string()); + if current_id != old_parent { + to_end(None) + } else { + let siblings = container_children(&tree, current); + let before = pending + .before + .as_deref() + .filter(|b| siblings.iter().any(|x| x == b)) + .and_then(|b| Id::parse(b).ok()) + .map(|b| { + if tree.groups.iter().any(|g| g.id.0 == b) { + NodeRef::Group(GroupId(b)) + } else { + NodeRef::Section(SectionId(b)) + } + }); + NodePlacement { + parent: current.map_or(ParentRef::Notebook, ParentRef::Group), + before, + } + } + }; + notebook.move_node(root, placement) + } + }; + if let Err(error) = result { + ::log::debug!("A restored node stays where its order key put it: {error}"); + } + } + + /// Deletes Trash items for good, with their page history, which is in their folders. Returns the pages that + /// went, so the caller can forget their search entries and transcripts. A notebook in Trash keeps its folder: + /// OpenNote never deletes a notebook folder outright, so it stays in Trash until File Explorer deletes it, or + /// Delete folder moves it to the Recycle Bin ([`Bridge::delete_notebook_folder`]). + pub(crate) fn purge(&mut self, ids: &[String]) -> IpcResult> { + let mut pages = Vec::new(); + for entry in self.entries_for(ids)? { + match entry { + Entry::Item(notebook, item) => { + self.notes.pending.remove(&item_key(notebook.path(), item)); + let dir = opennote_core::store::layout::NotebookLayout::new(notebook.path()).trash_item_dir(item); + let inside = super::purged::pages_under(&dir); + notebook.purge(item).map_err(from_core)?; + pages.extend(inside); + } + Entry::Notebook(removed) => { + if !removed.entry.path.exists() { + self.core.forget_notebook(&removed.entry.path).map_err(from_core)?; + } + } + } + } + Ok(pages) + } + + /// Delete folder for a notebook in Trash: its folder moves to the Recycle Bin, never deleted outright, and the + /// notebook leaves Trash. Returns the pages that were in it. Only a notebook that is in Trash qualifies. + pub(crate) fn delete_notebook_folder(&mut self, id: &str) -> IpcResult> { + let removed = self.core.library().removed; + let Some(removed) = removed.into_iter().find(|r| r.entry.notebook.to_string() == id) else { + return Err(not_found(id)); + }; + let path = removed.entry.path; + let pages = super::purged::pages_under(&path); + if path.exists() { + super::purged::recycle(&path)?; + } + self.notes.pending.remove(¬ebook_key(&path)); + self.core.forget_notebook(&path).map_err(from_core)?; + Ok(pages) + } +} diff --git a/app/src-tauri/src/notes/tree.rs b/app/src-tauri/src/notes/tree.rs new file mode 100644 index 00000000..c3e0e434 --- /dev/null +++ b/app/src-tauri/src/notes/tree.rs @@ -0,0 +1,272 @@ +//! Reading the tree: finding nodes, their summaries, children in display order, and the first load along a path. + +use std::collections::{BTreeMap, HashMap}; + +use opennote_core::{ + model::{NotebookTree, TreeChild}, + session::{ + notebook::{NodeRef, NotebookHandle}, + notes::{all_nodes, children_of, node, notebook_node, NodeInfo, NodeKind}, + }, + GroupId, Id, SectionId, +}; + +use super::{not_found, InitialTree, LibraryOut, NodeSummary}; +use crate::{core_bridge::Bridge, ipc::IpcResult}; + +/// The deepest section groups nest: a top-level group is at depth 1 (NOTES_LIMITS.groupDepth). +pub const GROUP_DEPTH: u32 = 4; + +/// A node the commands found: a notebook, or a group, section, or page of one. +#[derive(Clone, Debug)] +pub(crate) enum Found { + Notebook(NotebookHandle), + Node(NotebookHandle, NodeRef), +} + +impl Found { + pub(crate) fn notebook(&self) -> &NotebookHandle { + match self { + Found::Notebook(notebook) | Found::Node(notebook, _) => notebook, + } + } + + pub(crate) fn kind(&self) -> NodeKind { + match self { + Found::Notebook(_) => NodeKind::Notebook, + Found::Node(_, NodeRef::Group(_)) => NodeKind::SectionGroup, + Found::Node(_, NodeRef::Section(_)) => NodeKind::Section, + Found::Node(_, NodeRef::Page(_)) => NodeKind::Page, + } + } + + /// The node's ID text. + pub(crate) fn id(&self) -> String { + match self { + Found::Notebook(notebook) => notebook.id().to_string(), + Found::Node(_, node) => node_id(*node).to_string(), + } + } +} + +/// The ID of a group, section, or page. +pub(crate) fn node_id(node: NodeRef) -> Id { + match node { + NodeRef::Group(GroupId(id)) => id, + NodeRef::Section(SectionId(id)) => id, + NodeRef::Page(page) => page.0, + } +} + +pub(crate) fn kind_name(kind: NodeKind) -> &'static str { + match kind { + NodeKind::Notebook => "notebook", + NodeKind::SectionGroup => "sectionGroup", + NodeKind::Section => "section", + NodeKind::Page => "page", + } +} + +/// A node summary from the core's description of it. +pub(crate) fn summary(info: NodeInfo) -> NodeSummary { + let page = info.kind == NodeKind::Page; + NodeSummary { + id: info.id, + kind: kind_name(info.kind), + parent_id: info.parent_id, + title: info.title, + color: if page { + None + } else { + info.color.map(|color| color.to_text()) + }, + page_level: info.page_level, + child_count: info.child_count, + created: info.created.to_rfc3339(), + modified: info.modified.to_rfc3339(), + read_only: info.read_only, + pinned: info.pinned, + archived: info.archived, + encrypted: info.encrypted, + protected: match info.protection { + opennote_core::session::notes::Protection::None => None, + opennote_core::session::notes::Protection::Locked => Some("locked"), + opennote_core::session::notes::Protection::Unlocked => Some("unlocked"), + }, + } +} + +/// The groups that hold a group or section, innermost first. +pub(crate) fn group_chain(tree: &NotebookTree, mut group: Option) -> Vec { + let mut chain = Vec::new(); + while let Some(id) = group { + if chain.contains(&id) { + break; + } + chain.push(id); + group = tree.groups.iter().find(|g| g.id == id).and_then(|g| g.parent); + } + chain +} + +/// How deep a group sits: 1 at the top level. The notebook is at depth 0. +pub(crate) fn depth_of(tree: &NotebookTree, group: Option) -> u32 { + u32::try_from(group_chain(tree, group).len()).unwrap_or(u32::MAX) +} + +/// How many levels of groups a node takes: 0 for a section, 1 for a group without groups inside, and so on. +pub(crate) fn height_of(tree: &NotebookTree, node: NodeRef) -> u32 { + let NodeRef::Group(group) = node else { + return 0; + }; + let inner = tree + .groups + .iter() + .filter(|g| g.parent == Some(group)) + .map(|g| height_of(tree, NodeRef::Group(g.id))) + .max() + .unwrap_or(0); + inner.saturating_add(1) +} + +/// The child IDs of a notebook or group, in display order. +pub(crate) fn container_children(tree: &NotebookTree, group: Option) -> Vec { + tree.children(group) + .iter() + .map(|child| match child { + TreeChild::Group(g) => g.id.to_string(), + TreeChild::Section(s) => s.id.to_string(), + }) + .collect() +} + +/// A section's pages in display order, with their levels. +pub(crate) fn flat_pages(tree: &NotebookTree, section: SectionId) -> Vec<(String, u8)> { + tree.section(section) + .map(|s| s.pages.iter().map(|p| (p.id.to_string(), p.level)).collect()) + .unwrap_or_default() +} + +/// The page at `index` and the pages after it with a higher level: the page with its subpages. +pub(crate) fn block_at(flat: &[(String, u8)], index: usize) -> Vec { + let Some((_, root)) = flat.get(index) else { + return Vec::new(); + }; + let mut block = Vec::new(); + for (i, (id, level)) in flat.iter().enumerate().skip(index) { + if i > index && level <= root { + break; + } + block.push(id.clone()); + } + block +} + +impl Bridge { + /// The open notebooks of the library, in display order. + pub(crate) fn notebooks(&self) -> Vec { + self.core + .library() + .notebooks + .iter() + .filter_map(|entry| self.core.find_open(&entry.path)) + .collect() + } + + /// The node with this ID, or `not-found`. + pub(crate) fn find(&self, id: &str) -> IpcResult { + let parsed = Id::parse(id).map_err(|_| not_found(id))?; + let listed = self.notebooks(); + match self.core.find_node(parsed) { + Some((notebook, found)) if listed.iter().any(|n| n.id() == notebook.id()) => Ok(match found { + None => Found::Notebook(notebook), + Some(node) => Found::Node(notebook, node), + }), + _ => Err(not_found(id)), + } + } + + /// The summary of a node that was just found or made. + pub(crate) fn summary_of(&self, found: &Found) -> IpcResult { + let tree = found.notebook().tree(); + let info = match found { + Found::Notebook(_) => Some(notebook_node(&tree)), + Found::Node(_, node_ref) => node(&tree, node_id(*node_ref)), + }; + info.map(summary).ok_or_else(|| not_found(&found.id())) + } + + pub(crate) fn list_notebooks(&self) -> Vec { + self.notebooks() + .iter() + .map(|notebook| summary(notebook_node(¬ebook.tree()))) + .collect() + } + + pub(crate) fn list_children(&self, parent: &str) -> IpcResult> { + let found = self.find(parent)?; + if let Found::Node(_, NodeRef::Page(_)) = found { + return Ok(Vec::new()); + } + let tree = found.notebook().tree(); + let id = Id::parse(parent).map_err(|_| not_found(parent))?; + let children = children_of(&tree, id).ok_or_else(|| not_found(parent))?; + Ok(children.into_iter().map(summary).collect()) + } + + pub(crate) fn get_node(&self, id: &str) -> Option { + self.find(id).ok().and_then(|found| self.summary_of(&found).ok()) + } + + /// Every listed node's place in display order: the notebooks in library order, each followed by its tree. + pub(crate) fn display_order(&self) -> HashMap { + let mut order = HashMap::new(); + for notebook in self.notebooks() { + for info in all_nodes(¬ebook.tree()) { + let next = order.len(); + order.entry(info.id).or_insert(next); + } + } + order + } + + pub(crate) fn load_initial(&self, path: &[String]) -> IpcResult { + let mut resolved: Vec<(String, NodeSummary)> = Vec::new(); + for id in path { + let parent = resolved.last().map(|(id, _)| id.clone()); + let Some(found) = self.get_node(id) else { + break; + }; + if found.parent_id != parent { + break; + } + resolved.push((id.clone(), found)); + } + let mut children = BTreeMap::new(); + for (id, found) in &resolved { + if found.kind != "page" { + children.insert(id.clone(), self.list_children(id)?); + } + } + let page = resolved + .last() + .filter(|(_, found)| found.kind == "page") + .map(|(_, found)| found.clone()); + let folder = self + .notes + .folder + .as_ref() + .map(|f| f.display().to_string()) + .unwrap_or_default(); + Ok(InitialTree { + library: LibraryOut { + folder, + read_only: false, + }, + notebooks: self.list_notebooks(), + children, + resolved_path: resolved.into_iter().map(|(id, _)| id).collect(), + page, + }) + } +} diff --git a/app/src-tauri/src/page_docx.rs b/app/src-tauri/src/page_docx.rs new file mode 100644 index 00000000..d7f8da94 --- /dev/null +++ b/app/src-tauri/src/page_docx.rs @@ -0,0 +1,149 @@ +//! A Word file for a selection (Phase 6, "Export a selection"). The interface draws the selection as a picture, with +//! everything it holds: text, tables, ink, and the cropped part of an annotated image. This command puts that picture +//! in a `.docx` through the interop crate's writer, with the selection's text as the picture's description, so a +//! screen reader and Word's alt text both have it. +//! +//! The picture comes as the request body and its facts as a percent-encoded JSON header, as `export_write` does, so the +//! bytes never become base64 text. The file's bytes go back the same way. The interface saves them through the Save +//! dialog and `export_write`, so this command never names a path. + +use std::collections::HashMap; + +use opennote_core::Timestamp; +use opennote_interop::{ + doc::{Block, Inline}, + docx::{self, Media, Part, WordInput}, +}; +use serde::Deserialize; +use tauri::ipc::{InvokeBody, Response}; + +use crate::{ + images::import::percent_decode, + ipc::{IpcError, IpcResult}, +}; + +/// The largest picture the command takes: a PNG over 100 MB would not open in Word comfortably anyway. +const MAX_PICTURE: usize = 100 * 1024 * 1024; +/// The sides of the picture, in pixels, that Word is asked to hold. +const MAX_SIDE: u32 = 20_000; +const MAX_TEXT: usize = 20_000; +const PNG_SIGNATURE: [u8; 8] = [0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a]; + +/// What the interface says about the picture. +#[derive(Debug, Deserialize)] +struct Facts { + title: String, + /// The text of the selection, for the picture's description. + alt: String, + width: u32, + height: u32, +} + +/// The document for a PNG and its facts: a title line, then the picture alone. +fn build_document(png: &[u8], facts: &Facts, when: Timestamp) -> IpcResult> { + if png.len() > MAX_PICTURE || !png.starts_with(&PNG_SIGNATURE) { + return Err(IpcError::invalid("body", "The picture isn't a PNG.")); + } + let sides = 1..=MAX_SIDE; + if !sides.contains(&facts.width) || !sides.contains(&facts.height) { + return Err(IpcError::invalid("size", "The picture's size is out of range.")); + } + let clip = |text: &str| text.chars().take(MAX_TEXT).collect::(); + let picture = Inline::Image { + dest: "selection".to_owned(), + alt: clip(&facts.alt), + }; + let mut media = HashMap::new(); + media.insert( + "selection".to_owned(), + Media { + bytes: png.to_vec(), + ext: "png".to_owned(), + width: facts.width, + height: facts.height, + }, + ); + let input = WordInput { + title: clip(&facts.title), + created: when, + modified: when, + parts: vec![Part::Blocks(vec![Block::Paragraph(vec![picture])])], + media, + }; + docx::build(&input).map_err(|error| IpcError::new("export", error.to_string())) +} + +/// Makes a `.docx` that holds the selection's picture. The body is the PNG, and the `x-opennote-docx` header (percent- +/// encoded JSON) holds the title, the description, and the picture's size in pixels. +#[tauri::command] +pub async fn export_selection_docx(request: tauri::ipc::Request<'_>) -> IpcResult { + let header = request + .headers() + .get("x-opennote-docx") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| IpcError::invalid("x-opennote-docx", "The Word header is missing."))?; + let facts: Facts = serde_json::from_str(&percent_decode(header)) + .map_err(|_| IpcError::invalid("x-opennote-docx", "The Word header isn't valid JSON."))?; + let png = match request.body() { + InvokeBody::Raw(bytes) => bytes.clone(), + InvokeBody::Json(_) => return Err(IpcError::invalid("body", "The picture must come as raw bytes.")), + }; + let when = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| Timestamp::from_unix_ms(i64::try_from(elapsed.as_millis()).unwrap_or(0))) + .unwrap_or_default(); + let bytes = tauri::async_runtime::spawn_blocking(move || build_document(&png, &facts, when)) + .await + .map_err(|error| IpcError::new("internal", error.to_string()))??; + Ok(Response::new(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + use opennote_interop::archive::{ArchiveLimits, ZipArchive}; + + /// The smallest PNG signature plus junk: enough for the signature check, since the writer stores the bytes as given. + fn png() -> Vec { + let mut bytes = PNG_SIGNATURE.to_vec(); + bytes.extend_from_slice(b"not really an image"); + bytes + } + + fn facts(width: u32, height: u32) -> Facts { + Facts { + title: "Notes".to_owned(), + alt: "A sketch of a & axes".to_owned(), + width, + height, + } + } + + #[test] + fn the_document_holds_the_picture_with_its_description() { + let bytes = build_document(&png(), &facts(640, 480), Timestamp::from_unix_ms(1_000)).unwrap(); + let mut zip = ZipArchive::new(std::io::Cursor::new(bytes), ArchiveLimits::default()).unwrap(); + let names: Vec = zip.entries().iter().map(|entry| entry.name.clone()).collect(); + assert!(names.iter().any(|name| name == "word/document.xml"), "{names:?}"); + assert!(names.iter().any(|name| name.starts_with("word/media/")), "{names:?}"); + let xml = String::from_utf8_lossy(&zip.read_named("word/document.xml").unwrap()).into_owned(); + assert!(xml.contains("<graph> & axes"), "{xml}"); + assert!(xml.contains(">, + written: Mutex>, +} + +impl ExportGrants { + fn grant(&self, path: PathBuf) { + if let Ok(mut set) = self.chosen.lock() { + set.insert(path); + } + } + + pub(crate) fn take(&self, path: &Path) -> bool { + self.chosen.lock().map(|mut set| set.remove(path)).unwrap_or(false) + } + + pub(crate) fn mark_written(&self, path: PathBuf) { + if let Ok(mut set) = self.written.lock() { + set.insert(path); + } + } + + fn was_written(&self, path: &Path) -> bool { + self.written.lock().map(|set| set.contains(path)).unwrap_or(false) + } +} + +/// The label of a job's window. A job ID is letters, digits, `-`, and `_`, so it can't name another window. +fn job_label(job: &str) -> IpcResult { + let valid = + !job.is_empty() && job.len() <= 48 && job.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); + if valid { + Ok(format!("print-{job}")) + } else { + Err(IpcError::invalid("job", "The export job's name isn't valid.")) + } +} + +fn not_found(what: &str) -> IpcError { + IpcError::new("notFound", format!("{what} isn't open.")) +} + +fn close_window(app: &AppHandle, label: &str) { + if let Some(window) = app.get_webview_window(label) { + let _ = window.destroy(); + } +} + +/// Waits for the print window's report for `job`, and returns its result. +fn wait_for_report(rx: &mpsc::Receiver, job: &str, limit: Duration) -> IpcResult { + let deadline = Instant::now() + limit; + loop { + let left = deadline.saturating_duration_since(Instant::now()); + let payload = rx + .recv_timeout(left) + .map_err(|_| IpcError::new("timeout", "The print window took too long to prepare the page."))?; + // The payload is the JSON text of what the window emitted: an object, or a string holding one. + let value: Value = serde_json::from_str(&payload).unwrap_or(Value::Null); + let value = match value { + Value::String(text) => serde_json::from_str(&text).unwrap_or(Value::Null), + other => other, + }; + if value["job"].as_str() != Some(job) { + continue; + } + return if value["ok"].as_bool() == Some(true) { + Ok(value["result"].clone()) + } else { + let message = value["message"].as_str().unwrap_or("The page couldn't be prepared."); + Err(IpcError::new(codes::INTERNAL, message.to_owned())) + }; + } +} + +/// Opens the hidden print window on `input` (a `PrepareInput`) and returns what the window planned: the sheets, +/// the breaks, the print plan, and the warnings. The window stays open for `print_render`. +#[tauri::command] +pub async fn print_prepare(app: AppHandle, job: String, input: Value) -> IpcResult { + let label = job_label(&job)?; + close_window(&app, &label); + let setup = serde_json::to_string(&json!({ "job": job, "input": input })) + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))?; + let (tx, rx) = mpsc::channel::(); + let listener = app.listen_any(RESULT_EVENT, move |event| { + let _ = tx.send(event.payload().to_owned()); + }); + let built = WebviewWindowBuilder::new(&app, &label, WebviewUrl::App("print.html".into())) + .title("OpenNote print") + .inner_size(1280.0, 960.0) + .visible(false) + .decorations(false) + .skip_taskbar(true) + .focused(false) + .initialization_script(format!("window.__OPENNOTE_PRINT__ = {setup};")) + .on_navigation(crate::window::is_app_url) + .build(); + if let Err(error) = built { + app.unlisten(listener); + return Err(error.into()); + } + let wanted = job.clone(); + let outcome = tauri::async_runtime::spawn_blocking(move || wait_for_report(&rx, &wanted, PREPARE_TIMEOUT)).await; + app.unlisten(listener); + let outcome = outcome.map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))?; + if outcome.is_err() { + close_window(&app, &label); + } + outcome +} + +/// The sheet size in inches: both sides in range. +fn check_sheet(width: f64, height: f64) -> IpcResult<()> { + let fits = |side: f64| side.is_finite() && (MIN_SIDE..=MAX_SIDE).contains(&side); + if fits(width) && fits(height) { + Ok(()) + } else { + Err(IpcError::invalid("size", "The sheet size is out of range.")) + } +} + +/// Prints the shown document of a job's window to PDF with WebView2's `PrintToPdf`, and returns the file's bytes. +/// The sheet size is the box of the first sheet in inches; the document's own `@page` rule sizes every page. +#[tauri::command] +pub async fn print_render( + app: AppHandle, + job: String, + width: f64, + height: f64, + background: bool, + tagged: Option, + outline: Option, +) -> IpcResult { + check_sheet(width, height)?; + let label = job_label(&job)?; + let window = app + .get_webview_window(&label) + .ok_or_else(|| not_found("The print window"))?; + let (tagged, outline) = (tagged.unwrap_or(false), outline.unwrap_or(false)); + if tagged || outline { + // The DevTools route writes structure tags, a language, and bookmarks. If it fails, the plain route below + // still makes the PDF, and the export's check tells the person it has no tags. + match render_with_devtools(&window, (width, height), background, tagged, outline).await { + Ok(bytes) => return Ok(Response::new(bytes)), + Err(error) => log::warn!("Tagged PDF failed, printing without tags: {error:?}"), + } + } + let file = std::env::temp_dir().join(format!("opennote-print-{}-{job}.pdf", std::process::id())); + let (tx, rx) = mpsc::channel::>(); + start_print(&window, file.clone(), (width, height), background, tx)?; + let outcome = tauri::async_runtime::spawn_blocking(move || rx.recv_timeout(RENDER_TIMEOUT)) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))?; + let bytes = match outcome { + Ok(Ok(true)) => fs::read(&file).map_err(IpcError::from), + Ok(Ok(false)) => Err(IpcError::new(codes::INTERNAL, "WebView2 didn't write the PDF.")), + Ok(Err(message)) => Err(IpcError::new(codes::INTERNAL, format!("PrintToPdf failed: {message}"))), + Err(_) => Err(IpcError::new("timeout", "Printing to PDF took too long.")), + }; + let _ = fs::remove_file(&file); + Ok(Response::new(bytes?)) +} + +/// The parameters of the DevTools `Page.printToPDF` call for a sheet of `width` by `height` inches. The document's +/// own `@page` rule still sizes each page (`preferCSSPageSize`), as it does for `PrintToPdf`. +fn devtools_params(width: f64, height: f64, background: bool, tagged: bool, outline: bool) -> Value { + json!({ + "landscape": false, + "displayHeaderFooter": false, + "printBackground": background, + "scale": 1, + "paperWidth": width, + "paperHeight": height, + "marginTop": 0, + "marginBottom": 0, + "marginLeft": 0, + "marginRight": 0, + "preferCSSPageSize": true, + "generateTaggedPDF": tagged, + "generateDocumentOutline": outline, + }) +} + +/// Decodes standard base64 (with or without padding), or `None` when the text holds anything else. +fn decode_base64(text: &str) -> Option> { + let mut out = Vec::with_capacity(text.len() / 4 * 3); + let (mut bits, mut count) = (0u32, 0u8); + for byte in text.bytes() { + let value = match byte { + b'A'..=b'Z' => byte - b'A', + b'a'..=b'z' => byte - b'a' + 26, + b'0'..=b'9' => byte - b'0' + 52, + b'+' => 62, + b'/' => 63, + b'=' | b'\r' | b'\n' => continue, + _ => return None, + }; + bits = (bits << 6) | u32::from(value); + count += 6; + if count >= 8 { + count -= 8; + out.push((bits >> count) as u8); + bits &= (1 << count) - 1; + } + } + Some(out) +} + +/// The PDF bytes in a `Page.printToPDF` reply (`{"data": ""}`). +fn pdf_from_reply(reply: &str) -> Result, String> { + let value: Value = serde_json::from_str(reply).map_err(|error| error.to_string())?; + let data = value["data"].as_str().ok_or("The reply holds no PDF.")?; + decode_base64(data).ok_or_else(|| "The PDF in the reply isn't valid base64.".to_owned()) +} + +/// Prints the window's document with Chromium's `Page.printToPDF`, which can write structure tags and bookmarks. +async fn render_with_devtools( + window: &WebviewWindow, + (width, height): (f64, f64), + background: bool, + tagged: bool, + outline: bool, +) -> IpcResult> { + let params = devtools_params(width, height, background, tagged, outline).to_string(); + let (tx, rx) = mpsc::channel::>(); + start_devtools_print(window, params, tx)?; + let outcome = tauri::async_runtime::spawn_blocking(move || rx.recv_timeout(RENDER_TIMEOUT)) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))?; + match outcome { + Ok(Ok(reply)) => pdf_from_reply(&reply).map_err(|message| IpcError::new(codes::INTERNAL, message)), + Ok(Err(message)) => Err(IpcError::new( + codes::INTERNAL, + format!("Page.printToPDF failed: {message}"), + )), + Err(_) => Err(IpcError::new("timeout", "Printing to PDF took too long.")), + } +} + +#[cfg(windows)] +fn start_devtools_print( + window: &WebviewWindow, + params: String, + tx: mpsc::Sender>, +) -> IpcResult<()> { + use webview2_com::CallDevToolsProtocolMethodCompletedHandler; + use windows::core::HSTRING; + + window + .with_webview(move |webview| { + // SAFETY: the controller is a live COM object, and this closure runs on the main thread that owns the + // window. The completion handler sends the reply and nothing else. + let started = unsafe { + (|| -> windows::core::Result<()> { + let core = webview.controller().CoreWebView2()?; + let done = tx.clone(); + let handler = CallDevToolsProtocolMethodCompletedHandler::create(Box::new(move |result, reply| { + let _ = done.send(result.map(|()| reply).map_err(|error| error.to_string())); + Ok(()) + })); + core.CallDevToolsProtocolMethod( + &HSTRING::from("Page.printToPDF"), + &HSTRING::from(params.as_str()), + &handler, + ) + })() + }; + if let Err(error) = started { + let _ = tx.send(Err(error.to_string())); + } + }) + .map_err(IpcError::from) +} + +#[cfg(not(windows))] +fn start_devtools_print( + _window: &WebviewWindow, + _params: String, + _tx: mpsc::Sender>, +) -> IpcResult<()> { + Err(IpcError::not_implemented("print_render")) +} + +#[cfg(windows)] +fn start_print( + window: &WebviewWindow, + file: PathBuf, + (width, height): (f64, f64), + background: bool, + tx: mpsc::Sender>, +) -> IpcResult<()> { + use webview2_com::{ + Microsoft::Web::WebView2::Win32::{ + ICoreWebView2Environment6, ICoreWebView2_2, ICoreWebView2_7, COREWEBVIEW2_PRINT_ORIENTATION_LANDSCAPE, + COREWEBVIEW2_PRINT_ORIENTATION_PORTRAIT, + }, + PrintToPdfCompletedHandler, + }; + use windows::core::{Interface, HSTRING}; + + window + .with_webview(move |webview| { + // SAFETY: the controller and the print settings are live COM objects, and this closure runs on the + // main thread that owns the window. The completion handler sends the outcome and nothing else. + let started = unsafe { + (|| -> windows::core::Result<()> { + let core = webview.controller().CoreWebView2()?; + let environment: ICoreWebView2Environment6 = + core.cast::()?.Environment()?.cast()?; + let settings = environment.CreatePrintSettings()?; + settings.SetOrientation(if width > height { + COREWEBVIEW2_PRINT_ORIENTATION_LANDSCAPE + } else { + COREWEBVIEW2_PRINT_ORIENTATION_PORTRAIT + })?; + settings.SetScaleFactor(1.0)?; + // The orientation turns the page, so the sides go in the way round the paper has them. + settings.SetPageWidth(width.min(height))?; + settings.SetPageHeight(width.max(height))?; + settings.SetMarginTop(0.0)?; + settings.SetMarginBottom(0.0)?; + settings.SetMarginLeft(0.0)?; + settings.SetMarginRight(0.0)?; + settings.SetShouldPrintBackgrounds(background)?; + settings.SetShouldPrintHeaderAndFooter(false)?; + settings.SetShouldPrintSelectionOnly(false)?; + let done = tx.clone(); + let handler = PrintToPdfCompletedHandler::create(Box::new(move |result, succeeded| { + let _ = done.send(result.map(|()| succeeded).map_err(|error| error.to_string())); + Ok(()) + })); + core.cast::()? + .PrintToPdf(&HSTRING::from(file.as_path()), &settings, &handler) + })() + }; + if let Err(error) = started { + let _ = tx.send(Err(error.to_string())); + } + }) + .map_err(IpcError::from) +} + +#[cfg(not(windows))] +fn start_print( + _window: &WebviewWindow, + _file: PathBuf, + _size: (f64, f64), + _background: bool, + _tx: mpsc::Sender>, +) -> IpcResult<()> { + Err(IpcError::not_implemented("print_render")) +} + +/// Closes a job's print window. A window that is already gone is not an error. +#[tauri::command] +pub async fn print_close(app: AppHandle, job: String) -> IpcResult<()> { + close_window(&app, &job_label(&job)?); + Ok(()) +} + +/// Shows Windows' Save dialog and returns the chosen path, or `None` when the person cancels. The path is +/// remembered, so `export_write` accepts it once. +#[tauri::command] +pub async fn export_pick_save( + window: WebviewWindow, + suggested: String, + label: String, + extension: String, +) -> IpcResult> { + if extension.is_empty() || !extension.chars().all(|c| c.is_ascii_alphanumeric()) || extension.len() > 12 { + return Err(IpcError::invalid("extension", "The file type isn't valid.")); + } + let owner = window.hwnd().map(|hwnd| hwnd.0 as isize).unwrap_or(0); + let picked = tauri::async_runtime::spawn_blocking(move || save_dialog(owner, &suggested, &label, &extension)) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))??; + Ok(picked.map(|path| { + window.app_handle().state::().grant(path.clone()); + path.display().to_string() + })) +} + +#[cfg(windows)] +fn save_dialog(owner: isize, suggested: &str, label: &str, extension: &str) -> IpcResult> { + use windows::{ + core::{HSTRING, PCWSTR}, + Win32::{ + Foundation::{ERROR_CANCELLED, HWND}, + System::Com::{ + CoCreateInstance, CoInitializeEx, CoTaskMemFree, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_APARTMENTTHREADED, + }, + UI::Shell::{ + Common::COMDLG_FILTERSPEC, FileSaveDialog, IFileSaveDialog, FOS_FORCEFILESYSTEM, FOS_OVERWRITEPROMPT, + FOS_PATHMUSTEXIST, SIGDN_FILESYSPATH, + }, + }, + }; + + // SAFETY: COM is initialized for this thread and released before it returns, and every object below lives + // within that span. The owner handle names the main window, or is null for none. + unsafe { + let apartment = CoInitializeEx(None, COINIT_APARTMENTTHREADED); + let result = (|| -> windows::core::Result> { + let dialog: IFileSaveDialog = CoCreateInstance(&FileSaveDialog, None, CLSCTX_INPROC_SERVER)?; + dialog.SetOptions(dialog.GetOptions()? | FOS_FORCEFILESYSTEM | FOS_OVERWRITEPROMPT | FOS_PATHMUSTEXIST)?; + let name = HSTRING::from(format!("{label} (*.{extension})")); + let spec = HSTRING::from(format!("*.{extension}")); + let types = [COMDLG_FILTERSPEC { + pszName: PCWSTR(name.as_ptr()), + pszSpec: PCWSTR(spec.as_ptr()), + }]; + dialog.SetFileTypes(&types)?; + dialog.SetFileTypeIndex(1)?; + dialog.SetDefaultExtension(&HSTRING::from(extension))?; + dialog.SetFileName(&HSTRING::from(suggested))?; + let parent = (owner != 0).then_some(HWND(owner as *mut _)); + if let Err(error) = dialog.Show(parent) { + return if error.code() == ERROR_CANCELLED.to_hresult() { + Ok(None) + } else { + Err(error) + }; + } + let name = dialog.GetResult()?.GetDisplayName(SIGDN_FILESYSPATH)?; + let path = name.to_string().ok().map(PathBuf::from); + CoTaskMemFree(Some(name.0.cast_const().cast())); + Ok(path) + })(); + if apartment.is_ok() { + CoUninitialize(); + } + Ok(result?) + } +} + +#[cfg(not(windows))] +fn save_dialog(_owner: isize, _suggested: &str, _label: &str, _extension: &str) -> IpcResult> { + Err(IpcError::not_implemented("export_pick_save")) +} + +/// The header of `export_write` (percent-encoded JSON). It holds the path the Save dialog returned, and the files in +/// the body, one after the other. The first part is the file at `path`. Each later part is a path relative to that +/// file's folder. +#[derive(Debug, Deserialize)] +struct WriteHeader { + path: String, + parts: Vec, +} + +#[derive(Debug, Deserialize)] +struct Part { + /// Relative to the main file's folder; empty for the main file itself. + #[serde(default)] + path: String, + length: usize, +} + +/// A sibling's path: relative, no `..`, no drive, and no empty or reserved parts. +fn sibling_path(folder: &Path, relative: &str) -> IpcResult { + let bad = || IpcError::invalid("parts", "A file's path isn't valid."); + if relative.is_empty() || relative.len() > 240 || relative.contains(['\\', ':', '\0']) { + return Err(bad()); + } + let mut path = folder.to_path_buf(); + for part in relative.split('/') { + let clean = !part.is_empty() + && part != "." + && part != ".." + && !part.ends_with(['.', ' ']) + && !part.contains(['<', '>', '"', '|', '?', '*']); + if !clean || Path::new(part).components().any(|c| !matches!(c, Component::Normal(_))) { + return Err(bad()); + } + path.push(part); + } + Ok(path) +} + +/// Writes `bytes` to `path` through a temporary file next to it, so a failed write never leaves half a file. +fn write_safely(path: &Path, bytes: &[u8]) -> IpcResult<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let temp = path.with_extension(format!( + "{}.part", + path.extension().and_then(|e| e.to_str()).unwrap_or("tmp") + )); + let written = fs::write(&temp, bytes).and_then(|()| fs::rename(&temp, path)); + if written.is_err() { + let _ = fs::remove_file(&temp); + } + Ok(written?) +} + +/// Writes the files of an export. The body holds the parts' bytes in order, and the `x-opennote-export` header +/// (JSON) names the chosen path and each part's length. Only a path from `export_pick_save` is written, once. +#[tauri::command] +pub async fn export_write(app: AppHandle, request: tauri::ipc::Request<'_>) -> IpcResult<()> { + let header = request + .headers() + .get("x-opennote-export") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| IpcError::invalid("x-opennote-export", "The export header is missing."))?; + // The header is percent-encoded, because a header holds ASCII only and a path may hold any letter. + let header: WriteHeader = serde_json::from_str(&percent_decode(header)) + .map_err(|_| IpcError::invalid("x-opennote-export", "The export header isn't valid JSON."))?; + let body = match request.body() { + InvokeBody::Raw(bytes) => bytes.clone(), + InvokeBody::Json(_) => return Err(IpcError::invalid("body", "The files must come as raw bytes.")), + }; + let granted = PathBuf::from(&header.path); + if !app.state::().take(&granted) { + return Err(IpcError::invalid("path", "Choose where to save before writing.")); + } + let main = granted.clone(); + tauri::async_runtime::spawn_blocking(move || write_parts(&granted, &header.parts, &body)) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))??; + app.state::().mark_written(main); + Ok(()) +} + +/// Opens a file this session exported in its default app, or shows it in File Explorer. Any other path is refused. +#[tauri::command] +pub async fn export_open(app: AppHandle, path: String, reveal: bool) -> IpcResult<()> { + let file = PathBuf::from(&path); + if !app.state::().was_written(&file) { + return Err(IpcError::invalid( + "path", + "Only a file this export saved can be opened.", + )); + } + tauri::async_runtime::spawn_blocking(move || open_file(&file, reveal)) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))? +} + +#[cfg(windows)] +fn open_file(file: &Path, reveal: bool) -> IpcResult<()> { + use windows::{ + core::{w, HSTRING}, + Win32::UI::{Shell::ShellExecuteW, WindowsAndMessaging::SW_SHOWNORMAL}, + }; + + // SAFETY: both strings outlive the call, and ShellExecuteW takes no other pointers. + let result = unsafe { + if reveal { + let argument = HSTRING::from(format!("/select,\"{}\"", file.display())); + ShellExecuteW(None, w!("open"), w!("explorer.exe"), &argument, None, SW_SHOWNORMAL) + } else { + ShellExecuteW( + None, + w!("open"), + &HSTRING::from(file.as_os_str()), + None, + None, + SW_SHOWNORMAL, + ) + } + }; + // ShellExecute reports success with a value above 32. + if (result.0 as isize) > 32 { + Ok(()) + } else { + Err(IpcError::new(codes::INTERNAL, "Windows couldn't open the file.")) + } +} + +#[cfg(not(windows))] +fn open_file(_file: &Path, _reveal: bool) -> IpcResult<()> { + Err(IpcError::not_implemented("export_open")) +} + +fn write_parts(main: &Path, parts: &[Part], body: &[u8]) -> IpcResult<()> { + let total = parts.iter().try_fold(0usize, |sum, part| sum.checked_add(part.length)); + if total != Some(body.len()) || body.len() > MAX_EXPORT_BYTES || parts.is_empty() { + return Err(IpcError::invalid("parts", "The files don't add up to the data sent.")); + } + let folder = main.parent().unwrap_or(Path::new("")); + let mut offset = 0; + for (index, part) in parts.iter().enumerate() { + let target = if index == 0 { + main.to_path_buf() + } else { + sibling_path(folder, &part.path)? + }; + write_safely(&target, &body[offset..offset + part.length])?; + offset += part.length; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn devtools_params_ask_for_tags_and_bookmarks() { + let params = devtools_params(8.5, 11.0, true, true, false); + assert_eq!(params["generateTaggedPDF"], true); + assert_eq!(params["generateDocumentOutline"], false); + assert_eq!(params["preferCSSPageSize"], true); + assert_eq!(params["paperWidth"], 8.5); + assert_eq!(params["printBackground"], true); + } + + #[test] + fn base64_decodes_with_and_without_padding() { + assert_eq!(decode_base64("").unwrap(), b""); + assert_eq!(decode_base64("TWFu").unwrap(), b"Man"); + assert_eq!(decode_base64("TWE=").unwrap(), b"Ma"); + assert_eq!(decode_base64("TQ==").unwrap(), b"M"); + assert_eq!(decode_base64("TQ").unwrap(), b"M"); + assert_eq!(decode_base64("+/8=").unwrap(), [0xfb, 0xff]); + assert!(decode_base64("T*Q=").is_none()); + } + + #[test] + fn the_pdf_comes_out_of_the_devtools_reply() { + assert_eq!(pdf_from_reply(r#"{"data":"JVBERg=="}"#).unwrap(), b"%PDF"); + assert!(pdf_from_reply("{}").is_err()); + assert!(pdf_from_reply("not json").is_err()); + assert!(pdf_from_reply(r#"{"data":"%%"}"#).is_err()); + } + + #[test] + fn job_labels_hold_only_safe_characters() { + assert_eq!(job_label("a1-b_2").unwrap(), "print-a1-b_2"); + for bad in ["", "a b", "../x", "x/y", "é", &"a".repeat(49)] { + assert!(job_label(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn sheet_sizes_stay_in_range() { + assert!(check_sheet(8.5, 11.0).is_ok()); + assert!(check_sheet(0.0, 11.0).is_err()); + assert!(check_sheet(f64::NAN, 11.0).is_err()); + assert!(check_sheet(8.5, 500.0).is_err()); + } + + #[test] + fn siblings_stay_inside_the_folder() { + let folder = Path::new("C:/Notes"); + assert!(sibling_path(folder, "Title_files/a.png").is_ok()); + for bad in [ + "", "../a.png", "a/../b", "/abs.png", "C:/x.png", "a\\b", "a//b", "con?.png", "a.", "./a", + ] { + assert!(sibling_path(folder, bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn a_granted_path_is_good_once() { + let grants = ExportGrants::default(); + let path = PathBuf::from("C:/Notes/Page.pdf"); + assert!(!grants.take(&path)); + grants.grant(path.clone()); + assert!(grants.take(&path)); + assert!(!grants.take(&path)); + } + + #[test] + fn writes_the_main_file_and_its_siblings() { + let dir = tempfile::tempdir().unwrap(); + let main = dir.path().join("Page.md"); + let parts = [ + Part { + path: String::new(), + length: 3, + }, + Part { + path: "Page_files/a.png".into(), + length: 2, + }, + ]; + write_parts(&main, &parts, b"abcde").unwrap(); + assert_eq!(fs::read(&main).unwrap(), b"abc"); + assert_eq!(fs::read(dir.path().join("Page_files/a.png")).unwrap(), b"de"); + assert!(!dir.path().join("Page.md.part").exists()); + } + + #[test] + fn refuses_parts_that_do_not_match_the_body() { + let dir = tempfile::tempdir().unwrap(); + let main = dir.path().join("Page.md"); + let short = [Part { + path: String::new(), + length: 9, + }]; + assert!(write_parts(&main, &short, b"abc").is_err()); + assert!(write_parts(&main, &[], b"").is_err()); + assert!(!main.exists()); + let escape = [ + Part { + path: String::new(), + length: 1, + }, + Part { + path: "../evil".into(), + length: 1, + }, + ]; + assert!(write_parts(&main, &escape, b"ab").is_err()); + assert!(!dir.path().parent().unwrap().join("evil").exists()); + } + + #[test] + fn reads_the_report_for_its_own_job_only() { + let (tx, rx) = mpsc::channel(); + tx.send(r#"{"job":"other","ok":true,"result":1}"#.to_owned()).unwrap(); + tx.send(r#"{"job":"mine","ok":true,"result":{"sheets":2}}"#.to_owned()) + .unwrap(); + let result = wait_for_report(&rx, "mine", Duration::from_secs(1)).unwrap(); + assert_eq!(result["sheets"], 2); + } + + #[test] + fn a_failed_report_becomes_an_error_and_a_silent_window_times_out() { + let (tx, rx) = mpsc::channel(); + tx.send(r#""{\"job\":\"j\",\"ok\":false,\"message\":\"no fonts\"}""#.to_owned()) + .unwrap(); + let error = wait_for_report(&rx, "j", Duration::from_secs(1)).unwrap_err(); + assert_eq!(error.message, "no fonts"); + let error = wait_for_report(&rx, "j", Duration::from_millis(20)).unwrap_err(); + assert_eq!(error.code, "timeout"); + } +} diff --git a/app/src-tauri/src/page_extras/attach.rs b/app/src-tauri/src/page_extras/attach.rs new file mode 100644 index 00000000..6cd1f1d4 --- /dev/null +++ b/app/src-tauri/src/page_extras/attach.rs @@ -0,0 +1,1548 @@ +// checks-disable-file modifiability: one module whose parts share its state; split it when it grows again + +//! Attached files that save back. A file attached to a page is an asset of that page. +//! +//! Opening it writes a copy to a +//! temporary folder, hands the copy to Windows to open in its own app, and watches the copy. Each time the app +//! has saved a change (the file stops changing for one check), the core imports the new bytes as a new asset. +//! The attachment is then pointed at it: by the window in one undo step when it shows the page and says it made the +//! change, or else by the shell, so a change saved after the person moved on still reaches the note. +//! +//! Each copy has +//! a record of the bytes it last shared with the note, so a copy is never overwritten while it holds changes the +//! note lacks, and is removed once it holds none. The WebView names a page and an asset, never a path. + +use std::{ + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Condvar, Mutex, PoisonError, + }, + thread, + time::{Duration, Instant, SystemTime}, +}; + +use opennote_core::{ + model::JsonMap, + ops::resolve::{Edit, TxnRequest}, + session::page::PageHandle, + store::assets::{mime_for_name, AssetSource}, + AssetId, BlockId, EditError, PageId, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use tauri::{ipc::InvokeBody, AppHandle, Emitter, Manager}; + +use crate::{ + core_bridge::{run_notes, Bridge, CoreBridge}, + images::{ + import::{asset_json, on_blocking, open_page, percent_decode}, + ImportedAsset, + }, + ipc::{codes, IpcError, IpcResult}, +}; + +/// Files over 200 MB are refused. +pub const MAX_ATTACHMENT_BYTES: usize = 200 * 1024 * 1024; +/// The event the interface hears when a change is saved back. +pub const SAVED_EVENT: &str = "attachment://saved"; + +/// Types a note opens in their own app: documents, pictures, audio, video, and ZIP folders, whose apps show them +/// rather than run them. Every other type is shown selected in its folder instead, where the person decides what to +/// do with it, and [`BLOCKED`] types are not even shown. A list of what may open, rather than of what may not, means +/// a type nobody thought of (Windows has hundreds that run something) is never opened by a click in a note. +#[rustfmt::skip] +const OPENS: &[&str] = &[ + // Documents. + "pdf", "txt", "text", "md", "markdown", "csv", "tsv", "log", "rtf", "doc", "docx", "dotx", "xls", "xlsx", "xltx", + "ppt", "pptx", "potx", "ppsx", "odt", "ods", "odp", "odg", + // Pictures. + "png", "jpg", "jpeg", "jfif", "gif", "bmp", "webp", "tif", "tiff", "heic", "heif", "avif", + // Audio. + "mp3", "wav", "m4a", "aac", "flac", "ogg", "oga", "opus", "wma", "mid", "midi", + // Video. + "mp4", "m4v", "mov", "avi", "wmv", "mkv", "webm", "mpg", "mpeg", "3gp", + // A ZIP file opens as a folder. + "zip", +]; + +/// Types that run code, change Windows, or reach out to the network as soon as Windows opens them or Explorer +/// shows them. They can be attached and kept, but a note neither opens them nor shows them in a folder. +#[rustfmt::skip] +const BLOCKED: &[&str] = &[ + // Programs and installers. + "exe", "com", "scr", "pif", "msi", "msp", "mst", "msu", "msix", "msixbundle", "appx", "appxbundle", + "appinstaller", "application", "appref-ms", "xbap", "vsto", "gadget", "jar", "jnlp", "dll", "ocx", "cpl", "sys", + "drv", + // Scripts. + "bat", "cmd", "ps1", "ps1xml", "ps2", "ps2xml", "psc1", "psc2", "psd1", "psm1", "msh", "msh1", "msh2", + "mshxml", "msh1xml", "msh2xml", "vb", "vbs", "vbe", "js", "jse", "ws", "wsf", "wsh", "wsc", "sct", "hta", "py", + "pyw", "pyc", "pyo", "pyz", "pl", "rb", "shs", "shb", "scriptlet", + // Shortcuts and shell files that run or fetch what they point at. + "lnk", "url", "website", "scf", "library-ms", "search-ms", "searchconnector-ms", "settingcontent-ms", "inf", + "reg", "job", "rdp", "ica", + // Help, consoles, and troubleshooters that run script. + "chm", "hlp", "msc", "diagcab", "diagcfg", "diagpkg", + // Disk images, which mount as drives. + "iso", "img", "vhd", "vhdx", + // Office add-ins and databases that run code. + "xll", "xla", "xlam", "ppa", "ppam", "mda", "mde", "accde", "ade", "adp", + // Themes, which fetch files from where they point. + "theme", "themepack", "deskthemepack", +]; + +/// How a note opens an attached file. See [`OPENS`] and [`BLOCKED`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Opening { + /// In its own app. + App, + /// Shown selected in its folder. + Folder, + /// Not at all. + Blocked, +} + +/// Where the attachment was shown, for the interface's message. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Shown { + App, + Folder, +} + +#[derive(Debug, Deserialize)] +struct ImportHeader { + page: String, + name: String, + #[serde(default)] + mime: String, +} + +#[derive(Debug, Clone, Serialize)] +struct Saved { + page: String, + /// The asset the copy was last saved back as. + previous: String, + /// Every asset the copy has been, oldest first: after an undo, a block may show an earlier one. + replaces: Vec, + asset: ImportedAsset, +} + +/// A name that is safe as a file name in the temporary folder: no folders, no reserved characters. +pub fn safe_name(name: &str) -> String { + let last = name.rsplit(['/', '\\']).next().unwrap_or(""); + let cleaned: String = last + .chars() + .filter(|c| !c.is_control() && !"<>:\"|?*".contains(*c)) + .collect(); + let trimmed = cleaned.trim().trim_end_matches('.').trim(); + if trimmed.is_empty() { + "attachment".to_owned() + } else { + trimmed.chars().take(120).collect() + } +} + +/// How a note opens a file of this name. Only the last extension counts, as it does for Windows. +pub fn opening(name: &str) -> Opening { + let Some((_, extension)) = name.rsplit_once('.') else { + return Opening::Folder; + }; + let extension = extension.to_ascii_lowercase(); + if BLOCKED.contains(&extension.as_str()) { + Opening::Blocked + } else if OPENS.contains(&extension.as_str()) { + Opening::App + } else { + Opening::Folder + } +} + +/// Marks a copy as coming from the internet (zone 3), as a browser marks a download. Office then opens it in +/// Protected View, and Windows asks before it runs anything in it. A drive without alternate streams, such as a +/// FAT32 stick, can't hold the mark; the copy opens anyway. +fn mark_from_internet(path: &Path) { + let mut stream = path.as_os_str().to_owned(); + stream.push(":Zone.Identifier"); + if let Err(error) = std::fs::write(PathBuf::from(stream), "[ZoneTransfer]\r\nZoneId=3\r\n") { + ::log::warn!("Couldn't mark an attachment's copy as coming from the internet: {error}"); + } +} + +fn io_error(error: impl std::fmt::Display) -> IpcError { + IpcError::new(codes::IO, error.to_string()) +} + +/// A raw body with the header `x-opennote-attachment: {"page","name","mime"}`. +#[tauri::command] +pub async fn attachment_import(app: AppHandle, request: tauri::ipc::Request<'_>) -> IpcResult { + let header = request + .headers() + .get("x-opennote-attachment") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| IpcError::invalid("x-opennote-attachment", "The file header is missing."))?; + let header: ImportHeader = serde_json::from_str(header) + .map_err(|_| IpcError::invalid("x-opennote-attachment", "The file header isn't valid JSON."))?; + let bytes = match request.body() { + InvokeBody::Raw(bytes) => bytes.clone(), + InvokeBody::Json(_) => return Err(IpcError::invalid("body", "The file must come as raw bytes.")), + }; + if bytes.len() > MAX_ATTACHMENT_BYTES { + return Err(IpcError::new("tooLarge", "Files over 200 MB can't be attached.")); + } + on_blocking(move || { + let handle = open_page(&app, &header.page)?; + let name = safe_name(&percent_decode(&header.name)); + // The name decides the type, so a file called leaf.png is checked as a picture by the core. + let mime = match header.mime.trim() { + "" | "application/octet-stream" => mime_for_name(&name).to_owned(), + declared => declared.to_owned(), + }; + let asset = handle + .import_asset(AssetSource::bytes(name, mime, bytes)) + .map_err(io_error)?; + Ok(ImportedAsset { + id: asset.id.to_string(), + asset: asset_json(&asset), + }) + }) + .await +} + +/// The client the shell saves an attachment back as, when no window makes the change. Undo is kept per client, so a +/// change the shell makes is not on a window's Ctrl+Z; the earlier asset stays in the page's table, and in its +/// history. +const SAVE_BACK_CLIENT: &str = "attachments"; + +/// How often and how long a watch looks, so tests can run one quickly. +#[derive(Debug, Clone, Copy)] +struct Timing { + /// How often the copy is checked. + poll: Duration, + /// A watch ends after this long, so a forgotten copy does not keep a thread for days. + limit: Duration, + /// How long a window that shows the page has to make a change saved back, before the shell makes it. + window: Duration, +} + +const TIMING: Timing = Timing { + poll: Duration::from_millis(1000), + limit: Duration::from_secs(12 * 60 * 60), + window: Duration::from_secs(5), +}; + +/// What saving back needs from the app: the core's notes and the windows. Tests run it on a bridge alone. +trait Notes: Clone + Send + 'static { + /// Runs `work` on the started bridge. + fn bridge(&self, work: impl FnOnce(&mut Bridge) -> IpcResult) -> IpcResult; + /// Tells the windows that a change came back, so one that shows the page points the attachment at it. + fn tell(&self, saved: &Saved); +} + +#[derive(Clone)] +struct AppNotes(AppHandle); + +impl Notes for AppNotes { + fn bridge(&self, work: impl FnOnce(&mut Bridge) -> IpcResult) -> IpcResult { + let bridge = self.0.state::(); + run_notes(&self.0, &bridge, work) + } + + fn tell(&self, saved: &Saved) { + if let Err(error) = self.0.emit(SAVED_EVENT, saved) { + ::log::warn!("Couldn't send {SAVED_EVENT}: {error}"); + } + } +} + +/// Runs `work` on the page through the shell's own session of it (opened for the work and closed after it). The +/// bridge is held meanwhile, so two watches never share the session. `work` also learns whether a window has the +/// page open. +fn in_shell(notes: &N, page: &str, work: impl FnOnce(&PageHandle, bool) -> IpcResult) -> IpcResult { + notes.bridge(|bridge| { + let window = in_window(bridge, page); + let handle = bridge.handle(page, SAVE_BACK_CLIENT)?; + let result = work(&handle, window); + let closed = bridge.close_handle(page, SAVE_BACK_CLIENT); + let value = result?; + closed.map(|()| value) + }) +} + +/// Whether a window has the page open and can change it. A window that shows it read-only is never asked, so the +/// shell doesn't wait on a change the window won't make. +fn in_window(bridge: &Bridge, page: &str) -> bool { + let Ok(id) = PageId::parse(page) else { + return false; + }; + bridge.open.iter().any(|((open, client), handle)| { + *open == id && client.as_str() != SAVE_BACK_CLIENT && handle.read_only().is_none() + }) +} + +/// Changes saved back that a window made, as (page, asset), until the watch that waits for them takes them. +static APPLIED: Mutex> = Mutex::new(Vec::new()); +static APPLIED_WAKE: Condvar = Condvar::new(); + +fn applied(page: &str, asset: &str) { + let mut list = APPLIED.lock().unwrap_or_else(PoisonError::into_inner); + list.push((page.to_owned(), asset.to_owned())); + // One no watch waited for, as after a timeout, isn't kept for long. + if list.len() > 64 { + list.remove(0); + } + APPLIED_WAKE.notify_all(); +} + +/// How often a wait for a window checks that the window can still make the change. +const WINDOW_CHECK: Duration = Duration::from_millis(100); + +/// Takes the window's word that it pointed the page's attachment at `asset`, if it gave it. +fn take_applied(page: &str, asset: &str) -> bool { + let mut list = APPLIED.lock().unwrap_or_else(PoisonError::into_inner); + let found = list.iter().position(|(p, a)| p == page && a == asset); + found.map(|index| list.remove(index)).is_some() +} + +/// Waits up to `timeout` for a window to say it pointed the page's attachment at `asset`. Gives up early once +/// `window` says no window can make it any more, as when the page closed or turned read-only. +fn wait_applied(page: &str, asset: &str, timeout: Duration, window: impl Fn() -> bool) -> bool { + let deadline = Instant::now() + timeout; + loop { + let list = APPLIED.lock().unwrap_or_else(PoisonError::into_inner); + if list.iter().any(|(p, a)| p == page && a == asset) { + drop(list); + return take_applied(page, asset); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return false; + } + drop(APPLIED_WAKE.wait_timeout(list, left.min(WINDOW_CHECK))); + // Checked without the list held: the check takes the bridge, which a window's call may hold meanwhile. + if !window() { + return take_applied(page, asset); + } + } +} + +/// The window pointed its page's attachment at a change saved back (see [`SAVED_EVENT`]). Until it says so, the +/// shell can't know the change reached the note: the page may have closed, or turned read-only, before it could. +#[tauri::command] +pub fn attachment_applied(page: String, asset: String) -> IpcResult<()> { + PageId::parse(&page).map_err(|_| IpcError::invalid("page", "The page's ID isn't valid."))?; + AssetId::parse(&asset).map_err(|_| IpcError::invalid("asset", "The attachment's ID isn't valid."))?; + applied(&page, &asset); + Ok(()) +} + +/// A copy being watched. A watch outlives the page's view: a change saved after the person moved to another page +/// still comes back into the note. It ends when the app does, or after [`Timing::limit`]. +struct Watch { + page: String, + /// The assets the copy has been, oldest first: the one it was made from, then each change saved back. + lineage: Vec, + stop: Arc, + path: PathBuf, +} + +static WATCHES: Mutex> = Mutex::new(Vec::new()); + +fn temp_root() -> PathBuf { + std::env::temp_dir().join("OpenNote").join("attachments") +} + +pub(super) fn temp_folder(asset: &str) -> PathBuf { + temp_root().join(asset) +} + +/// The file in an asset's folder that names the copy it was saved back from, when that copy lives in the folder of +/// an earlier asset. +const COPY_MARK: &str = "copy-path.txt"; + +/// Where the copy of an asset named `name` is: the copy it was saved back from, if that is still there, so changes +/// made to it after its watch ended are found; otherwise a new one in the asset's own folder. +fn copy_path(asset: &str, name: &str) -> PathBuf { + let marked = std::fs::read_to_string(temp_folder(asset).join(COPY_MARK)) + .ok() + .map(|text| PathBuf::from(text.trim())) + // Only a copy of ours, of the same name: the name decided how it may open. + .filter(|path| { + path.starts_with(temp_root()) && path.file_name().is_some_and(|file| file == name) && path.exists() + }); + marked.unwrap_or_else(|| temp_folder(asset).join(name)) +} + +/// Notes in `asset`'s folder that its copy is the file at `path`. Returns the folder, so the watch can remove it +/// with the copy. +fn mark_copy(asset: &str, path: &Path) -> Option { + let folder = temp_folder(asset); + if path.parent() == Some(folder.as_path()) { + return None; + } + std::fs::create_dir_all(&folder).ok()?; + std::fs::write(folder.join(COPY_MARK), path.to_string_lossy().as_bytes()).ok()?; + Some(folder) +} + +/// Where the hash of the bytes a copy last shared with the note is kept: in a folder beside the copy's, never in +/// it, so the folder shown to the person holds only the copy. Asset folders have no dot in their names. +fn record_path(copy: &Path) -> Option { + let folder = copy.parent()?; + let mut name = folder.file_name()?.to_owned(); + name.push(".synced"); + Some(folder.with_file_name(name).join(copy.file_name()?)) +} + +fn hash(bytes: &[u8]) -> String { + Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Notes that the copy at `copy` holds bytes with this hash, which the note has as an asset. +fn write_record(copy: &Path, hash: &str) { + let Some(record) = record_path(copy) else { + return; + }; + let written = match record.parent() { + Some(folder) => std::fs::create_dir_all(folder).and_then(|()| std::fs::write(&record, hash)), + None => Ok(()), + }; + if let Err(error) = written { + ::log::warn!("Couldn't note what an attachment's copy holds: {error}"); + } +} + +fn read_record(copy: &Path) -> Option { + let text = std::fs::read_to_string(record_path(copy)?).ok()?; + Some(text.trim().to_owned()) +} + +/// Whether the copy holds just what the note had of it when it last came back or was written: removing or rewriting +/// it loses nothing. +fn in_sync(copy: &Path) -> bool { + match (read_record(copy), std::fs::read(copy)) { + (Some(record), Ok(bytes)) => record == hash(&bytes), + _ => false, + } +} + +/// Removes a copy, its record, and their folders once empty. An app that still holds the copy keeps it. +fn remove_copy(copy: &Path) { + if std::fs::remove_file(copy).is_err() { + return; + } + if let Some(record) = record_path(copy) { + let _ = std::fs::remove_file(&record); + if let Some(folder) = record.parent() { + let _ = std::fs::remove_dir(folder); + } + } + if let Some(folder) = copy.parent() { + let _ = std::fs::remove_dir(folder); + } +} + +/// Removes the copies earlier runs left that hold nothing the note lacks, and marks and records of copies that are +/// gone. A copy with changes that never came back stays until its attachment opens again, which saves them back. +fn sweep(root: &Path) { + let Ok(entries) = std::fs::read_dir(root) else { + return; + }; + let mut records = Vec::new(); + for entry in entries.flatten() { + let folder = entry.path(); + if !folder.is_dir() { + continue; + } + if folder.extension().is_some_and(|extension| extension == "synced") { + records.push(folder); + continue; + } + for file in std::fs::read_dir(&folder).into_iter().flatten().flatten() { + let path = file.path(); + if !path.is_file() { + continue; + } + if in_sync(&path) { + remove_copy(&path); + } else if path.file_name().is_some_and(|name| name == COPY_MARK) { + let marked = std::fs::read_to_string(&path).map(|text| PathBuf::from(text.trim())); + if marked.is_ok_and(|marked| marked.starts_with(root) && !marked.exists()) { + let _ = std::fs::remove_file(&path); + } + } + } + let _ = std::fs::remove_dir(&folder); + } + for folder in records { + let copies = folder.with_extension(""); + for file in std::fs::read_dir(&folder).into_iter().flatten().flatten() { + if !copies.join(file.file_name()).exists() { + let _ = std::fs::remove_file(file.path()); + } + } + let _ = std::fs::remove_dir(&folder); + } +} + +/// At start, in the background: clears the copies earlier runs left (see [`sweep`]). +pub fn sweep_old_copies() { + let spawned = thread::Builder::new() + .name("attachment-sweep".into()) + .spawn(|| sweep(&temp_root())); + if let Err(error) = spawned { + ::log::warn!("Couldn't start clearing old attachment copies: {error}"); + } +} + +/// At exit: ends the watches, and removes their copies that hold nothing the note lacks. +pub fn shutdown() { + let watches = std::mem::take(&mut *WATCHES.lock().unwrap_or_else(PoisonError::into_inner)); + for watch in watches { + watch.stop.store(true, Ordering::Relaxed); + if in_sync(&watch.path) { + remove_copy(&watch.path); + } + } +} + +/// The watched copy of a page's asset, if it is still there: the copy a watch made of it or of a later version, or +/// one now at `path`, where a copy of it goes. A second watch never starts on one file. +fn watched_copy(page: &str, asset: &str, path: &Path) -> Option { + let watches = WATCHES.lock().ok()?; + let watch = watches + .iter() + .find(|watch| watch.page == page && (watch.lineage.iter().any(|id| id == asset) || watch.path == path))?; + watch.path.exists().then(|| watch.path.clone()) +} + +/// Ends the watches of a page's asset or of the copy at `path`, such as one whose copy was deleted. +fn stop_watches(page: &str, asset: &str, path: &Path) { + if let Ok(mut watches) = WATCHES.lock() { + watches.retain(|watch| { + let matches = watch.page == page && (watch.lineage.iter().any(|id| id == asset) || watch.path == path); + if matches { + watch.stop.store(true, Ordering::Relaxed); + } + !matches + }); + } +} + +/// What a note found where it puts an attachment's copy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Copy { + /// Nothing, or a copy of a version the note had before, such as the one an undo took back: the copy was written + /// from the asset. + Written, + /// A copy with the asset's bytes, or one the app holding it won't let anyone read or write. It is opened as it + /// is. + Kept, + /// A copy with changes the note doesn't have: saved after its watch ended, such as after the app was closed. + /// They are saved back before the copy opens, never overwritten. + Edited, +} + +/// Makes sure `path` holds a copy of an asset with these bytes, without ever overwriting changes the note lacks. +fn prepare_copy(path: &Path, bytes: &[u8]) -> IpcResult { + let synced = hash(bytes); + if path.exists() { + let Ok(copy) = std::fs::read(path) else { + return Ok(Copy::Kept); + }; + if copy == bytes { + write_record(path, &synced); + return Ok(Copy::Kept); + } + // Without a record, as for a copy from an earlier version of the app, any difference is a change. + if read_record(path) != Some(hash(©)) { + return Ok(Copy::Edited); + } + return Ok(match std::fs::write(path, bytes) { + Ok(()) => { + write_record(path, &synced); + Copy::Written + } + Err(_) => Copy::Kept, + }); + } + if let Some(folder) = path.parent() { + std::fs::create_dir_all(folder).map_err(io_error)?; + } + std::fs::write(path, bytes).map_err(io_error)?; + write_record(path, &synced); + Ok(Copy::Written) +} + +/// Why a change couldn't be saved back. +#[derive(Debug)] +enum Failure { + /// The copy can't be read now, as while its app writes it. The next check tries again. + Retry(IpcError), + /// The page can't take it: it is gone or read-only, or refused the change. The copy keeps the change, and the + /// next open of the attachment tries again. + Stop(IpcError), +} + +fn read_only(reason: impl std::fmt::Debug) -> IpcError { + IpcError::new("readOnly", format!("The page is read-only: {reason:?}")) +} + +/// The file blocks that show any of `assets` except `except`. +fn blocks_showing(handle: &PageHandle, assets: &[AssetId], except: Option) -> Vec { + assets + .iter() + .filter(|asset| Some(**asset) != except) + .flat_map(|asset| handle.file_blocks_of(*asset)) + .collect() +} + +/// Points the file blocks that show any of `lineage` at `asset`, in one transaction of the handle's client. +fn point_at(handle: &PageHandle, lineage: &[AssetId], asset: AssetId) -> IpcResult<()> { + if let Some(reason) = handle.read_only() { + return Err(read_only(reason)); + } + let blocks = blocks_showing(handle, lineage, Some(asset)); + if blocks.is_empty() { + return Ok(()); + } + let mut edits = vec![Edit::AddAsset { asset }]; + for block in blocks { + let mut data = JsonMap::new(); + data.insert("asset".to_owned(), Value::String(asset.to_string())); + edits.push(Edit::PatchBlock { + block, + lock: None, + data: Some(data), + fallback: None, + }); + } + apply_next(handle, edits) +} + +/// Saves the copy's bytes back as an asset of the page, and points the file blocks that show any asset the copy has +/// been (`lineage`, oldest first) at it: after an undo a block may show an earlier one. A window that shows the page +/// makes that change, as one step its Ctrl+Z takes back. When none does, or the window doesn't say it did within +/// [`Timing::window`], the shell makes it. Answers the assets the save made, oldest first, the last being the one +/// the page shows; `None` when no block shows the copy any more. +fn save_back( + notes: &N, + page: &str, + lineage: &[String], + path: &Path, + timing: Timing, +) -> Result>, Failure> { + let ids = lineage + .iter() + .map(|id| AssetId::parse(id)) + .collect::, _>>() + .map_err(|_| Failure::Stop(IpcError::invalid("asset", "The attachment's ID isn't valid.")))?; + let bytes = std::fs::read(path).map_err(|error| Failure::Retry(io_error(error)))?; + if bytes.len() > MAX_ATTACHMENT_BYTES { + return Err(Failure::Stop(IpcError::new( + "tooLarge", + "Files over 200 MB can't be attached.", + ))); + } + let synced = hash(&bytes); + let name = safe_name(&path.file_name().unwrap_or_default().to_string_lossy()); + let mime = mime_for_name(&name).to_owned(); + let source = || AssetSource::bytes(name.clone(), mime.clone(), bytes.clone()); + // An import is kept only while a session of the page is open: with no window showing the page, the shell's own + // session closes after its work, so it makes the change in the same session as the import. + let imported = in_shell(notes, page, |handle, window| { + // Checked before the import, so a page that can't take the change gets no asset it never shows. + if let Some(reason) = handle.read_only() { + return Err(read_only(reason)); + } + if blocks_showing(handle, &ids, None).is_empty() { + return Ok(None); + } + let asset = handle.import_asset(source()).map_err(io_error)?; + let changes = !blocks_showing(handle, &ids, Some(asset.id)).is_empty(); + if changes && !window { + point_at(handle, &ids, asset.id)?; + } + Ok(Some((asset, changes && window))) + }) + .map_err(Failure::Stop)?; + let Some((asset, by_window)) = imported else { + return Ok(None); + }; + if by_window { + notes.tell(&Saved { + page: page.to_owned(), + previous: lineage.last().cloned().unwrap_or_default(), + replaces: lineage.to_vec(), + asset: ImportedAsset { + id: asset.id.to_string(), + asset: asset_json(&asset), + }, + }); + let window = || notes.bridge(|bridge| Ok(in_window(bridge, page))).unwrap_or(false); + if !wait_applied(page, &asset.id.to_string(), timing.window, window) { + // The window closed the page, or couldn't make the change, and the import may have gone with the page's + // session. The shell imports the copy again and makes the change itself. The window may still have made + // it, only late, so blocks that show the first import are pointed at the new one too, and the watch + // keeps both: a block that shows either is still the copy's. + let mut shown = ids.clone(); + shown.push(asset.id); + let again = in_shell(notes, page, |handle, _| { + let again = handle.import_asset(source()).map_err(io_error)?; + point_at(handle, &shown, again.id)?; + Ok(again) + }) + .map_err(Failure::Stop)?; + write_record(path, &synced); + return Ok(Some(vec![asset.id.to_string(), again.id.to_string()])); + } + } + write_record(path, &synced); + Ok(Some(vec![asset.id.to_string()])) +} + +/// Applies edits as the next transaction of the handle's client. The shell's client may have sent some before in +/// this session of the page, so a refused number is replaced by the one the page expects. +fn apply_next(handle: &PageHandle, edits: Vec) -> IpcResult<()> { + let request = |client_seq| TxnRequest { + page: handle.id(), + client: handle.client().clone(), + client_seq, + coalesce: None, + ui: None, + edits: edits.clone(), + }; + let edit_error = |error: EditError| IpcError::new(error.code(), error.to_string()); + match handle.apply(request(1)) { + Ok(_) => Ok(()), + Err(EditError::OutOfOrder { expected }) => handle.apply(request(expected)).map(|_| ()).map_err(edit_error), + Err(error) => Err(edit_error(error)), + } +} + +/// Opens the attached file in its own app, or shows it in its folder when it is not a type a note opens (see +/// [`OPENS`]), and watches the copy. +#[tauri::command] +pub async fn attachment_open(app: AppHandle, page: String, asset: String, name: String) -> IpcResult { + let name = safe_name(&name); + let opening = opening(&name); + if opening == Opening::Blocked { + return Err(IpcError::new( + "blockedType", + "Windows would run this file instead of opening it, so a note won't open it.", + )); + } + on_blocking(move || { + let path = open_copy(&AppNotes(app), &page, &asset, &name, TIMING)?; + mark_from_internet(&path); + if opening == Opening::App { + open_default(&path).map(|()| Shown::App) + } else { + crate::interop::commands::reveal(&path).map(|()| Shown::Folder) + } + }) + .await +} + +/// Puts the attachment's copy where it opens, and makes sure a watch saves its changes back. A copy with changes the +/// note lacks, such as ones saved after the app closed, is saved back first, never overwritten. +fn open_copy(notes: &N, page: &str, asset: &str, name: &str, timing: Timing) -> IpcResult { + let id = AssetId::parse(asset).map_err(|_| IpcError::invalid("asset", "The attachment's ID isn't valid."))?; + let watched = watched_copy(page, asset, ©_path(asset, name)); + let path = watched.clone().unwrap_or_else(|| copy_path(asset, name)); + let bytes = in_shell(notes, page, |handle, _| { + Ok(handle.asset_bytes(id, None).map_err(io_error)?.bytes) + })?; + let copy = prepare_copy(&path, &bytes)?; + if watched.is_some() { + // Its watch saves any change back. + return Ok(path); + } + stop_watches(page, asset, &path); + let mut lineage = vec![asset.to_owned()]; + if copy == Copy::Edited { + match save_back(notes, page, &lineage, &path, timing) { + Ok(Some(made)) => lineage.extend(made.into_iter().filter(|id| id != asset)), + Ok(None) => {} + Err(Failure::Retry(error) | Failure::Stop(error)) => { + ::log::warn!("Couldn't save an attachment's earlier changes back: {}", error.message); + } + } + } + let marks = lineage + .last() + .and_then(|current| mark_copy(current, &path)) + .into_iter() + .collect(); + start_watch(notes.clone(), page.to_owned(), lineage, path.clone(), marks, timing); + Ok(path) +} + +type Stamp = (u64, Option); + +fn stamp(path: &Path) -> Option { + let meta = std::fs::metadata(path).ok()?; + Some((meta.len(), meta.modified().ok())) +} + +/// Watches the copy at `path` of a page's attachment, whose versions so far are `lineage`. `marks` are folders whose +/// [`COPY_MARK`] points at the copy; they go with it. +fn start_watch( + notes: N, + page: String, + mut lineage: Vec, + path: PathBuf, + mut marks: Vec, + timing: Timing, +) { + let stop = Arc::new(AtomicBool::new(false)); + if let Ok(mut watches) = WATCHES.lock() { + watches.push(Watch { + page: page.clone(), + lineage: lineage.clone(), + stop: stop.clone(), + path: path.clone(), + }); + } + // Taken now, not when the thread starts: a change saved meanwhile would otherwise be taken for the copy as written. + let mut seen = stamp(&path); + thread::spawn(move || { + let started = Instant::now(); + let mut pending: Option = None; + while !stop.load(Ordering::Relaxed) && started.elapsed() < timing.limit { + thread::sleep(timing.poll); + if stop.load(Ordering::Relaxed) { + break; + } + let Some(now) = stamp(&path) else { continue }; + if Some(now) == seen { + pending = None; + continue; + } + // A change counts once the file has stayed the same for a whole check, so a half-written file isn't taken. + if pending != Some(now) { + pending = Some(now); + continue; + } + match save_back(¬es, &page, &lineage, &path, timing) { + Ok(Some(made)) => { + seen = Some(now); + pending = None; + for id in made { + marks.extend(mark_copy(&id, &path)); + if !lineage.contains(&id) { + lineage.push(id); + } + } + if let Ok(mut watches) = WATCHES.lock() { + if let Some(watch) = watches.iter_mut().find(|watch| Arc::ptr_eq(&watch.stop, &stop)) { + watch.lineage = lineage.clone(); + } + } + } + Ok(None) => { + ::log::info!("An attachment's copy changed, but no block on its page shows it any more."); + break; + } + // The app may still hold the file; the next check tries again. + Err(Failure::Retry(_)) => pending = None, + // The page can't take the change. The copy keeps it, and the next open tries again. + Err(Failure::Stop(error)) => { + ::log::warn!("Stopped saving an attachment back: {}", error.message); + break; + } + } + } + if let Ok(mut watches) = WATCHES.lock() { + watches.retain(|watch| !Arc::ptr_eq(&watch.stop, &stop)); + } + // A copy that holds what the note has is removed. One with a change that didn't come back is kept, and the + // next open saves it back. + if in_sync(&path) { + remove_copy(&path); + for folder in marks { + let _ = std::fs::remove_file(folder.join(COPY_MARK)); + let _ = std::fs::remove_dir(folder); + } + } + }); +} + +#[cfg(windows)] +fn open_default(file: &Path) -> IpcResult<()> { + use windows::{ + core::{w, HSTRING}, + Win32::UI::{Shell::ShellExecuteW, WindowsAndMessaging::SW_SHOWNORMAL}, + }; + + // SAFETY: both strings outlive the call, and ShellExecuteW takes no other pointers. + let result = unsafe { + ShellExecuteW( + None, + w!("open"), + &HSTRING::from(file.as_os_str()), + None, + None, + SW_SHOWNORMAL, + ) + }; + // ShellExecute reports success with a value above 32. + if (result.0 as isize) > 32 { + Ok(()) + } else { + Err(IpcError::new( + "noApp", + "Windows has no app set up to open this kind of file.", + )) + } +} + +#[cfg(not(windows))] +fn open_default(_file: &Path) -> IpcResult<()> { + Err(IpcError::not_implemented("attachment_open")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_names_lose_folders_and_reserved_characters() { + assert_eq!(safe_name("..\\..\\Budget: 2026?.xlsx"), "Budget 2026.xlsx"); + assert_eq!(safe_name("a/b/report.pdf"), "report.pdf"); + assert_eq!(safe_name(" ... "), "attachment"); + assert_eq!(safe_name(&"x".repeat(300)).chars().count(), 120); + } + + #[test] + fn only_documents_and_media_open_in_their_app() { + for name in [ + "notes.docx", + "sheet.XLSX", + "slides.pptx", + "scan.pdf", + "photo.jpeg", + "voice.m4a", + "clip.mp4", + "archive.zip", + ] { + assert_eq!(opening(name), Opening::App, "{name}"); + } + } + + #[test] + fn files_that_run_something_are_never_opened() { + // The types the review named, which the old list of what may not open had missed, and the usual programs. + for name in [ + "Timesheet.settingcontent-ms", + "help.chm", + "console.MSC", + "fix.diagcab", + "disk.iso", + "disk.img", + "disk.vhd", + "disk.vhdx", + "a.wsc", + "a.sct", + "x.library-ms", + "x.search-ms", + "x.website", + "tool.pyw", + "tool.py", + "a.vb", + "a.ws", + "dark.theme", + "dark.themepack", + "dark.deskthemepack", + "addin.xll", + "x.xbap", + "x.msh", + "x.mshxml", + "x.ps1xml", + "x.psd1", + "setup.EXE", + "run.bat", + "a.ps1", + "shortcut.lnk", + "x.JS", + "report.pdf.lnk", + ] { + assert_eq!(opening(name), Opening::Blocked, "{name}"); + } + } + + #[test] + fn other_types_are_shown_in_their_folder() { + // A type no list names, like one Windows adds next year, is never handed to Windows to open. + for name in [ + "readme", + "drawing.dwg", + "macro.xlsm", + "page.html", + "logo.svg", + "new.unheardof", + ] { + assert_eq!(opening(name), Opening::Folder, "{name}"); + } + } + + #[test] + fn a_copy_with_changes_is_never_overwritten_with_the_older_asset() { + // F3-2: a copy edited after its watch ended was rewritten with the asset's old bytes on the next open. + let dir = tempfile::tempdir().expect("a temp folder"); + let path = dir.path().join("asset").join("Budget.xlsx"); + assert_eq!(prepare_copy(&path, b"old cells").expect("writes"), Copy::Written); + assert_eq!(std::fs::read(&path).expect("reads"), b"old cells"); + assert!(in_sync(&path), "the copy's record names what it holds"); + assert_eq!(prepare_copy(&path, b"old cells").expect("keeps"), Copy::Kept); + std::fs::write(&path, b"new cells").expect("the other app saves"); + assert!(!in_sync(&path)); + assert_eq!(prepare_copy(&path, b"old cells").expect("keeps"), Copy::Edited); + assert_eq!(std::fs::read(&path).expect("reads"), b"new cells"); + } + + #[test] + fn a_copy_of_a_version_the_note_moved_away_from_is_rewritten() { + // F3-2 (G3): after a save-back and an undo, the copy holds the newer version the note already has as an asset. + // That is no change of the person's, so opening the earlier version rewrites the copy rather than saving it + // back over the undo. + let dir = tempfile::tempdir().expect("a temp folder"); + let path = dir.path().join("asset").join("Budget.xlsx"); + prepare_copy(&path, b"v1").expect("writes"); + std::fs::write(&path, b"v2").expect("the other app saves"); + write_record(&path, &hash(b"v2")); + assert_eq!(prepare_copy(&path, b"v1").expect("rewrites"), Copy::Written); + assert_eq!(std::fs::read(&path).expect("reads"), b"v1"); + // A copy from before records existed counts any difference as a change. + let old = dir.path().join("old").join("Notes.docx"); + std::fs::create_dir_all(old.parent().expect("a folder")).expect("a folder"); + std::fs::write(&old, b"edited").expect("writes"); + assert_eq!(prepare_copy(&old, b"original").expect("keeps"), Copy::Edited); + // The record lives beside the copy's folder, never in it. + let record = record_path(&path).expect("a record"); + assert_ne!(record.parent(), path.parent()); + } + + #[test] + fn the_sweep_removes_only_copies_that_hold_nothing_new() { + // F3-2 (G8): copies of every attachment ever opened piled up in the temporary folder. + let dir = tempfile::tempdir().expect("a temp folder"); + let root = dir.path(); + let copy = |folder: &str, name: &str, bytes: &[u8], record: Option<&[u8]>| { + let path = root.join(folder).join(name); + std::fs::create_dir_all(path.parent().expect("a folder")).expect("a folder"); + std::fs::write(&path, bytes).expect("writes"); + if let Some(record) = record { + write_record(&path, &hash(record)); + } + path + }; + let synced = copy("a1", "Budget.xlsx", b"cells", Some(b"cells")); + let edited = copy("a2", "Notes.docx", b"edited", Some(b"original")); + let unknown = copy("a3", "Plan.pptx", b"slides", None); + let mark = copy( + "a4", + COPY_MARK, + root.join("gone").join("x.pdf").to_string_lossy().as_bytes(), + None, + ); + std::fs::create_dir_all(root.join("a5.synced")).expect("a folder"); + std::fs::write(root.join("a5.synced").join("Gone.pdf"), "hash").expect("writes"); + sweep(root); + assert!( + !synced.exists() && !root.join("a1").exists(), + "a copy the note has is removed" + ); + assert!(!record_path(&synced).expect("a record").exists()); + assert!(!root.join("a1.synced").exists()); + assert!( + edited.exists() && record_path(&edited).expect("a record").exists(), + "changes are kept" + ); + assert!(unknown.exists(), "a copy without a record may hold changes"); + assert!( + !mark.exists() && !root.join("a4").exists(), + "a mark of a copy that is gone is removed" + ); + assert!( + !root.join("a5.synced").exists(), + "a record of a copy that is gone is removed" + ); + } + + #[test] + fn a_saved_back_asset_finds_the_copy_it_came_from() { + // After a save-back the attachment shows a newer asset, but its copy stays in the first asset's folder. + // Changes made to that copy after its watch ended must be found when the newer asset opens. + let first = format!("test-{}", std::process::id()); + let later = format!("{first}-later"); + let copy = temp_folder(&first).join("Budget.xlsx"); + std::fs::create_dir_all(temp_folder(&first)).expect("a folder"); + std::fs::write(©, b"edited").expect("writes"); + assert_eq!( + copy_path(&later, "Budget.xlsx"), + temp_folder(&later).join("Budget.xlsx") + ); + let marked = mark_copy(&later, ©).expect("marked"); + assert_eq!(copy_path(&later, "Budget.xlsx"), copy); + // A copy of another name is never opened in place of this one: its name decided how it may open. + assert_eq!(copy_path(&later, "Budget.exe"), temp_folder(&later).join("Budget.exe")); + let _ = std::fs::remove_dir_all(marked); + let _ = std::fs::remove_dir_all(temp_folder(&first)); + } + + /// The window's client in these tests. + const WINDOW: &str = "main-1"; + + const FAST: Timing = Timing { + poll: Duration::from_millis(30), + limit: Duration::from_secs(120), + window: Duration::from_millis(400), + }; + + /// The app for a watch, without windows: a bridge, and a window that shows the page when `WINDOW` has it open. + #[derive(Clone)] + struct TestNotes { + bridge: Arc, + notes: PathBuf, + /// Whether the window makes a change it is told about, as its interface does, and says so. + window_applies: bool, + told: Arc>>, + /// Every use of the bridge fails with this code once set, as for a page that turned read-only. + fail: Arc>>, + calls: Arc, + /// The window makes a change but its word never reaches the shell, as when it comes too late. + silent: Arc, + } + + impl Notes for TestNotes { + fn bridge(&self, work: impl FnOnce(&mut Bridge) -> IpcResult) -> IpcResult { + self.calls.fetch_add(1, Ordering::SeqCst); + if let Some(code) = *self.fail.lock().expect("the flag") { + return Err(IpcError::new(code, "refused")); + } + self.bridge.notes(Some(self.notes.clone()), work) + } + + fn tell(&self, saved: &Saved) { + self.told.lock().expect("the list").push(saved.clone()); + if !self.window_applies { + return; + } + // As the interface does: one transaction of the window's client, then it says so. + let made = self.bridge.notes(Some(self.notes.clone()), |bridge| { + let handle = bridge.handle(&saved.page, WINDOW)?; + let asset = AssetId::parse(&saved.asset.id).map_err(io_error)?; + let lineage = saved + .replaces + .iter() + .map(|id| AssetId::parse(id).map_err(io_error)) + .collect::>>()?; + point_at(&handle, &lineage, asset) + }); + if made.is_ok() && !self.silent.load(Ordering::SeqCst) { + applied(&saved.page, &saved.asset.id); + } + } + } + + struct Setup { + _dir: tempfile::TempDir, + notes: TestNotes, + page: String, + asset: AssetId, + /// The assets the attachment may show: the first, and each one a change came back as. + known: Mutex>, + } + + impl Setup { + /// A page with a file block that shows `Budget.csv`. `window` keeps it open in the window. + fn new(window: bool, window_applies: bool) -> Setup { + let dir = tempfile::tempdir().expect("a temp folder"); + let notes_folder = dir.path().join("Notes"); + let bridge = Arc::new(CoreBridge::at(dir.path().join("local"))); + let (page, asset) = a_page_with_an_attachment(&bridge, ¬es_folder, window); + let notes = TestNotes { + bridge, + notes: notes_folder, + window_applies, + told: Arc::default(), + fail: Arc::default(), + calls: Arc::default(), + silent: Arc::default(), + }; + Setup { + _dir: dir, + notes, + page, + asset, + known: Mutex::new(vec![asset]), + } + } + + /// Runs `work` on the window's session of the page, which stays closed if it was. + fn with_window(&self, work: impl FnOnce(&PageHandle) -> IpcResult) -> T { + self.notes + .bridge + .notes(Some(self.notes.notes.clone()), |bridge| { + let was_open = in_window_of(bridge, &self.page); + let result = work(&bridge.handle(&self.page, WINDOW)?); + if !was_open { + bridge.close_handle(&self.page, WINDOW)?; + } + result + }) + .expect("the page") + } + + /// The asset the file block shows. + fn shown(&self) -> AssetId { + let mut known = self.known.lock().expect("the list"); + let watched: Vec = WATCHES + .lock() + .expect("the watches") + .iter() + .filter(|watch| watch.page == self.page) + .flat_map(|watch| watch.lineage.clone()) + .collect(); + let told: Vec = self + .notes + .told + .lock() + .expect("the list") + .iter() + .map(|saved| saved.asset.id.clone()) + .collect(); + for id in watched.into_iter().chain(told) { + let id = AssetId::parse(&id).expect("an asset"); + if !known.contains(&id) { + known.push(id); + } + } + let known = known.clone(); + self.with_window(|handle| { + Ok(known + .iter() + .copied() + .find(|asset| !handle.file_blocks_of(*asset).is_empty()) + .unwrap_or(self.asset)) + }) + } + + fn bytes(&self, asset: AssetId) -> Vec { + self.with_window(|handle| Ok(handle.asset_bytes(asset, None).map_err(io_error)?.bytes)) + } + + fn open(&self) -> PathBuf { + open_copy(&self.notes, &self.page, &self.asset.to_string(), "Budget.csv", FAST).expect("opens") + } + + fn watches(&self) -> usize { + WATCHES + .lock() + .expect("the watches") + .iter() + .filter(|watch| watch.page == self.page) + .count() + } + + /// Ends the page's watches and waits for their threads to finish. + fn stop(&self, copy: &Path) { + stop_watches(&self.page, &self.asset.to_string(), copy); + thread::sleep(FAST.poll * 4); + } + } + + impl Drop for Setup { + fn drop(&mut self) { + if let Ok(mut watches) = WATCHES.lock() { + watches.retain(|watch| { + let ours = watch.page == self.page; + if ours { + watch.stop.store(true, Ordering::Relaxed); + } + !ours + }); + } + thread::sleep(FAST.poll * 4); + self.notes.bridge.shutdown(); + } + } + + fn in_window_of(bridge: &Bridge, page: &str) -> bool { + bridge + .open + .keys() + .any(|(open, client)| open.to_string() == page && client.as_str() == WINDOW) + } + + fn eventually(what: &str, done: impl Fn() -> bool) { + let deadline = Instant::now() + Duration::from_secs(20); + while !done() { + assert!(Instant::now() < deadline, "timed out waiting until {what}"); + thread::sleep(Duration::from_millis(20)); + } + } + + /// The other app saves the copy: the bytes, then a later modified time, as an app's save does. + fn save_in_other_app(copy: &Path, bytes: &[u8]) { + thread::sleep(Duration::from_millis(20)); + std::fs::write(copy, bytes).expect("the other app saves"); + } + + /// A page with a file block that shows an attachment, in a notebook of `notes`. Returns the page and the asset. + /// Unless `window`, the window's session closes, as when the person moves to another page. + fn a_page_with_an_attachment(bridge: &CoreBridge, notes: &Path, window: bool) -> (String, AssetId) { + bridge + .notes(Some(notes.to_path_buf()), |bridge| { + let create = |bridge: &mut Bridge, kind: &str, parent: Option| { + let input = + serde_json::json!({ "kind": kind, "placement": { "parentId": parent, "beforeId": null } }); + let node = bridge.dispatch("notes_create", &serde_json::json!({ "input": input }))?; + Ok::<_, IpcError>(node["id"].as_str().unwrap_or_default().to_owned()) + }; + let notebook = create(bridge, "notebook", None)?; + let section = create(bridge, "section", Some(notebook))?; + let page = create(bridge, "page", Some(section))?; + let handle = bridge.handle(&page, WINDOW)?; + let asset = handle + .import_asset(AssetSource::bytes( + "Budget.csv".to_owned(), + "text/csv".to_owned(), + b"a,b\n1,2\n".to_vec(), + )) + .map_err(io_error)?; + let block: opennote_core::ops::resolve::NewBlock = serde_json::from_value(serde_json::json!({ + "id": "01k6f00000000000000000b001", "type": "file", "data": { "asset": asset.id.to_string() } + })) + .map_err(io_error)?; + apply_next( + &handle, + vec![ + Edit::AddAsset { asset: asset.id }, + Edit::InsertBlock { + block, + after: None, + before: None, + }, + ], + )?; + if !window { + bridge.close_handle(&page, WINDOW)?; + } + Ok((page, asset.id)) + }) + .expect("a page") + } + + #[test] + fn a_change_saved_after_the_page_closed_reaches_the_note() { + // F3-2: the watch gave up when the page wasn't open in a window, so the change never came back. + let setup = Setup::new(false, false); + let copy = setup.open(); + assert_eq!(std::fs::read(©).expect("the copy"), b"a,b\n1,2\n"); + save_in_other_app(©, b"a,b\n1,3\n"); + eventually("the file block shows the change", || setup.shown() != setup.asset); + assert_eq!(setup.bytes(setup.shown()), b"a,b\n1,3\n"); + assert!( + setup.notes.told.lock().expect("the list").is_empty(), + "no window shows the page" + ); + let shell_open = setup.notes.bridge.notes(Some(setup.notes.notes.clone()), |bridge| { + Ok(bridge + .open + .keys() + .any(|(_, client)| client.as_str() == SAVE_BACK_CLIENT)) + }); + assert!( + !shell_open.expect("the bridge"), + "the shell's own session of the page is closed again" + ); + // A second change follows the asset the first became. + let first = setup.shown(); + save_in_other_app(©, b"a,b\n1,4\n"); + // Until the watch learns the new asset, `shown` can't name it and answers the first one. + eventually("the second change comes back", || { + ![first, setup.asset].contains(&setup.shown()) + }); + assert_eq!(setup.bytes(setup.shown()), b"a,b\n1,4\n"); + // The watch ends; its copy holds what the note has, so it goes (G8). + assert!(in_sync(©)); + setup.stop(©); + eventually("the copy is removed", || !copy.exists()); + assert!(!record_path(©).expect("a record").exists()); + } + + #[test] + fn a_window_that_shows_the_page_makes_the_change_as_one_undo_step() { + let setup = Setup::new(true, true); + let copy = setup.open(); + save_in_other_app(©, b"a,b\n9,9\n"); + eventually("the file block shows the change", || setup.shown() != setup.asset); + let told = setup.notes.told.lock().expect("the list").clone(); + assert_eq!(told.len(), 1); + assert_eq!(told[0].replaces, vec![setup.asset.to_string()]); + // The window's Ctrl+Z takes the change back (G7). + setup.with_window(|handle| { + handle + .undo(handle.client()) + .map_err(|error| IpcError::new(error.code(), error.to_string())) + }); + assert_eq!(setup.shown(), setup.asset); + } + + #[test] + fn a_change_the_window_never_makes_is_made_by_the_shell() { + // F3-2 (G4): the window was told, but it closed the page (or the page turned read-only) before it made the + // change. The watch moved on anyway and later deleted the copy, so the change was left only as an asset no + // block showed. + let setup = Setup::new(true, false); + let copy = setup.open(); + save_in_other_app(©, b"a,b\n7,7\n"); + eventually("the shell makes the change", || setup.shown() != setup.asset); + assert_eq!(setup.notes.told.lock().expect("the list").len(), 1); + assert_eq!(setup.bytes(setup.shown()), b"a,b\n7,7\n"); + } + + #[test] + fn a_change_the_window_made_too_late_to_say_so_keeps_coming_back() { + // F3-2: the window made the change but its word came after the wait, so the shell made it again. The block + // shows the window's asset, which the shell must point at its own, and later changes must still come back. + let setup = Setup::new(true, true); + setup.notes.silent.store(true, Ordering::SeqCst); + let copy = setup.open(); + save_in_other_app(©, b"a,b\n3,3\n"); + eventually("the window makes the change", || setup.shown() != setup.asset); + // The shell waits out the window, then makes the change itself. + thread::sleep(FAST.window * 3); + let shown = setup.shown(); + assert_eq!( + setup.bytes(shown), + b"a,b +3,3 +" + ); + let lineage: Vec = WATCHES + .lock() + .expect("the watches") + .iter() + .filter(|watch| watch.page == setup.page) + .flat_map(|watch| watch.lineage.clone()) + .collect(); + assert!( + lineage.contains(&shown.to_string()), + "the watch knows what the block shows" + ); + setup.notes.silent.store(false, Ordering::SeqCst); + save_in_other_app(©, b"a,b\n4,4\n"); + eventually("the next change comes back", || { + setup.bytes(setup.shown()) == b"a,b\n4,4\n" + }); + assert_eq!(setup.watches(), 1, "the watch goes on"); + } + + #[test] + fn a_wait_for_the_window_ends_once_no_window_can_make_the_change() { + let started = Instant::now(); + assert!(!wait_applied("page", "asset", Duration::from_secs(30), || false)); + assert!(started.elapsed() < Duration::from_secs(5)); + applied("page", "said"); + assert!(wait_applied("page", "said", Duration::from_secs(30), || false)); + } + + #[test] + fn an_undone_save_back_stays_undone_after_a_restart() { + // F3-2 (G3): after a save-back (A1 to A2), Ctrl+Z (back to A1), and a restart, opening the attachment found the + // copy differed from A1, took it for an edit, and pointed the block at A2's bytes again. + let setup = Setup::new(true, true); + let copy = setup.open(); + save_in_other_app(©, b"a,b\n5,5\n"); + eventually("the change comes back", || setup.shown() != setup.asset); + setup.with_window(|handle| { + handle + .undo(handle.client()) + .map_err(|error| IpcError::new(error.code(), error.to_string())) + }); + assert_eq!(setup.shown(), setup.asset); + // The app ends without clearing the copy, as when the other app still holds it. + let held = std::fs::read(©).expect("the copy"); + let record = std::fs::read(record_path(©).expect("a record")).expect("the record"); + setup.stop(©); + std::fs::create_dir_all(copy.parent().expect("a folder")).expect("a folder"); + std::fs::write(©, &held).expect("the copy stays"); + write_record(©, &String::from_utf8(record).expect("text")); + let opened = setup.open(); + assert_eq!(opened, copy); + assert_eq!(setup.shown(), setup.asset, "the undo stands"); + assert_eq!( + std::fs::read(©).expect("the copy"), + b"a,b\n1,2\n", + "the copy shows what the note does" + ); + } + + #[test] + fn a_copy_edited_while_no_watch_ran_is_saved_back_when_it_opens() { + let setup = Setup::new(false, false); + let copy = copy_path(&setup.asset.to_string(), "Budget.csv"); + std::fs::create_dir_all(copy.parent().expect("a folder")).expect("a folder"); + std::fs::write(©, b"a,b\n3,3\n").expect("edited after the app closed"); + assert_eq!(setup.open(), copy); + assert_ne!(setup.shown(), setup.asset, "the change came back as the copy opened"); + assert_eq!(setup.bytes(setup.shown()), b"a,b\n3,3\n"); + assert_eq!( + std::fs::read(©).expect("the copy"), + b"a,b\n3,3\n", + "the copy was not overwritten" + ); + setup.stop(©); + } + + #[test] + fn opening_a_watched_copy_again_keeps_its_one_watch() { + let setup = Setup::new(false, false); + let copy = setup.open(); + assert_eq!(setup.watches(), 1); + assert_eq!(setup.open(), copy); + assert_eq!(setup.watches(), 1, "a second watch never starts on one file"); + stop_watches(&setup.page, &setup.asset.to_string(), ©); + assert_eq!(setup.watches(), 0); + eventually("a copy without changes goes with its watch", || !copy.exists()); + } + + #[test] + fn a_page_that_cannot_take_the_change_ends_the_watch() { + // F3-2 (G5): on a read-only page, every check imported the copy again and failed, every 2 seconds for 12 hours. + let setup = Setup::new(false, false); + let copy = setup.open(); + *setup.notes.fail.lock().expect("the flag") = Some("readOnly"); + let before = setup.notes.calls.load(Ordering::SeqCst); + save_in_other_app(©, b"a,b\n6,6\n"); + eventually("the watch ends", || setup.watches() == 0); + thread::sleep(FAST.poll * 10); + assert_eq!( + setup.notes.calls.load(Ordering::SeqCst) - before, + 1, + "one try, then the watch ends" + ); + assert!( + copy.exists() && !in_sync(©), + "the copy keeps the change for the next open" + ); + } + + #[cfg(windows)] + #[test] + fn a_copy_is_marked_as_coming_from_the_internet() { + let dir = tempfile::tempdir().expect("a temp folder"); + let path = dir.path().join("Budget.xlsx"); + std::fs::write(&path, b"cells").expect("writes"); + mark_from_internet(&path); + let mut stream = path.as_os_str().to_owned(); + stream.push(":Zone.Identifier"); + let mark = std::fs::read_to_string(PathBuf::from(stream)).expect("the mark"); + assert!(mark.contains("ZoneId=3"), "{mark}"); + assert_eq!(std::fs::read(&path).expect("reads"), b"cells"); + } +} diff --git a/app/src-tauri/src/page_extras/caret.rs b/app/src-tauri/src/page_extras/caret.rs new file mode 100644 index 00000000..f6f6cf57 --- /dev/null +++ b/app/src-tauri/src/page_extras/caret.rs @@ -0,0 +1,120 @@ +//! The Windows text cursor settings (Settings, then Accessibility, then Text cursor): its width and how fast it +//! blinks. The interface draws a thicker or steady caret from these, so a caret set up in Windows looks the same +//! in a note. + +use serde::Serialize; + +use crate::ipc::IpcResult; + +/// The widest caret Windows offers. +const MAX_WIDTH: u32 = 20; +/// The blink time Windows reports when the caret does not blink. +const NO_BLINK: u32 = u32::MAX; +/// The blink time when Windows can't say: its default. +const DEFAULT_BLINK_MS: u32 = 530; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CaretMetrics { + /// The caret's width in device pixels, from 1 to 20. + pub width_px: u32, + /// How long the caret shows before it hides, or `None` when it does not blink. + pub blink_ms: Option, + /// The Windows text cursor indicator color as `#rrggbb`, or `None` when none is set. + pub color: Option, +} + +/// A Windows color value (`0x00BBGGRR`) as `#rrggbb`. +pub fn hex_from_colorref(value: u32) -> String { + let (r, g, b) = (value & 0xff, (value >> 8) & 0xff, (value >> 16) & 0xff); + format!("#{r:02x}{g:02x}{b:02x}") +} + +/// Metrics from the raw values Windows reports. +pub fn metrics_from(width: u32, blink: u32, color: Option) -> CaretMetrics { + CaretMetrics { + color: color.map(hex_from_colorref), + width_px: width.clamp(1, MAX_WIDTH), + blink_ms: match blink { + NO_BLINK => None, + 0 => Some(DEFAULT_BLINK_MS), + time => Some(time), + }, + } +} + +#[cfg(windows)] +fn read() -> CaretMetrics { + use windows::Win32::UI::WindowsAndMessaging::{ + GetCaretBlinkTime, SystemParametersInfoW, SPI_GETCARETWIDTH, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS, + }; + + let mut width: u32 = 1; + // SAFETY: `width` outlives the call, and SPI_GETCARETWIDTH writes one u32 through the pointer. + let read = unsafe { + SystemParametersInfoW( + SPI_GETCARETWIDTH, + 0, + Some(std::ptr::addr_of_mut!(width).cast()), + SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0), + ) + }; + if read.is_err() { + width = 1; + } + // SAFETY: the call takes no pointers. + let blink = unsafe { GetCaretBlinkTime() }; + metrics_from(width, blink, indicator_color()) +} + +/// The color of the text cursor indicator (Settings, then Accessibility, then Text cursor), when it is turned on. +/// Windows keeps it among the accessibility values of the current user; where it isn't there, the caret keeps the +/// text color. +#[cfg(windows)] +fn indicator_color() -> Option { + use crate::appearance::registry_dword; + + registry_dword(r"Software\Microsoft\Accessibility\CursorIndicator", "IndicatorColor") + .or_else(|| registry_dword(r"Software\Microsoft\Accessibility", "CursorIndicatorColor")) +} + +#[cfg(not(windows))] +fn read() -> CaretMetrics { + metrics_from(1, DEFAULT_BLINK_MS, None) +} + +#[tauri::command] +pub async fn page_extras_caret() -> IpcResult { + Ok(read()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_width_stays_in_windows_range() { + assert_eq!(metrics_from(0, 530, None).width_px, 1); + assert_eq!(metrics_from(6, 530, None).width_px, 6); + assert_eq!(metrics_from(99, 530, None).width_px, 20); + } + + #[test] + fn a_caret_that_never_blinks_has_no_time() { + assert_eq!(metrics_from(1, NO_BLINK, None).blink_ms, None); + assert_eq!(metrics_from(1, 400, None).blink_ms, Some(400)); + assert_eq!(metrics_from(1, 0, None).blink_ms, Some(DEFAULT_BLINK_MS)); + } + + #[test] + fn the_indicator_color_reads_as_hex() { + // Windows keeps colors as 0x00BBGGRR, so red is the low byte. + assert_eq!(hex_from_colorref(0x0000_00ff), "#ff0000"); + assert_eq!(hex_from_colorref(0x00ff_8000), "#0080ff"); + assert_eq!( + metrics_from(1, 530, Some(0x0000_ff00)).color.as_deref(), + Some("#00ff00") + ); + assert_eq!(metrics_from(1, 530, None).color, None); + } +} diff --git a/app/src-tauri/src/page_extras/mod.rs b/app/src-tauri/src/page_extras/mod.rs new file mode 100644 index 00000000..2f7802bf --- /dev/null +++ b/app/src-tauri/src/page_extras/mod.rs @@ -0,0 +1,8 @@ +//! Native help for the quality-of-life features of typed notes: the Windows text cursor settings, link titles on +//! paste, and attached files that open in their own app and save back. Each command names a page by its ID and +//! never a path, and nothing here reaches the network unless the person turned the feature on. + +pub mod attach; +pub mod caret; +pub mod thumbnail; +pub mod title; diff --git a/app/src-tauri/src/page_extras/thumbnail.rs b/app/src-tauri/src/page_extras/thumbnail.rs new file mode 100644 index 00000000..4442ad0d --- /dev/null +++ b/app/src-tauri/src/page_extras/thumbnail.rs @@ -0,0 +1,188 @@ +//! Thumbnails of attached PDF and Office files. Windows already draws a picture of the first page for these files +//! in File Explorer, through the thumbnail handler of the app that opens them. OpenNote asks for that same picture, +//! so a card shows the document's first page rather than a plain icon. Where Windows has no handler, the card keeps +//! its icon. Only document types are asked; a program is never handed to the shell. Nothing leaves the PC. + +use std::path::Path; + +use opennote_core::AssetId; +use tauri::{ipc::Response, AppHandle}; + +use super::attach::{safe_name, temp_folder}; +use crate::{ + images::import::{on_blocking, open_page}, + ipc::{codes, IpcError, IpcResult}, +}; + +/// The document types whose first page is drawn. Everything else keeps its icon. +const DOCUMENT_EXTENSIONS: &[&str] = &[ + "pdf", "doc", "docx", "docm", "dot", "dotx", "rtf", "odt", "xls", "xlsx", "xlsm", "ods", "ppt", "pptx", "pptm", + "odp", +]; + +/// The smallest and largest thumbnail, in device pixels along the longer side. +const MIN_SIZE: u32 = 64; +const MAX_SIZE: u32 = 512; + +/// Whether a file of this name is a document that gets a thumbnail. +pub fn is_document(name: &str) -> bool { + name.rsplit_once('.') + .is_some_and(|(_, extension)| DOCUMENT_EXTENSIONS.contains(&extension.to_ascii_lowercase().as_str())) +} + +/// The size asked for, kept in range. +pub fn clamp_size(size: u32) -> u32 { + size.clamp(MIN_SIZE, MAX_SIZE) +} + +fn io_error(error: impl std::fmt::Display) -> IpcError { + IpcError::new(codes::IO, error.to_string()) +} + +/// A PNG of the first page of an attached document, or `noThumbnail` when Windows can't draw one. +#[tauri::command] +pub async fn attachment_thumbnail( + app: AppHandle, + page: String, + asset: String, + name: String, + size: u32, +) -> IpcResult { + let name = safe_name(&name); + if !is_document(&name) { + return Err(IpcError::new("noThumbnail", "Only documents get a thumbnail.")); + } + let size = clamp_size(size); + let png = on_blocking(move || { + let handle = open_page(&app, &page)?; + let id = AssetId::parse(&asset).map_err(|_| IpcError::invalid("asset", "The attachment's ID isn't valid."))?; + let bytes = handle.asset_bytes(id, None).map_err(io_error)?.bytes; + let folder = temp_folder(&format!("thumbnail-{asset}")); + std::fs::create_dir_all(&folder).map_err(io_error)?; + let path = folder.join(&name); + let drawn = std::fs::write(&path, bytes) + .map_err(io_error) + .and_then(|()| draw(&path, size)); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_dir(&folder); + drawn + }) + .await?; + Ok(Response::new(png)) +} + +#[cfg(not(windows))] +fn draw(_file: &Path, _size: u32) -> IpcResult> { + Err(IpcError::not_implemented("attachment_thumbnail")) +} + +/// Asks the Windows shell for the file's thumbnail (never its icon) and encodes it as a PNG. +#[cfg(windows)] +fn draw(file: &Path, size: u32) -> IpcResult> { + use windows::{ + core::HSTRING, + Win32::{ + Foundation::SIZE, + Graphics::Gdi::{ + CreateCompatibleDC, DeleteDC, DeleteObject, GetDIBits, GetObjectW, BITMAP, BITMAPINFO, + BITMAPINFOHEADER, BI_RGB, DIB_RGB_COLORS, + }, + System::Com::{CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED}, + UI::Shell::{ + IShellItemImageFactory, SHCreateItemFromParsingName, SIIGBF_BIGGERSIZEOK, SIIGBF_THUMBNAILONLY, + }, + }, + }; + + // SAFETY: COM starts and stops around the work. The bitmap Windows hands back is read once and deleted, and + // `pixels` has exactly the width * height * 4 bytes GetDIBits is told about. + let drawn = unsafe { + let apartment = CoInitializeEx(None, COINIT_APARTMENTTHREADED); + let result = (|| -> Result<(Vec, u32, u32), ()> { + let factory: IShellItemImageFactory = + SHCreateItemFromParsingName(&HSTRING::from(file.as_os_str()), None).map_err(|_| ())?; + let wanted = SIZE { + cx: size as i32, + cy: size as i32, + }; + let bitmap = factory + .GetImage(wanted, SIIGBF_THUMBNAILONLY | SIIGBF_BIGGERSIZEOK) + .map_err(|_| ())?; + let mut header = BITMAP::default(); + if GetObjectW( + bitmap.into(), + std::mem::size_of::() as i32, + Some((&raw mut header).cast()), + ) == 0 + { + let _ = DeleteObject(bitmap.into()); + return Err(()); + } + let (width, height) = (header.bmWidth.max(1) as u32, header.bmHeight.max(1) as u32); + let mut info = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: std::mem::size_of::() as u32, + biWidth: width as i32, + biHeight: -(height as i32), + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + ..Default::default() + }, + ..Default::default() + }; + let mut pixels = vec![0u8; width as usize * height as usize * 4]; + let context = CreateCompatibleDC(None); + let lines = GetDIBits( + context, + bitmap, + 0, + height, + Some(pixels.as_mut_ptr().cast()), + &mut info, + DIB_RGB_COLORS, + ); + let _ = DeleteDC(context); + let _ = DeleteObject(bitmap.into()); + if lines == 0 { + return Err(()); + } + Ok((pixels, width, height)) + })(); + if apartment.is_ok() { + CoUninitialize(); + } + result + }; + let (mut pixels, width, height) = + drawn.map_err(|()| IpcError::new("noThumbnail", "Windows has no picture of this file."))?; + // A thumbnail without transparency comes back with its alpha bytes unset; show those pixels solid. + if pixels.as_chunks::<4>().0.iter().all(|pixel| pixel[3] == 0) { + for pixel in pixels.as_chunks_mut::<4>().0 { + pixel[3] = 255; + } + } + crate::audio_more::snap::encode_png(&pixels, width, height).map_err(io_error) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_documents_get_a_thumbnail() { + for name in ["scan.PDF", "plan.docx", "budget.xlsx", "talk.pptx", "a.b.odt"] { + assert!(is_document(name), "{name}"); + } + for name in ["setup.exe", "run.bat", "photo.png", "notes.txt", "readme", "x.lnk"] { + assert!(!is_document(name), "{name}"); + } + } + + #[test] + fn the_size_stays_in_range() { + assert_eq!(clamp_size(0), MIN_SIZE); + assert_eq!(clamp_size(200), 200); + assert_eq!(clamp_size(9999), MAX_SIZE); + } +} diff --git a/app/src-tauri/src/page_extras/title.rs b/app/src-tauri/src/page_extras/title.rs new file mode 100644 index 00000000..feda30ab --- /dev/null +++ b/app/src-tauri/src/page_extras/title.rs @@ -0,0 +1,151 @@ +//! Link titles on paste: the page title of a pasted web address, fetched only when the person turned the feature +//! on. The request is a plain GET for the address, with no cookies and no `Referer`, so the site sees this PC's +//! network address and the request, and nothing else. It never runs while Work offline is on, and it never reaches +//! an address on the local network (the same guard as saving a web image). + +use crate::{ + images::{import::on_blocking, web::fetch_html_head}, + ipc::IpcResult, +}; + +/// The longest title shown as link text. +const MAX_TITLE_CHARS: usize = 200; + +/// The text between the first `` and ``, case-insensitively. +fn element_text<'a>(html: &'a str, lower: &str, tag: &str) -> Option<&'a str> { + let open = lower.find(&format!("<{tag}"))?; + let start = open + lower[open..].find('>')? + 1; + let end = start + lower[start..].find(&format!("`. +fn open_graph_title(html: &str, lower: &str) -> Option { + let mut from = 0; + while let Some(found) = lower[from..].find("')?; + let tag = &html[start..end]; + let tag_lower = &lower[start..end]; + if tag_lower.contains("og:title") { + let at = tag_lower.find("content=")? + "content=".len(); + let rest = &tag[at..]; + let quote = rest.chars().next().filter(|c| *c == '"' || *c == '\'')?; + let value = &rest[1..]; + return value.split(quote).next().map(str::to_owned); + } + from = end; + } + None +} + +fn decode_entities(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(at) = rest.find('&') { + out.push_str(&rest[..at]); + rest = &rest[at..]; + let Some(end) = rest.find(';').filter(|end| *end <= 10) else { + out.push('&'); + rest = &rest[1..]; + continue; + }; + let name = &rest[1..end]; + let decoded = match name { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" => Some('\''), + "nbsp" => Some(' '), + _ => name + .strip_prefix('#') + .and_then(|number| match number.strip_prefix(['x', 'X']) { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => number.parse().ok(), + }) + .and_then(char::from_u32), + }; + match decoded { + Some(c) => { + out.push(c); + rest = &rest[end + 1..]; + } + None => { + out.push('&'); + rest = &rest[1..]; + } + } + } + out.push_str(rest); + out +} + +/// The title of an HTML page: its ``, else its Open Graph title. Whitespace is collapsed, and a title +/// that is empty or has control characters is no title at all. +pub fn title_from_html(bytes: &[u8]) -> Option<String> { + let html = String::from_utf8_lossy(bytes); + // Lowercasing can change byte lengths for a few letters, so work only where it keeps them. + let lower = if html.to_lowercase().len() == html.len() { + html.to_lowercase() + } else { + html.to_ascii_lowercase() + }; + let raw = element_text(&html, &lower, "title") + .map(str::to_owned) + .or_else(|| open_graph_title(&html, &lower))?; + let decoded = decode_entities(&raw); + let title = decoded.split_whitespace().collect::<Vec<_>>().join(" "); + if title.is_empty() || title.chars().any(char::is_control) { + return None; + } + Some(title.chars().take(MAX_TITLE_CHARS).collect()) +} + +#[tauri::command] +pub async fn page_extras_link_title(url: String) -> IpcResult<Option<String>> { + // Work offline blocks every network use, including this one (docs/FEATURES.md, Privacy panel). + if crate::hardening::offline() { + return Ok(None); + } + on_blocking(move || { + let head = fetch_html_head(&url, false)?; + Ok(title_from_html(&head)) + }) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_the_title_and_cleans_it() { + let html = b"<html><head><TITLE lang=en>\n Mitosis & meiosis – Biology\n"; + assert_eq!( + title_from_html(html).as_deref(), + Some("Mitosis & meiosis \u{2013} Biology") + ); + } + + #[test] + fn falls_back_to_the_open_graph_title() { + let html = br#""#; + assert_eq!(title_from_html(html).as_deref(), Some("A \"quoted\" page")); + } + + #[test] + fn no_title_is_none() { + assert_eq!(title_from_html(b"hi"), None); + assert_eq!(title_from_html(b" "), None); + } + + #[test] + fn a_long_title_is_cut() { + let html = format!("{}", "a".repeat(500)); + assert_eq!( + title_from_html(html.as_bytes()).unwrap().chars().count(), + MAX_TITLE_CHARS + ); + } +} diff --git a/app/src-tauri/src/paths.rs b/app/src-tauri/src/paths.rs index 003a0474..d14a0b67 100644 --- a/app/src-tauri/src/paths.rs +++ b/app/src-tauri/src/paths.rs @@ -14,6 +14,27 @@ pub const PROFILE_DIR_VAR: &str = "OPENNOTE_PROFILE_DIR"; const APP_FOLDER: &str = "OpenNote"; +/// The file whose presence next to the program turns on portable mode. +pub const PORTABLE_FILE: &str = "portable"; + +/// The folder beside the program that holds everything in portable mode. +pub const PORTABLE_FOLDER: &str = "OpenNote-data"; + +/// The profile folder portable mode uses: `OpenNote-data` beside the program, when a file named `portable` is +/// there too. Settings, caches, models, and the Documents default all live inside it, so nothing goes to the +/// Windows profile. The notes folder the person picks is theirs and stays wherever they chose. +pub fn portable_profile(program_dir: &Path) -> Option { + program_dir + .join(PORTABLE_FILE) + .is_file() + .then(|| program_dir.join(PORTABLE_FOLDER)) +} + +/// The profile override: the environment variable wins (tests and E2E runs), then portable mode. +pub fn profile_override(env: Option, program_dir: Option<&Path>) -> Option { + env.or_else(|| program_dir.and_then(portable_profile)) +} + /// Whether `path` is a full path to a place on this PC. A network path (`\\host\share`) and the device paths /// (`\\?\`, `\\.\`) aren't: touching one makes Windows sign in to that host, and the app writes only /// where the person's own folders are. @@ -77,7 +98,7 @@ impl Paths { let root = self.roaming.to_string_lossy(); // Windows paths ignore case, so C:\Users and c:\users name the same profile. let root = if cfg!(windows) { - root.to_lowercase() + root.to_lowercase().replace('/', "\\").trim_end_matches('\\').to_owned() } else { root.into_owned() }; @@ -200,6 +221,22 @@ mod tests { } } + #[test] + fn a_portable_file_moves_the_profile_beside_the_program() { + let dir = tempfile::tempdir().expect("a temp folder"); + assert_eq!(portable_profile(dir.path()), None); + std::fs::write(dir.path().join(PORTABLE_FILE), b"").expect("the marker file"); + let profile = portable_profile(dir.path()).expect("portable mode is on"); + assert_eq!(profile, dir.path().join(PORTABLE_FOLDER)); + let paths = Paths::resolve(Some(profile.clone())).expect("resolves"); + assert!(paths.settings_file.starts_with(&profile) && paths.webview.starts_with(&profile)); + // The environment variable still wins, so tests can run from a portable folder. + let env = PathBuf::from(r"C:\profile"); + assert_eq!(profile_override(Some(env.clone()), Some(dir.path())), Some(env)); + assert_eq!(profile_override(None, Some(dir.path())), Some(profile)); + assert_eq!(profile_override(None, None), None); + } + #[test] fn names_each_profile_with_a_short_stable_key() { let one = Paths::under_profile(Path::new("C:\\one")); @@ -209,4 +246,15 @@ mod tests { assert_eq!(key, Paths::under_profile(Path::new("C:\\one")).profile_key()); assert_ne!(key, Paths::under_profile(Path::new("C:\\two")).profile_key()); } + + #[cfg(windows)] + #[test] + fn spells_one_windows_profile_the_same_with_either_slash() { + let back = Paths::under_profile(Path::new("C:\\one\\profile")).profile_key(); + assert_eq!(back, Paths::under_profile(Path::new("C:/one/profile")).profile_key()); + assert_eq!( + back, + Paths::under_profile(Path::new("c:\\ONE\\profile\\")).profile_key() + ); + } } diff --git a/app/src-tauri/src/platform_flags.rs b/app/src-tauri/src/platform_flags.rs new file mode 100644 index 00000000..9d430993 --- /dev/null +++ b/app/src-tauri/src/platform_flags.rs @@ -0,0 +1,122 @@ +//! The flags the shell itself reads, for the install, updater, local API, share target, and phone scan features. +//! Each one has the channel default the interface's `FLAGS` registry gives it (`app/src/app/flags.ts` and +//! `features/integrations/flags.ts`), and development and nightly builds take overrides in the same order as the +//! interface's `initFlags`: the `OPENNOTE_FLAGS` variable, then `settings.experimental.flags`. A test reads the +//! two TypeScript files, so the defaults here can't drift from the interface's. + +use std::collections::BTreeMap; + +use crate::boot::{self, Channel}; + +/// A per-user entry in Installed apps, with `OpenNote.exe --uninstall`. +pub const UNINSTALL_ENTRY: &str = "install.uninstallEntry"; +/// Resume a download that stopped partway. +pub const UPDATES_RESUME: &str = "updates.resume"; +/// Wait for an unmetered network before an automatic download. +pub const UPDATES_METERED: &str = "updates.meteredCheck"; +/// The local API, its app permissions, the `opennote` tool, and the MCP server. Off means no listener. +pub const API_LOCAL: &str = "api.local"; +/// "Share to OpenNote" from other apps. +pub const SHARE_TARGET: &str = "integrations.shareTarget"; +/// Insert > From phone. +pub const PHONE_SCAN: &str = "integrations.phoneScan"; + +/// Every flag above. Each is on in development, nightly, and beta builds, and off in Stable. +pub const BETA_BUILD_FLAGS: [&str; 6] = [ + UNINSTALL_ENTRY, + UPDATES_RESUME, + UPDATES_METERED, + API_LOCAL, + SHARE_TARGET, + PHONE_SCAN, +]; + +/// The channel of this exe, as the boot payload reports it. +pub fn channel() -> Channel { + boot::channel_for( + env!("CARGO_PKG_VERSION"), + cfg!(debug_assertions) || cfg!(feature = "test-endpoints"), + ) +} + +/// Whether `id` is on in `channel`, with `overrides` applied in order, the last one that names the flag winning. +/// Overrides apply only to development and nightly builds. A flag this module doesn't know is off. +pub fn is_on(channel: Channel, id: &str, overrides: &[&BTreeMap]) -> bool { + let default = BETA_BUILD_FLAGS.contains(&id) && !matches!(channel, Channel::Stable); + if !matches!(channel, Channel::Dev | Channel::Nightly) { + return default; + } + overrides + .iter() + .rev() + .find_map(|overrides| overrides.get(id).copied()) + .unwrap_or(default) +} + +/// Whether `id` is on for this exe, with the `OPENNOTE_FLAGS` variable and the settings' experimental flags. +pub fn is_on_now(id: &str, settings_flags: &BTreeMap) -> bool { + let env = boot::flag_overrides(&std::env::var("OPENNOTE_FLAGS").unwrap_or_default()); + is_on(channel(), id, &[&env, settings_flags]) +} + +/// Whether `id` is on for the running app, reading its settings. +pub fn is_on_for(app: &tauri::AppHandle, id: &str) -> bool { + use tauri::Manager; + let flags = app + .try_state::() + .map(|store| store.get().experimental.flags) + .unwrap_or_default(); + is_on_now(id, &flags) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn overrides(id: &str, on: bool) -> BTreeMap { + BTreeMap::from([(id.to_owned(), on)]) + } + + #[test] + fn every_flag_is_on_in_beta_and_off_in_stable() { + for id in BETA_BUILD_FLAGS { + assert!(is_on(Channel::Beta, id, &[]), "{id}"); + assert!(is_on(Channel::Nightly, id, &[]), "{id}"); + assert!(is_on(Channel::Dev, id, &[]), "{id}"); + assert!(!is_on(Channel::Stable, id, &[]), "{id}"); + } + assert!(!is_on(Channel::Dev, "something.else", &[])); + } + + #[test] + fn overrides_count_only_in_development_and_nightly_builds() { + let off = overrides(API_LOCAL, false); + let on = overrides(API_LOCAL, true); + assert!(!is_on(Channel::Dev, API_LOCAL, &[&off])); + assert!(is_on(Channel::Nightly, API_LOCAL, &[&off, &on])); + assert!(is_on(Channel::Beta, API_LOCAL, &[&off])); + assert!(!is_on(Channel::Stable, API_LOCAL, &[&on])); + } + + /// The interface's registry gives each of these flags the same default, `betaBuilds`. + #[test] + fn the_defaults_match_the_interfaces_registry() { + let registry = [ + include_str!("../../src/app/flags.ts"), + include_str!("../../src/features/integrations/flags.ts"), + ] + .concat(); + for id in BETA_BUILD_FLAGS { + let line = registry + .lines() + .skip_while(|line| !line.contains(&format!("flag('{id}'")) && !line.contains(&format!("'{id}',"))) + .take(4) + .collect::>() + .join(" "); + assert!( + line.contains("betaBuilds"), + "{id} isn't betaBuilds in the interface: {line}" + ); + } + } +} diff --git a/app/src-tauri/src/print_job.rs b/app/src-tauri/src/print_job.rs new file mode 100644 index 00000000..c3b9710d --- /dev/null +++ b/app/src-tauri/src/print_job.rs @@ -0,0 +1,51 @@ +//! Real printing (File > Print): the printer list, and one job that sends a PDF the interface made to a printer. The +//! PDF comes from `print_render` (page_export.rs), so the range, headers and paper pattern are already in it. The +//! work is in the `opennote-print` crate, behind a trait; these two commands only carry bytes and errors. + +use opennote_print::{ErrorCode, PrintError, PrintRequest, PrinterInfo, Receipt, SystemBackend}; +use tauri::ipc::InvokeBody; + +use crate::{ + images::import::percent_decode, + ipc::{codes, IpcError, IpcResult}, +}; + +/// The error codes the interface reads, one for each kind of refusal. +fn ipc_error(error: PrintError) -> IpcError { + let code = match error.code { + ErrorCode::BadDocument | ErrorCode::BadCopies | ErrorCode::UnknownPrinter => codes::INVALID, + ErrorCode::NoPrinter => "noPrinter", + ErrorCode::Spool => "spool", + }; + IpcError::new(code, error.message) +} + +/// The printers the system lists. Empty where printing is not available. +#[tauri::command] +pub async fn print_printers() -> IpcResult> { + use opennote_print::PrintBackend; + tauri::async_runtime::spawn_blocking(|| SystemBackend.printers()) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string())) +} + +/// Sends a PDF to a printer. The body is the PDF, and the `x-opennote-print` header (percent-encoded JSON) holds the +/// `PrintRequest`. Returns when the spooler has the job. +#[tauri::command] +pub async fn print_submit(request: tauri::ipc::Request<'_>) -> IpcResult { + let header = request + .headers() + .get("x-opennote-print") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| IpcError::invalid("x-opennote-print", "The print header is missing."))?; + let job: PrintRequest = serde_json::from_str(&percent_decode(header)) + .map_err(|_| IpcError::invalid("x-opennote-print", "The print header isn't valid JSON."))?; + let pdf = match request.body() { + InvokeBody::Raw(bytes) => bytes.clone(), + InvokeBody::Json(_) => return Err(IpcError::invalid("body", "The document must come as raw bytes.")), + }; + tauri::async_runtime::spawn_blocking(move || opennote_print::print(&SystemBackend, &pdf, &job, &mut |_, _| {})) + .await + .map_err(|error| IpcError::new(codes::INTERNAL, error.to_string()))? + .map_err(ipc_error) +} diff --git a/app/src-tauri/src/settings/commands.rs b/app/src-tauri/src/settings/commands.rs index 35ce5c07..4e9c9342 100644 --- a/app/src-tauri/src/settings/commands.rs +++ b/app/src-tauri/src/settings/commands.rs @@ -39,6 +39,10 @@ pub fn settings_reset( } fn announce(window: &WebviewWindow, settings: &Settings) { + // How long history is kept reaches the core with every change (A4-13). + if let Some(pages) = window.try_state::() { + pages.set_retention(settings.editing.history.keep); + } let payload = SettingsChanged { settings: settings.clone(), origin: window.label().to_owned(), diff --git a/app/src-tauri/src/settings/editing.rs b/app/src-tauri/src/settings/editing.rs index 0b336dfd..2c6eeae1 100644 --- a/app/src-tauri/src/settings/editing.rs +++ b/app/src-tauri/src/settings/editing.rs @@ -13,8 +13,6 @@ pub struct EditingSettings { pub markdown_shortcuts: bool, pub slash_menu: bool, pub formatting_bar: FormattingBar, - /// Put the date and time under the title of a new page. - pub new_page_date_time: bool, #[cfg_attr(test, ts(inline))] pub autocorrect: Autocorrect, #[cfg_attr(test, ts(inline))] @@ -33,7 +31,6 @@ impl Default for EditingSettings { markdown_shortcuts: true, slash_menu: true, formatting_bar: FormattingBar::TouchAndPen, - new_page_date_time: true, autocorrect: Autocorrect::default(), paste: Paste::default(), spelling: Spelling::default(), @@ -137,7 +134,7 @@ impl Default for Paste { fn default() -> Self { Self { source_link: SourceLink::Ask, - save_web_images: true, + save_web_images: false, join_pdf_lines: true, } } @@ -230,9 +227,8 @@ mod tests { "markdownShortcuts": true, "slashMenu": true, "formattingBar": "touchAndPen", - "newPageDateTime": true, "autocorrect": { "enabled": true, "entries": [] }, - "paste": { "sourceLink": "ask", "saveWebImages": true, "joinPdfLines": true }, + "paste": { "sourceLink": "ask", "saveWebImages": false, "joinPdfLines": true }, "spelling": { "enabled": true, "languages": [], "personalWords": [], "ignoreUppercase": true, "ignoreWithDigits": true diff --git a/app/src-tauri/src/settings/ink.rs b/app/src-tauri/src/settings/ink.rs index cd3dfe03..bfa5dd45 100644 --- a/app/src-tauri/src/settings/ink.rs +++ b/app/src-tauri/src/settings/ink.rs @@ -29,6 +29,20 @@ pub struct InkSettings { pub zoom_box: ZoomBox, pub contrast_ink: ContrastInk, pub low_latency: LowLatency, + /// Named pen palettes the person saved (flag `ink.penLibrary`). Absent until the first one is saved. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(test, ts(optional))] + pub pen_sets: Option>, +} + +/// A named palette of pens, saved so a person can switch between a writing set and a drawing set. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS), ts(export, export_to = crate::BINDINGS))] +pub struct PenSet { + pub id: String, + pub name: String, + pub pens: Vec, } impl Default for InkSettings { @@ -61,6 +75,7 @@ impl Default for InkSettings { zoom_box: ZoomBox::default(), contrast_ink: ContrastInk::KeepLegible, low_latency: LowLatency::Auto, + pen_sets: None, } } } @@ -68,6 +83,8 @@ impl Default for InkSettings { impl InkSettings { /// The most pen slots the palette holds. pub const MAX_PENS: usize = 32; + /// The most pen sets a person can keep. + pub const MAX_PEN_SETS: usize = 12; pub fn check(&self, check: &mut Check) { let pens = &self.pens; @@ -88,6 +105,33 @@ impl InkSettings { "A pen's color isn't valid.", ); } + let sets = self.pen_sets.as_deref().unwrap_or_default(); + check.that( + "ink.penSets", + sets.len() <= Self::MAX_PEN_SETS, + "There can be at most 12 pen sets.", + ); + for (index, set) in sets.iter().enumerate() { + check.that( + "ink.penSets", + sets[..index].iter().all(|other| other.id != set.id), + "Two pen sets have the same id.", + ); + check.chars("ink.penSets", &set.name, (1, 40)); + check.that( + "ink.penSets", + (1..=Self::MAX_PENS).contains(&set.pens.len()), + "A pen set holds 1 to 32 pens.", + ); + for pen in &set.pens { + check.range("ink.penSets", pen.width, (0.1, 20.0)); + check.that( + "ink.penSets", + validate::ink_color(&pen.color, pen.tool == PenTool::Highlighter), + "A pen set's color isn't valid.", + ); + } + } check.range( "ink.touch.palmGraceMs", f64::from(self.touch.palm_grace_ms), @@ -134,6 +178,16 @@ pub enum PenTool { Pen, Pencil, Highlighter, + /// Pen library (flag `ink.penLibrary`): width from pressure and speed. + Fountain, + /// Pen library: a soft brush that is thick when pressed or slow. + Brush, + /// Pen library: a flat nib held at a fixed angle. + Calligraphy, + /// Pen library: a line of dashes. + Dashed, + /// Pen library: a line of dots. + Dotted, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] @@ -143,6 +197,20 @@ pub enum Handedness { #[default] Right, Left, + /// Phase 5: the palm filter learns the writing hand from where the palm rests. + Auto, +} + +/// Phase 5: when a finger draws while an ink tool is active. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS), ts(export, export_to = crate::BINDINGS))] +pub enum FingerDraw { + /// Until a pen is seen on this device. + #[default] + Auto, + On, + Off, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -151,6 +219,8 @@ pub enum Handedness { pub struct TouchInk { /// Draw with a finger when no pen is near. pub draws: bool, + /// When a finger draws; it replaces `draws`, which older versions read. + pub finger: FingerDraw, /// How long a touch stroke waits for a pen before it's committed, 300 to 2,000 ms. pub palm_grace_ms: u32, pub two_finger_scroll_near_pen: bool, @@ -160,6 +230,7 @@ impl Default for TouchInk { fn default() -> Self { Self { draws: false, + finger: FingerDraw::Auto, palm_grace_ms: 500, two_finger_scroll_near_pen: true, } diff --git a/app/src-tauri/src/settings/ink_tests.rs b/app/src-tauri/src/settings/ink_tests.rs index e4c39ee7..e7fa5193 100644 --- a/app/src-tauri/src/settings/ink_tests.rs +++ b/app/src-tauri/src/settings/ink_tests.rs @@ -16,7 +16,7 @@ fn settings_defaults_match_phase_5s_schema() { ], "penSelectsAndTypes": false, "handedness": "right", - "touch": { "draws": false, "palmGraceMs": 500, "twoFingerScrollNearPen": true }, + "touch": { "draws": false, "finger": "auto", "palmGraceMs": 500, "twoFingerScrollNearPen": true }, "eraser": { "mode": "stroke", "size": 4, "erases": "all", "returnToLastTool": false, "showTarget": true }, "lasso": { "shape": "free", "picks": ["ink", "highlighter", "text", "images", "shapes"], "inside": "mostly" }, "shapes": { "hold": true, "holdMs": 500, "inkToShape": false }, @@ -109,3 +109,43 @@ fn a_custom_curve_never_goes_down() { } assert_eq!(failed_field(|c| device.check(c)).as_deref(), Some("ink.pens")); } + +#[test] +fn pen_sets_are_optional_and_checked() { + let mut ink = InkSettings::default(); + assert!(serde_json::to_value(&ink).expect("serializes").get("penSets").is_none()); + let set = PenSet { + id: "s1".into(), + name: "Drawing".into(), + pens: vec![ + PenSlot { + id: "f1".into(), + tool: PenTool::Fountain, + color: "ink".into(), + width: 0.7, + pressure: Some(true), + }, + PenSlot { + id: "c1".into(), + tool: PenTool::Calligraphy, + color: "indigo".into(), + width: 2.0, + pressure: None, + }, + ], + }; + ink.pen_sets = Some(vec![set.clone()]); + assert_eq!(failed_field(|c| ink.check(c)), None); + let json = serde_json::to_value(&ink).expect("serializes"); + assert_eq!(json["penSets"][0]["pens"][1]["tool"], "calligraphy"); + let back: InkSettings = serde_json::from_value(json).expect("reads back"); + assert_eq!(back, ink); + ink.pen_sets = Some(vec![set.clone(), set]); + assert_eq!(failed_field(|c| ink.check(c)).as_deref(), Some("ink.penSets")); + ink.pen_sets = Some(vec![PenSet { + id: "s2".into(), + name: String::new(), + pens: vec![], + }]); + assert_eq!(failed_field(|c| ink.check(c)).as_deref(), Some("ink.penSets")); +} diff --git a/app/src-tauri/src/settings/lenient_tests.rs b/app/src-tauri/src/settings/lenient_tests.rs index d3091bc3..340949ba 100644 --- a/app/src-tauri/src/settings/lenient_tests.rs +++ b/app/src-tauri/src/settings/lenient_tests.rs @@ -48,7 +48,7 @@ fn keeps_the_good_entries_of_a_map_and_the_good_items_of_a_list() { "shortcuts": { "theme.toggle": ["Ctrl+Alt+K"], "app.bad": ["K"], "view.x": [] }, "ink": { "pens": [ { "id": "a", "tool": "pen", "color": "indigo", "width": 1.0 }, - { "id": "b", "tool": "brush", "color": "indigo", "width": 1.0 }, + { "id": "b", "tool": "chisel", "color": "indigo", "width": 1.0 }, { "id": "c", "tool": "highlighter", "color": "#f2cf4a66", "width": 4.0 } ] } })); diff --git a/app/src-tauri/src/share/activation.rs b/app/src-tauri/src/share/activation.rs new file mode 100644 index 00000000..84ba9ca1 --- /dev/null +++ b/app/src-tauri/src/share/activation.rs @@ -0,0 +1,140 @@ +//! Reads a "Share to OpenNote" launch from Windows. Only an exe with package identity (the sparse package in +//! packaging/msix) can be a share target, so an exe without one returns at once. The share operation is reported +//! complete as soon as its data is read, so the share sheet closes even if the app then takes a moment to start. + +use windows::{ + core::{Interface, HSTRING}, + ApplicationModel::{ + Activation::{ActivationKind, ShareTargetActivatedEventArgs}, + AppInstance, + DataTransfer::{DataPackageView, StandardDataFormats}, + }, + Storage::{FileIO, StorageFile, Streams::DataReader}, + Win32::{Foundation::APPMODEL_ERROR_NO_PACKAGE, Storage::Packaging::Appx::GetCurrentPackageFullName}, +}; + +use super::{Shared, SharedFile, MAX_FILE, MAX_FILES}; + +/// Whether this process runs with package identity. +fn has_identity() -> bool { + let mut length = 0u32; + // SAFETY: asks only for the length; a null buffer with length 0 is the documented way to do that. + let result = unsafe { GetCurrentPackageFullName(&mut length, None) }; + result != APPMODEL_ERROR_NO_PACKAGE +} + +/// What was shared, when this launch is a share. None for any other launch. +pub fn read() -> Option { + if !has_identity() { + return None; + } + let args = AppInstance::GetActivatedEventArgs().ok()?; + if args.Kind().ok()? != ActivationKind::ShareTarget { + return None; + } + let share: ShareTargetActivatedEventArgs = args.cast().ok()?; + let operation = share.ShareOperation().ok()?; + let shared = operation.Data().ok().map(|data| read_data(&data)).unwrap_or_default(); + if let Err(error) = operation.ReportCompleted() { + log::warn!("Couldn't tell Windows the share was received: {error}"); + } + Some(shared) +} + +fn has(data: &DataPackageView, format: windows::core::Result) -> bool { + format.and_then(|format| data.Contains(&format)).unwrap_or(false) +} + +fn read_data(data: &DataPackageView) -> Shared { + let mut shared = Shared::default(); + if has(data, StandardDataFormats::Text()) { + shared.text = data + .GetTextAsync() + .and_then(|task| task.join()) + .ok() + .map(|text| text.to_string()); + } + if has(data, StandardDataFormats::WebLink()) { + shared.uri = data + .GetWebLinkAsync() + .and_then(|task| task.join()) + .and_then(|uri| uri.AbsoluteUri()) + .ok() + .map(|uri| uri.to_string()); + } + if has(data, StandardDataFormats::Bitmap()) { + match read_bitmap(data) { + Ok(Some(file)) => shared.files.push(file), + Ok(None) => shared.skipped.push("Shared picture".to_owned()), + Err(error) => log::warn!("Couldn't read a shared picture: {error}"), + } + } + if has(data, StandardDataFormats::StorageItems()) { + if let Ok(items) = data.GetStorageItemsAsync().and_then(|task| task.join()) { + for item in items { + let Ok(file) = item.cast::() else { + // A folder: only files are added. + if let Ok(name) = item.Name() { + shared.skipped.push(name.to_string()); + } + continue; + }; + let name = file.Name().map(|name| name.to_string()).unwrap_or_default(); + if shared.files.len() == MAX_FILES { + shared.skipped.push(name); + continue; + } + match read_file(&file) { + Ok(Some(bytes)) => shared.files.push(SharedFile { + mime: file.ContentType().map(|mime| mime.to_string()).unwrap_or_default(), + name, + bytes, + }), + Ok(None) => shared.skipped.push(name), + Err(error) => log::warn!("Couldn't read a shared file: {error}"), + } + } + } + } + shared +} + +/// The picture, or None when it is larger than a file may be. +fn read_bitmap(data: &DataPackageView) -> windows::core::Result> { + let reference = data.GetBitmapAsync()?.join()?; + let stream = reference.OpenReadAsync()?.join()?; + let size = stream.Size()?; + if size > MAX_FILE { + return Ok(None); + } + let mime = stream.ContentType().map(|mime| mime.to_string()).unwrap_or_default(); + let reader = DataReader::CreateDataReader(&stream.GetInputStreamAt(0)?)?; + let length = u32::try_from(size).unwrap_or(u32::MAX); + reader.LoadAsync(length)?.join()?; + let mut bytes = vec![0u8; length as usize]; + reader.ReadBytes(&mut bytes)?; + let extension = match mime.as_str() { + "image/jpeg" => "jpg", + "image/gif" => "gif", + "image/bmp" => "bmp", + _ => "png", + }; + Ok(Some(SharedFile { + name: format!("Shared picture.{extension}"), + mime, + bytes, + })) +} + +/// The file's bytes, or None when it is larger than a file may be. +fn read_file(file: &StorageFile) -> windows::core::Result>> { + let size = file.GetBasicPropertiesAsync()?.join()?.Size()?; + if size > MAX_FILE { + return Ok(None); + } + let buffer = FileIO::ReadBufferAsync(file)?.join()?; + let reader = DataReader::FromBuffer(&buffer)?; + let mut bytes = vec![0u8; buffer.Length()? as usize]; + reader.ReadBytes(&mut bytes)?; + Ok(Some(bytes)) +} diff --git a/app/src-tauri/src/share/mod.rs b/app/src-tauri/src/share/mod.rs new file mode 100644 index 00000000..4cdf7f5e --- /dev/null +++ b/app/src-tauri/src/share/mod.rs @@ -0,0 +1,342 @@ +//! "Share to OpenNote" (flag `integrations.shareTarget`; docs/help/import-and-export.md). With the sparse package +//! in packaging/msix registered, OpenNote is in the Windows share sheet for text, links, pictures, and files. +//! +//! Windows starts `OpenNote.exe` for each share. That launch reads what was shared ([`activation`]), saves it to +//! the share inbox in the local data folder, tells Windows the share is done, and then starts as usual: either it +//! becomes the app, or it hands its arguments to the running window and exits. Either way the interface takes the +//! inbox with `share_take` and adds each share to the open page, or to a new page in Quick notes when no page is +//! open (app/src/features/integrations/share). +//! +//! What is kept is limited: text up to 2 MB, at most 10 files of up to 15 MB each, and only `http`, `https`, and +//! `mailto` links. Nothing is read from the inbox but the files this module wrote, and a share older than a day is +//! thrown away unread. + +#[cfg(windows)] +pub mod activation; + +use std::{ + fs, io, + path::{Path, PathBuf}, + time::{Duration, SystemTime}, +}; + +use base64::{engine::general_purpose::STANDARD, Engine as _}; +use serde::{Deserialize, Serialize}; +use tauri::{AppHandle, State}; + +use crate::{ + ipc::{IpcError, IpcResult}, + paths::Paths, + platform_flags, +}; + +/// The most text a share keeps. +pub const MAX_TEXT: usize = 2 * 1024 * 1024; +/// The most files a share keeps. +pub const MAX_FILES: usize = 10; +/// The largest file a share keeps. +pub const MAX_FILE: u64 = 15 * 1024 * 1024; +/// The most shares waiting in the inbox; more are left for the next take. +pub const MAX_WAITING: usize = 20; +/// A share left this long is thrown away. +pub const STALE: Duration = Duration::from_secs(24 * 60 * 60); + +const INBOX: &str = "share-inbox"; +const META: &str = "share.json"; + +/// A file that was shared, or a picture. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SharedFile { + pub name: String, + pub mime: String, + #[serde(skip)] + pub bytes: Vec, +} + +/// What another app shared, as Windows hands it over. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Shared { + pub text: Option, + pub uri: Option, + pub files: Vec, + /// Files that were too large or too many, by name. + #[serde(default)] + pub skipped: Vec, +} + +/// A share as the page gets it: a title for a new page, Markdown for a text block, and the files. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ShareBlocks { + pub title: String, + /// Plain text, as it was shared. + pub text: String, + /// A link that may be kept: `http`, `https`, or `mailto`. + pub link: Option, + /// The text and the link as Markdown, for a new page. + pub markdown: String, + pub files: Vec, + pub skipped: Vec, +} + +/// A file as the interface gets it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ShareFileOut { + pub name: String, + pub mime: String, + /// Base64. + pub data: String, +} + +/// A link a page may keep. +pub fn safe_link(uri: &str) -> Option { + let uri = uri.trim(); + let lower = uri.to_ascii_lowercase(); + let scheme_ok = ["http://", "https://", "mailto:"] + .iter() + .any(|scheme| lower.starts_with(scheme)); + (scheme_ok && uri.len() <= 2048 && !uri.chars().any(|c| c.is_control() || c.is_whitespace())) + .then(|| uri.to_owned()) +} + +/// Text without control characters other than tabs and line breaks, cut at [`MAX_TEXT`] on a character boundary. +pub fn clean_text(text: &str) -> String { + let mut out = String::with_capacity(text.len().min(MAX_TEXT)); + for c in text.chars().filter(|c| !c.is_control() || matches!(c, '\n' | '\t')) { + if out.len() + c.len_utf8() > MAX_TEXT { + break; + } + out.push(c); + } + out +} + +/// The type of a file from its name, for a share that didn't say. +pub fn mime_for(name: &str) -> &'static str { + let extension = name + .rsplit_once('.') + .map(|(_, ext)| ext.to_ascii_lowercase()) + .unwrap_or_default(); + match extension.as_str() { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "webp" => "image/webp", + "bmp" => "image/bmp", + "pdf" => "application/pdf", + "txt" => "text/plain", + "md" => "text/markdown", + "csv" => "text/csv", + "docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "mp3" => "audio/mpeg", + "m4a" => "audio/mp4", + "mp4" => "video/mp4", + _ => "application/octet-stream", + } +} + +/// A type the page's paste can trust: the shared one when it looks like a type, else the name's. +fn clean_mime(mime: &str, name: &str) -> String { + let mime = mime.trim().to_ascii_lowercase(); + let looks = mime.len() <= 100 + && mime + .split_once('/') + .is_some_and(|(kind, sub)| !kind.is_empty() && !sub.is_empty()) + && mime.bytes().all(|b| b.is_ascii_alphanumeric() || b"/.+-".contains(&b)); + if looks { + mime + } else { + mime_for(name).to_owned() + } +} + +/// Maps a share to what goes on a page: the first line of text (or the link, or the first file) is the title of a +/// new page; the link goes first, then the text; files keep safe names and a type. +pub fn to_blocks(shared: Shared) -> ShareBlocks { + let text = shared.text.as_deref().map(clean_text).unwrap_or_default(); + let link = shared.uri.as_deref().and_then(safe_link); + let files: Vec = shared + .files + .into_iter() + .take(MAX_FILES) + .map(|file| { + let name = opennote_api::routes::file_name(&file.name); + ShareFileOut { + mime: clean_mime(&file.mime, &name), + name, + data: STANDARD.encode(&file.bytes), + } + }) + .collect(); + let first_line = text.lines().map(str::trim).find(|line| !line.is_empty()); + let title_source = first_line + .map(str::to_owned) + .or_else(|| link.as_deref().map(link_title)) + .or_else(|| files.first().map(|file| file.name.clone())) + .unwrap_or_else(|| "Shared item".to_owned()); + let title: String = title_source.chars().take(80).collect(); + let mut markdown = String::new(); + if let Some(link) = &link { + markdown.push('<'); + markdown.push_str(link); + markdown.push('>'); + } + if !text.trim().is_empty() && Some(text.trim()) != link.as_deref() { + if !markdown.is_empty() { + markdown.push_str("\n\n"); + } + markdown.push_str(text.trim()); + } + ShareBlocks { + title, + text, + link, + markdown, + files, + skipped: shared.skipped, + } +} + +/// A short title for a link: its host and path, without the scheme. +fn link_title(link: &str) -> String { + let rest = link.split_once("://").map_or(link, |(_, rest)| rest); + let rest = rest.strip_prefix("mailto:").unwrap_or(rest); + rest.trim_end_matches('/').to_owned() +} + +/// The inbox folder in the local data folder. +pub fn inbox(paths: &Paths) -> PathBuf { + paths.local.join(INBOX) +} + +fn new_id() -> String { + let millis = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis()) + .unwrap_or_default(); + format!("{millis:013}-{}", opennote_api::grants::random_hex(4)) +} + +/// Saves a share to the inbox. Files go beside `share.json` as `0.bin`, `1.bin`, and so on, never by their own name. +pub fn save(inbox: &Path, mut shared: Shared) -> io::Result { + let id = new_id(); + let folder = inbox.join(&id); + fs::create_dir_all(&folder)?; + let mut kept = Vec::new(); + for file in shared.files.drain(..) { + if kept.len() == MAX_FILES || file.bytes.len() as u64 > MAX_FILE { + shared.skipped.push(file.name); + continue; + } + fs::write(folder.join(format!("{}.bin", kept.len())), &file.bytes)?; + kept.push(SharedFile { + bytes: Vec::new(), + ..file + }); + } + shared.files = kept; + shared.text = shared.text.as_deref().map(clean_text); + let json = serde_json::to_vec(&shared).map_err(io::Error::other)?; + // Written last, so a half-saved share is never taken. + fs::write(folder.join(META), json)?; + Ok(id) +} + +/// Takes every complete share from the inbox, oldest first, and removes them. Stale or broken ones are removed +/// unread. +pub fn take_all(inbox: &Path, now: SystemTime) -> Vec { + let Ok(entries) = fs::read_dir(inbox) else { + return Vec::new(); + }; + let mut folders: Vec = entries + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) + .map(|entry| entry.path()) + .collect(); + folders.sort(); + let mut out = Vec::new(); + for folder in folders { + if out.len() == MAX_WAITING { + break; + } + let meta = folder.join(META); + let modified = fs::metadata(&meta).and_then(|meta| meta.modified()); + let Ok(modified) = modified else { + // Still being written, unless it has sat there a day. + let old = fs::metadata(&folder) + .and_then(|meta| meta.modified()) + .is_ok_and(|time| now.duration_since(time).unwrap_or_default() > STALE); + if old { + let _ = fs::remove_dir_all(&folder); + } + continue; + }; + let fresh = now.duration_since(modified).unwrap_or_default() <= STALE; + if fresh { + if let Some(blocks) = read_share_blocks(&folder) { + out.push(blocks); + } + } + if let Err(error) = fs::remove_dir_all(&folder) { + log::warn!("Couldn't clear a share from the inbox: {error}"); + } + } + out +} + +fn read_share(folder: &Path) -> Option { + let mut shared: Shared = serde_json::from_slice(&fs::read(folder.join(META)).ok()?).ok()?; + let mut files = Vec::with_capacity(shared.files.len()); + for (index, file) in shared.files.drain(..).take(MAX_FILES).enumerate() { + let path = folder.join(format!("{index}.bin")); + if fs::metadata(&path).is_ok_and(|meta| meta.len() <= MAX_FILE) { + if let Ok(bytes) = fs::read(&path) { + files.push(SharedFile { bytes, ..file }); + } + } + } + shared.files = files; + Some(shared) +} + +fn read_share_blocks(folder: &Path) -> Option { + read_share(folder).map(to_blocks) +} + +/// Saves a share this launch was started for, before the instance lock hands the launch to a running window. It runs +/// before the settings load, so it follows the channel's default and `OPENNOTE_FLAGS`; `share_take` then applies +/// the settings too. Best effort: a failure is logged, and the app starts anyway. +pub fn receive_launch(paths: &Paths) { + if !platform_flags::is_on_now(platform_flags::SHARE_TARGET, &std::collections::BTreeMap::new()) { + return; + } + #[cfg(windows)] + if let Some(shared) = activation::read() { + match save(&inbox(paths), shared) { + Ok(_) => log::info!("Saved a share for the window to add."), + Err(error) => log::warn!("Couldn't save what was shared: {error}"), + } + } + #[cfg(not(windows))] + let _ = paths; +} + +/// The paths the share commands need. +pub struct ShareState { + pub inbox: PathBuf, +} + +/// Takes the shares waiting in the inbox. Refused when the flag is off. +#[tauri::command] +pub fn share_take(app: AppHandle, state: State<'_, ShareState>) -> IpcResult> { + if !platform_flags::is_on_for(&app, platform_flags::SHARE_TARGET) { + return Err(IpcError::new("off", "Share to OpenNote is off.")); + } + Ok(take_all(&state.inbox, SystemTime::now())) +} + +#[cfg(test)] +mod tests; diff --git a/app/src-tauri/src/share/tests.rs b/app/src-tauri/src/share/tests.rs new file mode 100644 index 00000000..5bf9c95b --- /dev/null +++ b/app/src-tauri/src/share/tests.rs @@ -0,0 +1,185 @@ +use std::time::{Duration, SystemTime}; + +use super::*; + +fn file(name: &str, mime: &str, bytes: &[u8]) -> SharedFile { + SharedFile { + name: name.into(), + mime: mime.into(), + bytes: bytes.to_vec(), + } +} + +#[test] +fn text_becomes_a_page_titled_by_its_first_line() { + let blocks = to_blocks(Shared { + text: Some("\n Shopping list\nEggs\nMilk\u{0007}".into()), + ..Shared::default() + }); + assert_eq!(blocks.title, "Shopping list"); + assert_eq!(blocks.markdown, "Shopping list\nEggs\nMilk"); + assert_eq!(blocks.link, None); + assert!(blocks.files.is_empty()); +} + +#[test] +fn a_link_goes_first_and_titles_the_page_when_there_is_no_text() { + let blocks = to_blocks(Shared { + uri: Some("https://example.org/rivers/".into()), + ..Shared::default() + }); + assert_eq!(blocks.title, "example.org/rivers"); + assert_eq!(blocks.markdown, ""); + let both = to_blocks(Shared { + text: Some("Read this".into()), + uri: Some("https://example.org/a".into()), + ..Shared::default() + }); + assert_eq!(both.markdown, "\n\nRead this"); + // A browser shares the address as the text too; it isn't added twice. + let same = to_blocks(Shared { + text: Some("https://example.org/a".into()), + uri: Some("https://example.org/a".into()), + ..Shared::default() + }); + assert_eq!(same.markdown, ""); +} + +#[test] +fn only_web_and_mail_links_are_kept() { + for bad in [ + "javascript:alert(1)", + "file:///C:/Windows/win.ini", + "ms-settings:privacy", + "https://a b", + "data:text/html,x", + ] { + assert_eq!(safe_link(bad), None, "{bad}"); + let blocks = to_blocks(Shared { + uri: Some(bad.into()), + ..Shared::default() + }); + assert_eq!(blocks.link, None); + assert!(!blocks.markdown.contains(':'), "{bad}: {}", blocks.markdown); + } + assert_eq!( + safe_link("mailto:a@example.org").as_deref(), + Some("mailto:a@example.org") + ); +} + +#[test] +fn pictures_and_files_keep_safe_names_and_a_trusted_type() { + let blocks = to_blocks(Shared { + files: vec![ + file("Shared picture.png", "image/png", b"png"), + file("..\\..\\evil.exe", "application/x-msdownload", b"MZ"), + file("notes.pdf", "", b"%PDF"), + file("odd.docx", "b')).toBe('ab'); + expect(markupText('shown')).toBe('shown'); + }); + + it('reads split, nested, and broken tags as a browser does, and runs none of them', () => { + // What a browser shows for each. Characters such as "<" can be left over, but only ever as text: the result is + // never parsed again, and no script inside is kept. + const shown: [string, string][] = [ + ['ipt>alert(1)', 'ipt>alert(1)'], + ['</script>', ''], + ['-- -->', '-- -->'], + ['ipt>alert(1)', 'ipt>alert(1)'], + ['x', 'x'], + [' b -->c', 'b -->c'], + ]; + for (const [hostile, text] of shown) expect(markupText(hostile), hostile).toBe(text); + const box = document.createElement('div'); + box.textContent = markupText('</script>'); + expect(box.children).toHaveLength(0); + }); + + it('decodes entities once, so escaped markup reads as the characters it names', () => { + expect(markupText('<script>alert(1)</script>')).toBe(''); + expect(markupText('&lt;b&gt;')).toBe('<b>'); + expect(markupText('C:\\Notes\\a.txt & \\textbf{x}')).toBe('C:\\Notes\\a.txt & \\textbf{x}'); + }); + + it('breaks lines at
, and after blocks when asked', () => { + expect(markupText('a
b
c
d')).toBe('a\nbcd'); + expect(markupText('a
b
c
d', { lines: true })).toBe('a\nbc\nd'); + }); +}); diff --git a/app/src/core/markupText.ts b/app/src/core/markupText.ts new file mode 100644 index 00000000..ae3e25a3 --- /dev/null +++ b/app/src/core/markupText.ts @@ -0,0 +1,26 @@ +// The words in a piece of HTML, read with the browser's own parser, for the places that take text out of markup +// (pasted tables, Anki fields). Removing tags with patterns can be fooled: one pass over "ipt>" leaves +// a new tag behind, and comments, CDATA, and unclosed tags each need their own rule. The parser already has them +// all, and a DOMParser document is inert: nothing in it runs or loads. + +/** Elements whose content is never shown as words. */ +const HIDDEN = 'script, style, template, noscript, title, object, embed, iframe'; +/** Elements that end a line when `lines` is on. */ +const BLOCKS = 'address, article, blockquote, dd, div, dt, figcaption, h1, h2, h3, h4, h5, h6, li, p, pre, section, tr'; + +export interface MarkupTextOptions { + /** End a line after each block, such as a paragraph or a list item, as well as at each `
`. */ + lines?: boolean; +} + +/** + * The text of an HTML fragment, with entities decoded. A `
` becomes a line break, and with `lines` so does the + * end of each block. Everything else is the parser's text content, so the result is plain text, never markup. + */ +export function markupText(html: string, options: MarkupTextOptions = {}): string { + const body = new DOMParser().parseFromString(html, 'text/html').body; + for (const hidden of body.querySelectorAll(HIDDEN)) hidden.remove(); + for (const br of body.querySelectorAll('br')) br.replaceWith('\n'); + if (options.lines) for (const block of body.querySelectorAll(BLOCKS)) block.append('\n'); + return body.textContent ?? ''; +} diff --git a/app/src/core/paperLattice.test.ts b/app/src/core/paperLattice.test.ts new file mode 100644 index 00000000..b64c2a1d --- /dev/null +++ b/app/src/core/paperLattice.test.ts @@ -0,0 +1,143 @@ +// The paper lattice's arithmetic: which line a point snaps to on each kind of paper, how far it reaches at each zoom, +// the header above the first sheet's lines, and the sheets of a paginated page. +import { describe, expect, it } from 'vitest'; +import { + linesArea, + nearestLine, + nearestRow, + sheetIndex, + snapReach, + snapToLattice, + stepToLine, + UNBOUNDED, +} from './paperLattice'; +import type { PaperLattice } from './paperLattice'; + +const H = 1056; +const grid: PaperLattice = { + kind: 'grid', + step: 20, + origin: { x: 96, y: 72 }, + area: { x: 96, y: 72, w: 620, h: 900 }, + sheet: H, + margin: null, +}; +const dots: PaperLattice = { ...grid, kind: 'dots' }; +const ruled: PaperLattice = { + kind: 'ruled', + step: 26, + origin: { x: 96, y: 72 }, + area: { x: 0, y: 98, w: 816, h: 886 }, + sheet: H, + margin: 96, +}; +const infinite: PaperLattice = { ...grid, area: UNBOUNDED, sheet: null }; + +describe('snapReach', () => { + it('is a third of the spacing, at least a few screen pixels, at most half the spacing', () => { + expect(snapReach(30, 1)).toBe(10); + expect(snapReach(30, 2)).toBe(10); + expect(snapReach(30, 0.5)).toBe(12); + expect(snapReach(9, 1)).toBe(4.5); + expect(snapReach(12, 0.1)).toBe(6); + }); + + it('reaches the same distance on the screen at 50, 100, and 200 percent for fine paper', () => { + for (const zoom of [0.5, 1, 2]) expect(snapReach(8, zoom) * zoom).toBeGreaterThanOrEqual(Math.min(6, 4 * zoom)); + }); +}); + +describe('snapToLattice on grid paper', () => { + it('snaps each coordinate to its nearest line, so a point near a crossing lands on it', () => { + expect(snapToLattice(grid, { x: 118, y: 95 }, 7)).toEqual({ point: { x: 116, y: 92 }, x: true, y: true }); + }); + + it('snaps one coordinate when only that one is within reach', () => { + expect(snapToLattice(grid, { x: 106, y: 93 }, 6)).toEqual({ point: { x: 106, y: 92 }, x: false, y: true }); + }); + + it('snaps at exactly the reach and not beyond it', () => { + expect(snapToLattice(grid, { x: 122, y: 400 }, 6).x).toBe(true); + expect(snapToLattice(grid, { x: 122.01, y: 400 }, 6).x).toBe(false); + }); + + it('leaves a point in the margin, away from the lines, where it is', () => { + expect(snapToLattice(grid, { x: 30, y: 30 }, 6)).toEqual({ point: { x: 30, y: 30 }, x: false, y: false }); + }); + + it('snaps to the lines of the sheet the point is on, which start again at its own top margin', () => { + const hit = snapToLattice(grid, { x: 117, y: H + 73 }, 6); + expect(hit.point).toEqual({ x: 116, y: H + 72 }); + expect(snapToLattice(grid, { x: 117, y: 2 * H + 72 + 19 }, 6).point).toEqual({ x: 116, y: 2 * H + 72 + 20 }); + }); + + it('never snaps to a line that is not drawn above the header', () => { + const headed = { ...grid, from: 172 }; + expect(snapToLattice(headed, { x: 300, y: 151 }, 6).y).toBe(false); + expect(snapToLattice(headed, { x: 300, y: 168 }, 6).point.y).toBe(172); + // The column ends at the header too: a point above it keeps its x. + expect(snapToLattice(headed, { x: 117, y: 140 }, 6).x).toBe(false); + }); + + it('runs every line on and on with no sheets', () => { + expect(snapToLattice(infinite, { x: 5017, y: 8994 }, 6).point).toEqual({ x: 5016, y: 8992 }); + }); +}); + +describe('snapToLattice on dot paper', () => { + it('snaps to a dot when both coordinates are near it', () => { + expect(snapToLattice(dots, { x: 118, y: 95 }, 7)).toEqual({ point: { x: 116, y: 92 }, x: true, y: true }); + }); + + it('leaves a point between the dots where it is, even near a row', () => { + expect(snapToLattice(dots, { x: 106, y: 93 }, 6)).toEqual({ point: { x: 106, y: 93 }, x: false, y: false }); + }); +}); + +describe('snapToLattice on ruled paper', () => { + it('snaps y to the nearest rule and x to the margin line', () => { + expect(snapToLattice(ruled, { x: 99, y: 128 }, 8)).toEqual({ point: { x: 96, y: 124 }, x: true, y: true }); + expect(snapToLattice(ruled, { x: 300, y: 128 }, 8)).toEqual({ point: { x: 300, y: 124 }, x: false, y: true }); + }); + + it('has no rule in the top margin, where the first rule is a step below it', () => { + expect(snapToLattice(ruled, { x: 300, y: 74 }, 8).y).toBe(false); + expect(snapToLattice(ruled, { x: 300, y: 92 }, 8).point.y).toBe(98); + }); + + it('snaps x only to the margin line it draws', () => { + expect(snapToLattice({ ...ruled, margin: null }, { x: 97, y: 124 }, 8).x).toBe(false); + }); +}); + +describe('the lattice helpers', () => { + it('finds the nearest drawn line within a range, like the renderer', () => { + expect(nearestLine(31, 0, 10, 0, 100)).toBe(30); + expect(nearestLine(-30, 0, 10, 0, 100)).toBe(0); + expect(nearestLine(500, 0, 10, 0, 100)).toBe(100); + expect(nearestLine(5, 0, 10, 12, 18)).toBeNull(); + }); + + it('cuts the first sheet at the header only', () => { + const headed = { ...ruled, from: 202 }; + expect(linesArea(headed, 0).y).toBe(202); + expect(linesArea(headed, 1).y).toBe(H + 98); + expect(sheetIndex(headed, H - 0.001)).toBe(1); + expect(sheetIndex(headed, H - 1)).toBe(0); + }); + + it('finds the rule under a point within reach, and none beyond it', () => { + expect(nearestRow(ruled, { x: 300, y: 136 }, 13)).toBe(124); + expect(nearestRow(ruled, { x: 300, y: 138 }, 13)).toBe(150); + expect(nearestRow(ruled, { x: 300, y: 137 }, 8)).toBeNull(); + }); + + it('steps to the next line, or to the one a point between two is moving toward', () => { + expect(stepToLine(grid, 116, 'x', 1, 0)).toBe(20); + expect(stepToLine(grid, 110, 'x', 1, 0)).toBe(6); + expect(stepToLine(grid, 110, 'x', -1, 0)).toBe(-14); + expect(stepToLine(grid, H + 92, 'y', -1, H + 92)).toBe(-20); + expect(stepToLine(ruled, 300, 'x', 1, 0)).toBe(26); + expect(stepToLine(ruled, 130, 'y', 1, 130)).toBe(20); + }); +}); diff --git a/app/src/core/paperLattice.ts b/app/src/core/paperLattice.ts new file mode 100644 index 00000000..078e235d --- /dev/null +++ b/app/src/core/paperLattice.ts @@ -0,0 +1,224 @@ +// The lines of a page's paper as a lattice: where ruled, grid, and dot paper draw their lines, in page units. The +// paper renderer (features/pages/paper/patterns.ts) draws from it and the drawing tools snap to it, so a point that +// snaps sits exactly on a line the page shows, on every sheet and at every zoom. It is pure arithmetic, shared by +// the page view, print, and the ink view without either importing the other. +import type { RuleGrid } from './ruled'; + +/** A box in page units. */ +export interface LatticeRect { + readonly x: number; + readonly y: number; + readonly w: number; + readonly h: number; +} + +export interface LatticePoint { + readonly x: number; + readonly y: number; +} + +export type LatticeKind = 'ruled' | 'grid' | 'dots'; + +/** + * The lines of ruled, grid, or dot paper. Horizontal lines are at `origin.y + j * step` and, on grid and dot paper, + * vertical lines at `origin.x + i * step`; dot paper has a dot where they cross. On a paginated page the lattice + * starts again on every sheet, `sheet` page units further down. + */ +export interface PaperLattice { + readonly kind: LatticeKind; + /** The distance between two lines, in page units. */ + readonly step: number; + /** A crossing of the lattice: the left and top margins. */ + readonly origin: LatticePoint; + /** Where one sheet draws its lines, from the sheet's top-left corner, or where the whole page draws them. */ + readonly area: LatticeRect; + /** The height of a sheet when the page is paginated, or null for one long page. */ + readonly sheet: number | null; + /** The y of the first line below the page's title band, when it has one: the first sheet has no lines above it. */ + readonly from?: number; + /** The x of ruled paper's margin line, which runs the whole height of the page, or null when it has none. */ + readonly margin: number | null; +} + +/** Tolerance for a line at the edge of the area, in page units: the renderer's own (pagination/geometry.ts EPS). */ +export const LATTICE_EPS = 0.01; + +/** An area that has no edges, for a page with no sheets. Finite, so its edges stay ordinary numbers. */ +export const UNBOUNDED: LatticeRect = { x: -1e7, y: -1e7, w: 2e7, h: 2e7 }; + +/** + * The box with everything above `from` cut off, or the box itself when `from` is not given. A page's title and its + * date sit in a header the rules leave blank: the rules (and the squares and dots) of the first sheet begin at `from`. + */ +export function below(box: LatticeRect, from: number | undefined): LatticeRect { + if (from === undefined || from <= box.y) return box; + const y = Math.min(from, box.y + box.h); + return { ...box, y, h: box.y + box.h - y }; +} + +/** The part of `a` inside `b`. An empty overlap is a box of no size at the nearer edge. */ +export function intersect(a: LatticeRect, b: LatticeRect): LatticeRect { + const x = Math.max(a.x, b.x); + const y = Math.max(a.y, b.y); + const right = Math.min(a.x + a.w, b.x + b.w); + const bottom = Math.min(a.y + a.h, b.y + b.h); + return { x: Math.min(x, right), y: Math.min(y, bottom), w: Math.max(0, right - x), h: Math.max(0, bottom - y) }; +} + +/** The sheet that holds `y`, from 0, or 0 on a page with no sheets. */ +export function sheetIndex(lattice: PaperLattice, y: number): number { + return lattice.sheet === null ? 0 : Math.max(0, Math.floor((y + LATTICE_EPS) / lattice.sheet)); +} + +/** Where sheet `k` draws its lines, in page units: its area moved down to the sheet, below the header on sheet 0. */ +export function linesArea(lattice: PaperLattice, k: number): LatticeRect { + const dy = lattice.sheet === null ? 0 : k * lattice.sheet; + const area = { ...lattice.area, y: lattice.area.y + dy }; + return k === 0 ? below(area, lattice.from) : area; +} + +/** The y of rule 0 on sheet `k`: the lattice's origin, moved down to the sheet. */ +export function sheetOriginY(lattice: PaperLattice, k: number): number { + return lattice.origin.y + (lattice.sheet === null ? 0 : k * lattice.sheet); +} + +/** + * The line nearest `v` of the lines at `origin + k * step` that lie within [lo, hi], or null when none does. The + * same arithmetic as the renderer's `ticks`, so the line it gives is one the renderer draws. + */ +export function nearestLine(v: number, origin: number, step: number, lo: number, hi: number): number | null { + if (!(step > 0) || !(hi >= lo)) return null; + const first = Math.ceil((lo - origin - LATTICE_EPS) / step); + const last = Math.floor((hi - origin + LATTICE_EPS) / step); + if (last < first) return null; + const k = Math.min(last, Math.max(first, Math.round((v - origin) / step))); + return origin + k * step; +} + +/** How far a point reaches for a line: a third of the spacing, at least `minPx` screen pixels, at most half. */ +export function snapReach(step: number, zoom: number, minPx = 6): number { + return Math.min(step / 2, Math.max(step / 3, minPx / Math.max(zoom, 1e-3))); +} + +/** A point after snapping: where it went, and which of its coordinates now lie on a line. */ +export interface LatticeSnap { + readonly point: LatticePoint; + readonly x: boolean; + readonly y: boolean; +} + +interface AxisHit { + readonly value: number; + readonly distance: number; + /** Where the line is drawn along its length: x for a row, y for a column. */ + readonly span: readonly [number, number]; +} + +const inSpan = (v: number, [lo, hi]: readonly [number, number]) => v >= lo - LATTICE_EPS && v <= hi + LATTICE_EPS; + +const closer = (a: AxisHit | null, b: AxisHit | null): AxisHit | null => + !a ? b : !b ? a : b.distance < a.distance ? b : a; + +/** The horizontal line of sheet `k` within `reach` of `p`, where that line is drawn under `p`. */ +function rowHit(lattice: PaperLattice, k: number, p: LatticePoint, reach: number): AxisHit | null { + const area = linesArea(lattice, k); + if (p.x < area.x - reach || p.x > area.x + area.w + reach) return null; + const y = nearestLine(p.y, sheetOriginY(lattice, k), lattice.step, area.y, area.y + area.h); + if (y === null || Math.abs(y - p.y) > reach) return null; + return { value: y, distance: Math.abs(y - p.y), span: [area.x, area.x + area.w] }; +} + +/** The vertical line of sheet `k` within `reach` of `p`, where that line is drawn beside `p`. */ +function columnHit(lattice: PaperLattice, k: number, p: LatticePoint, reach: number): AxisHit | null { + const area = linesArea(lattice, k); + if (p.y < area.y - reach || p.y > area.y + area.h + reach) return null; + const x = nearestLine(p.x, lattice.origin.x, lattice.step, area.x, area.x + area.w); + if (x === null || Math.abs(x - p.x) > reach) return null; + return { value: x, distance: Math.abs(x - p.x), span: [area.y, area.y + area.h] }; +} + +/** The sheets whose lines can be within reach of `y`: its own and the ones next to it. */ +function nearSheets(lattice: PaperLattice, y: number): number[] { + if (lattice.sheet === null) return [0]; + const k = sheetIndex(lattice, y); + return [k - 1, k, k + 1].filter((s) => s >= 0); +} + +/** + * A point snapped to the paper's lines within `reach` page units. Grid paper snaps each coordinate to its nearest + * line, so a point near a crossing lands on it. Dot paper snaps to a dot when both coordinates are within reach of + * one. Ruled paper snaps y to its nearest rule, and x to the margin line when one is drawn. + */ +export function snapToLattice(lattice: PaperLattice, p: LatticePoint, reach: number): LatticeSnap { + if (lattice.kind === 'dots') return snapToDot(lattice, p, reach); + let row: AxisHit | null = null; + let column: AxisHit | null = null; + for (const k of nearSheets(lattice, p.y)) { + row = closer(row, rowHit(lattice, k, p, reach)); + if (lattice.kind === 'grid') column = closer(column, columnHit(lattice, k, p, reach)); + } + if (lattice.kind === 'ruled' && lattice.margin !== null && Math.abs(p.x - lattice.margin) <= reach) { + column = { value: lattice.margin, distance: Math.abs(p.x - lattice.margin), span: [0, Infinity] }; + } + // A coordinate snaps only where its line is drawn under the point it makes: near the end of a line, the point + // keeps the other coordinate it had, and the line's end must reach it. + let x = column ? column.value : p.x; + let y = row ? row.value : p.y; + if (column && !inSpan(y, column.span)) { + column = null; + x = p.x; + } + if (row && !inSpan(x, row.span)) { + row = null; + y = p.y; + } + if (column && !inSpan(y, column.span)) { + column = null; + x = p.x; + } + return { point: { x, y }, x: column !== null, y: row !== null }; +} + +/** The dot nearest `p`, where both its row and its column are within reach and drawn on the same sheet. */ +function snapToDot(lattice: PaperLattice, p: LatticePoint, reach: number): LatticeSnap { + let best: { point: LatticePoint; distance: number } | null = null; + for (const k of nearSheets(lattice, p.y)) { + const row = rowHit(lattice, k, p, reach); + const column = columnHit(lattice, k, p, reach); + if (!row || !column) continue; + const distance = Math.hypot(row.distance, column.distance); + if (!best || distance < best.distance) best = { point: { x: column.value, y: row.value }, distance }; + } + return best ? { point: best.point, x: true, y: true } : { point: p, x: false, y: false }; +} + +/** The horizontal line nearest `p` within `reach`, where one is drawn under `p`; or null. */ +export function nearestRow(lattice: PaperLattice, p: LatticePoint, reach: number): number | null { + let best: AxisHit | null = null; + for (const k of nearSheets(lattice, p.y)) best = closer(best, rowHit(lattice, k, p, reach)); + return best ? best.value : null; +} + +/** + * How far to move `v` so it lands on the next line in `direction`. On a line that is the one a step on, and between two lines it is the + * one it is moving toward. `axis` says which lines: 'x' for vertical lines and 'y' for horizontal ones, which start + * again on each sheet. Ruled paper has no vertical lines but its margin, so along x it moves one whole step. + */ +export function stepToLine(lattice: PaperLattice, v: number, axis: 'x' | 'y', direction: 1 | -1, at: number): number { + const { step } = lattice; + if (axis === 'x' && lattice.kind === 'ruled') return direction * step; + const origin = axis === 'x' ? lattice.origin.x : sheetOriginY(lattice, sheetIndex(lattice, at)); + const t = (v - origin) / step; + const tolerance = LATTICE_EPS / step; + const next = direction === 1 ? Math.floor(t + tolerance) + 1 : Math.ceil(t - tolerance) - 1; + return origin + next * step - v; +} + +/** + * The rules text is laid out in (core/ruled.ts), read from the lattice the paper draws: rule k at the lattice's + * origin plus whole spacings, down the page or down each sheet. Text boxes snap through it, so text and drawn shapes + * land on the same lines. + */ +export function ruleGridOf(lattice: PaperLattice): RuleGrid { + return { step: lattice.step, origin: lattice.origin.y, sheet: lattice.sheet }; +} diff --git a/app/src/core/ruled.ts b/app/src/core/ruled.ts new file mode 100644 index 00000000..20310038 --- /dev/null +++ b/app/src/core/ruled.ts @@ -0,0 +1,116 @@ +// Ruled paper as the unit of text layout, shared by the page view, print, and export. When a page's paper has rules +// (ruled, grid, or dots), its spacing is the one unit the page lays out in. Body text takes one rule for each line +// with its baseline on the rule. Headings and larger text take whole rules. A text box sits so that its first +// baseline lands on a rule. The CSS does the work with +// the custom properties that `ruleProperties` writes, so a change of spacing, paper, text size, or zoom keeps +// everything aligned without moving any stored value. This file is the math for what CSS cannot do: placing a text +// box that is created, moved, or resized, and padding the blocks that don't come in whole rules. + +/** The lattice of rules. Rule k is at `origin + k * step`, down the page (or down each sheet, when `sheet` is set). */ +export interface RuleGrid { + /** The distance between two rules, in page units. */ + readonly step: number; + /** The y of rule 0: the top margin. The first drawn rule of ruled paper is one step below it. */ + readonly origin: number; + /** The height of a sheet when the page is paginated: the lattice starts again at the top margin of every sheet. */ + readonly sheet: number | null; +} + +/** Tolerance for a height that is a whole number of rules, in page units. Layout rounds to fractions of a pixel. */ +const WHOLE = 0.5; + +/** + * The finer tolerance for the pad under a block: layout measures in 64ths of a pixel, so the cap-height lengths of + * a callout or a code block leave a block a few hundredths of a unit short of its rule. Let alone, each block adds + * its share and a long page drifts off its rules by more than a pixel; the pad takes it up at the end of each block. + */ +export const FINE = 0.02; + +/** The y of the rule at or above `y` in its sheet's lattice, and the sheet's own top. */ +function cell(y: number, grid: RuleGrid): { top: number; into: number } { + const top = grid.sheet === null ? 0 : y - (((y % grid.sheet) + grid.sheet) % grid.sheet); + return { top, into: y - top }; +} + +/** The first rule on a sheet: a text box never snaps above the top of the page, or of its sheet. */ +const firstRule = (grid: RuleGrid): number => ((grid.origin % grid.step) + grid.step) % grid.step; + +/** The rule nearest to `y`, which is where a text box's top goes so its first baseline lands on a rule. */ +export function snapY(y: number, grid: RuleGrid | null): number { + if (!grid) return y; + const { top, into } = cell(y, grid); + const at = Math.max(into, firstRule(grid)); + return top + grid.origin + Math.round((at - grid.origin) / grid.step) * grid.step; +} + +/** The rule one step above or below `y`'s nearest rule: what a nudge with the arrow keys does on ruled paper. */ +export function stepY(y: number, direction: 1 | -1, grid: RuleGrid | null): number { + if (!grid) return y + direction; + const here = snapY(y, grid); + const moved = snapY(here + direction * grid.step, grid); + // A y that was between rules snaps toward the direction first, so one press never skips a rule. + if (direction === 1 && here > y + WHOLE) return here; + if (direction === -1 && here < y - WHOLE) return here; + return moved; +} + +/** A height rounded up to whole rules, at least one: what a block that does not come in rules is padded to. */ +export function wholeRules(height: number, grid: RuleGrid | null): number { + if (!grid) return height; + return Math.max(1, Math.ceil((height - WHOLE) / grid.step)) * grid.step; +} + +/** How far down from `top` the next rule is, from 0 to a step. A flow that starts there has its lines on the rules. */ +export function leadFor(top: number, grid: RuleGrid, tolerance = WHOLE): number { + const { into } = cell(top, grid); + const lead = (((grid.origin - into) % grid.step) + grid.step) % grid.step; + return lead < tolerance || grid.step - lead < tolerance ? 0 : lead; +} + +/** + * The y of the first rule drawn on a page that has a header: the rule the page's first line of text sits above. The + * title and the date under it sit in an unruled header, like the top margin of a notebook page. The rules begin + * below it. `flowTop` is the top of the flow (the bottom of the title band and its margin). `lineMiddle` is the + * middle of the first line's letters, when it has any. A line that takes several rules (large text, or fine ruling) + * sits above the last of them. With no letters, the first rule below the flow's top is the first. + */ +export function firstRuleBelow(flowTop: number, grid: RuleGrid, lineMiddle?: number): number { + const first = flowTop + leadFor(flowTop, grid) + grid.step; + if (lineMiddle === undefined) return first; + const beneath = grid.origin + (Math.floor((lineMiddle - grid.origin) / grid.step) + 1) * grid.step; + return Math.max(first, beneath); +} + +/** + * The lift: how far above its rule every line's baseline sits, the way handwriting sits just clear of the line, so + * the rule reads as a line under the words and never touches the bottoms of the letters. It is a share of the rule + * spacing, the same for every size of text on the page, and never under MIN_LIFT. Descenders may reach or cross the + * rule. College ruling (26) lifts its text 3 units. + * + * It is a whole number of page units, which at 100 percent text and zoom is a whole number of device pixels. A + * browser paints a baseline on a whole layout pixel, so a lift between two would round up on some lines and down on + * others. The gap under the letters would then change from line to line. + */ +const LIFT_SHARE = 0.12; +const MIN_LIFT = 2; + +/** The lift for a rule spacing, in page units. */ +export function ruleLift(step: number): number { + return Math.max(MIN_LIFT, Math.round(LIFT_SHARE * step)); +} + +/** + * The custom properties the stylesheet reads. They are on the page's world, so everything inside follows them. The + * stylesheet finds each line's baseline from its own font (text-box-trim), so no font metric is passed. + */ +export function ruleProperties(grid: RuleGrid): Record { + return { + '--rule': `${grid.step}px`, + '--rule-origin': `${grid.origin}px`, + // An unpaginated page is one long lattice, which a sheet height far past any page makes the same arithmetic. + '--rule-sheet': `${grid.sheet ?? 1e9}px`, + '--rule-lift': `${ruleLift(grid.step)}px`, + }; +} + +export const RULE_PROPERTIES = ['--rule', '--rule-origin', '--rule-sheet', '--rule-lift'] as const; diff --git a/app/src/core/wire.ts b/app/src/core/wire.ts index 21614c12..242b3520 100644 --- a/app/src/core/wire.ts +++ b/app/src/core/wire.ts @@ -30,7 +30,15 @@ export interface DecodedEnvelope { /** The JSON part of a frame (crates/core/src/wire/frames.rs, FrameInfo). */ export interface FrameInfo { seq: number; - changes: { pageFields: boolean; blocksChanged: string[]; blocksRemoved: string[]; assetsChanged: string[] }; + changes: { + pageFields: boolean; + blocksChanged: string[]; + blocksRemoved: string[]; + assetsChanged: string[]; + strokesAdded?: string[]; + strokesRemoved?: string[]; + strokesChanged?: string[]; + }; ui: unknown; texts: Record; blocks: Record[]; @@ -41,6 +49,8 @@ export interface FrameInfo { canUndo: boolean; canRedo: boolean; strokes: number; + /** Phase 5: the stroke records after the JSON, the added and changed strokes as they are now. */ + records?: Uint8Array; } const HEADER = 24; @@ -70,5 +80,6 @@ export function decodeEnvelope(buffer: ArrayBuffer): DecodedEnvelope { export function decodeFrame(buffer: ArrayBuffer): FrameInfo { const length = new DataView(buffer).getUint32(0, true); - return JSON.parse(decoder.decode(new Uint8Array(buffer, 4, length))) as FrameInfo; + const info = JSON.parse(decoder.decode(new Uint8Array(buffer, 4, length))) as FrameInfo; + return { ...info, records: new Uint8Array(buffer, 4 + length) }; } diff --git a/app/src/dev/gallery/Gallery.module.css b/app/src/dev/gallery/Gallery.module.css new file mode 100644 index 00000000..1542c95e --- /dev/null +++ b/app/src/dev/gallery/Gallery.module.css @@ -0,0 +1,92 @@ +/* The component gallery page. Every value comes from a design token. */ +@layer components { + .page { + display: grid; + grid-template-rows: auto 1fr; + min-block-size: 100%; + background: var(--color-surface-app); + color: var(--color-text-primary); + } + + .header { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-5); + padding: var(--space-3) var(--space-5); + border-block-end: 1px solid var(--color-border-subtle); + } + + .title { + margin: 0; + font-size: inherit; + font-weight: var(--font-weight-semibold); + } + + .options { + display: flex; + gap: var(--space-5); + } + + .main { + padding: var(--space-5); + } + + .group { + margin: var(--space-5) 0 var(--space-2); + font-size: inherit; + font-weight: var(--font-weight-semibold); + } + + .list { + margin: 0; + padding-inline-start: var(--space-5); + } + + /* Each link is its own target, at least as tall as the density asks. */ + .list li { + display: flex; + flex-wrap: wrap; + align-items: center; + column-gap: var(--space-2); + min-block-size: var(--target-size); + } + + .title a, + .options a { + display: inline-flex; + align-items: center; + min-block-size: var(--target-size); + } + + .description { + color: var(--color-text-secondary); + } + + .empty { + max-inline-size: 60ch; + color: var(--color-text-secondary); + } + + .entry { + max-inline-size: 960px; + } + + .entryTitle { + margin: 0 0 var(--space-2); + font-size: inherit; + font-weight: var(--font-weight-semibold); + } + + .stage { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + gap: var(--space-5); + margin-block-start: var(--space-4); + padding: var(--space-5); + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-md); + background: var(--color-surface-page); + } +} diff --git a/app/src/dev/gallery/Gallery.test.tsx b/app/src/dev/gallery/Gallery.test.tsx new file mode 100644 index 00000000..d6c0c68a --- /dev/null +++ b/app/src/dev/gallery/Gallery.test.tsx @@ -0,0 +1,81 @@ +import { render, screen, within } from '@testing-library/react'; +import { useState } from 'react'; +import { describe, expect, it } from 'vitest'; +import { expectNoAxeViolations } from '../../test'; +import { Button } from '../../ui'; +import { Gallery, galleryHref } from './Gallery'; +import type { GalleryEntry } from './registry'; +import { entries as realEntries } from './entries'; + +function Counter() { + const [count, setCount] = useState(0); + return ; +} + +const entries: GalleryEntry[] = [ + { id: 'ui.button', title: 'Buttons', group: 'Primitives', description: 'The variants.', render: () => }, + { id: 'ui.other', title: 'Other', group: 'Primitives', render: () =>

Other content

}, +]; + +describe('the list', () => { + it('lists each entry under its group as a link to its own page', async () => { + const { container } = render(); + expect(screen.getByRole('heading', { level: 2, name: 'Primitives' })).toBeTruthy(); + const link = screen.getByRole('link', { name: 'Buttons' }); + expect(link.getAttribute('href')).toBe('?entry=ui.button'); + await expectNoAxeViolations(container); + }); + + it('says how to add an entry when there are none', () => { + render(); + expect(screen.getByText(/No entries yet/)).toBeTruthy(); + }); +}); + +describe('one entry', () => { + it('shows the entry alone, in a region a screenshot can target', async () => { + const { container } = render(); + const region = screen.getByRole('region', { name: 'Buttons' }); + expect(region.getAttribute('data-gallery-entry')).toBe('ui.button'); + expect(within(region).getByRole('button', { name: 'Pressed 0 times' })).toBeTruthy(); + expect(screen.queryByText('Other content')).toBeNull(); + await expectNoAxeViolations(container); + }); + + it('lets an entry use state', () => { + render(); + screen.getByRole('button', { name: 'Pressed 0 times' }).click(); + return expect.poll(() => screen.queryByRole('button', { name: 'Pressed 1 times' })).toBeTruthy(); + }); + + it('says so when the entry does not exist', () => { + render(); + expect(screen.getByText(/There is no entry named no\.such/)).toBeTruthy(); + }); +}); + +describe('the appearance links', () => { + it('keep the entry and switch one choice', () => { + render(); + expect(screen.getByRole('link', { name: 'Dark theme' }).getAttribute('href')).toBe('?entry=ui.button&theme=dark'); + expect(screen.getByRole('link', { name: 'Touch density' }).getAttribute('href')).toBe( + '?entry=ui.button&density=touch', + ); + }); + + it('build addresses with only what differs from the defaults', () => { + expect(galleryHref(null, 'light', 'mouse')).toBe('?'); + expect(galleryHref('a.b', 'dark', 'touch')).toBe('?entry=a.b&theme=dark&density=touch'); + }); +}); + +describe('the entries the packages list', () => { + it('have no repeated id, and each renders', () => { + for (const entry of realEntries) { + const { unmount } = render(); + expect(screen.getByRole('region', { name: entry.title })).toBeTruthy(); + unmount(); + } + expect(realEntries.length).toBeGreaterThan(0); + }); +}); diff --git a/app/src/dev/gallery/Gallery.tsx b/app/src/dev/gallery/Gallery.tsx new file mode 100644 index 00000000..eb0d75b7 --- /dev/null +++ b/app/src/dev/gallery/Gallery.tsx @@ -0,0 +1,100 @@ +// The component gallery page: a list of entries, and one entry shown on its own. The page is controlled by the +// address, so a screenshot test opens an entry by its link and gets the same page every time. It uses literal +// text, as app/src/dev may (the lint rule for interface text skips this folder). + +import type { GalleryEntry } from './registry'; +import { groupEntries } from './registry'; +import styles from './Gallery.module.css'; + +export type GalleryTheme = 'light' | 'dark'; +export type GalleryDensity = 'mouse' | 'touch'; + +export interface GalleryProps { + entries: readonly GalleryEntry[]; + /** The id of the entry to show, or null for the list. */ + entryId: string | null; + theme: GalleryTheme; + density: GalleryDensity; +} + +/** The address of an entry, or of the list, in a theme and density. */ +export function galleryHref(entryId: string | null, theme: GalleryTheme, density: GalleryDensity): string { + const params = new URLSearchParams(); + if (entryId) params.set('entry', entryId); + if (theme !== 'light') params.set('theme', theme); + if (density !== 'mouse') params.set('density', density); + const query = params.toString(); + return query ? `?${query}` : '?'; +} + +/** Renders an entry as a component of its own, so an entry may use hooks. */ +function Stage({ entry }: { entry: GalleryEntry }) { + return <>{entry.render()}; +} + +function EntryList({ entries, theme, density }: Omit) { + if (entries.length === 0) { + return ( +

+ No entries yet. A package adds one in app/src/dev/gallery/entries, in a file named for its area and ending in + .gallery.tsx. +

+ ); + } + return ( + <> + {groupEntries(entries).map(({ group, entries: inGroup }) => ( +
+

+ {group} +

+
    + {inGroup.map((entry) => ( +
  • + {entry.title} + {entry.description ? {entry.description} : null} +
  • + ))} +
+
+ ))} + + ); +} + +export function Gallery({ entries, entryId, theme, density }: GalleryProps) { + const entry = entryId ? entries.find((e) => e.id === entryId) : undefined; + return ( +
+
+

+ Component gallery +

+ +
+
+ {entryId && !entry ?

There is no entry named {entryId}.

: null} + {entry ? ( +
+

+ {entry.title} +

+ {entry.description ?

{entry.description}

: null} +
+ +
+
+ ) : ( + !entryId && + )} +
+
+ ); +} diff --git a/app/src/dev/gallery/README.md b/app/src/dev/gallery/README.md new file mode 100644 index 00000000..a9c0604f --- /dev/null +++ b/app/src/dev/gallery/README.md @@ -0,0 +1,65 @@ +# Component gallery + +A page that shows interface components in the states that matter, one address for each, so a screenshot test and an accessibility test can visit every one. It is for development and tests. The shipped app does not include it: the page is built only in test builds (`vite build --mode test`), and the bundle check never sees it. + +## Contents + +- [What it does](#what-it-does) +- [Public API](#public-api) +- [Adding an entry](#adding-an-entry) +- [Running it](#running-it) +- [How the tests use it](#how-the-tests-use-it) +- [What the UI wiring will need](#what-the-ui-wiring-will-need) + +## What it does + +- `gallery.html` loads `main.tsx`, which starts none of the app: no platform, no notes, no settings. It reads the theme, the density, and the entry from the address. +- With no entry in the address it shows the list, grouped by heading. With `?entry=` it shows that one entry, in a frame marked `data-gallery-entry`. +- `window.__OPENNOTE_GALLERY__` lists the entries (`id`, `title`, `group`), so a test can walk them without importing app code. + +## Public API + +| Item | File | Use | +| -------------------------------------------------- | ------------- | -------------------------------------------------------------------------------------------------------------- | +| `defineGallery(entries)` | `registry.ts` | Type-checks the list a file exports. | +| `GalleryEntry` | `registry.ts` | `id`, `title`, `group`, an optional `description`, and `render()`. `render` may call hooks. | +| `collectEntries(modules)`, `groupEntries(entries)` | `registry.ts` | Join the lists of every module, sorted. Throws on an id that repeats or is not lowercase words joined by dots. | +| `Gallery`, `galleryHref` | `Gallery.tsx` | The page, and the address of an entry. | +| `entries` | `entries.ts` | Every entry found in `entries/*.gallery.tsx`. | + +The address takes three optional parameters: `entry` (an id), `theme` (`light` or `dark`), and `density` (`mouse` or `touch`). + +## Adding an entry + +Make a file in `entries/` named for your area and ending in `.gallery.tsx`, such as `entries/search.gallery.tsx`. Its default export is a list from `defineGallery`. Nothing else lists it: `import.meta.glob` finds the file, so two packages never edit the same file. + +```tsx +import { defineGallery } from '../registry'; +import { Button } from '../../../ui'; + +export default defineGallery([ + { + id: 'search.result.empty', + title: 'Search with no results', + group: 'Search', + description: 'The message when nothing matches, and the focus it leaves.', + render: () => , + }, +]); +``` + +An id names a screenshot baseline and a link, so keep it stable. Text in an entry may be literal, because the lint rule for interface text skips `app/src/dev`. + +## Running it + +`npm run app:gallery` opens the page in the development server. The tests start it through the test build and `vite preview`. + +## How the tests use it + +- `tests/ui/a11y/gallery.spec.ts` runs axe on every entry in both themes and both densities. It needs no baselines, so a new entry is checked on the day it is added. +- `tests/ui/visual/gallery.spec.ts` takes a picture of every entry in the same four combinations. An entry with no baseline is skipped and says so, because baselines come only from the `update-screenshots` workflow. +- `Gallery.test.tsx` and `registry.test.ts` check the page and the registry. + +## What the UI wiring will need + +Nothing in the app imports this folder. Each later phase adds its own `entries/.gallery.tsx` for the components it builds, with one entry for each state that looks different (empty, error, disabled, long text, touch density). diff --git a/app/src/dev/gallery/entries.ts b/app/src/dev/gallery/entries.ts new file mode 100644 index 00000000..2c2681cd --- /dev/null +++ b/app/src/dev/gallery/entries.ts @@ -0,0 +1,5 @@ +// Every entry that the packages list in entries/*.gallery.tsx. + +import { collectEntries } from './registry'; + +export const entries = collectEntries(import.meta.glob('./entries/*.gallery.tsx', { eager: true })); diff --git a/app/src/dev/gallery/entries/later.gallery.tsx b/app/src/dev/gallery/entries/later.gallery.tsx new file mode 100644 index 00000000..80244e32 --- /dev/null +++ b/app/src/dev/gallery/entries/later.gallery.tsx @@ -0,0 +1,145 @@ +// The components of Phases 7 to 13 and the quality-of-life features that take plain props, in their states (A4-37): +// the connector card, the flashcard face, and the cloud folder notice. They get pictures and an axe pass from the day +// they are listed. The screens that need the app around them are in tests/ui/screens/later.ts. + +import { ConnectorCard } from '../../../features/connectors'; +import type { ConnectorInfo } from '../../../features/connectors'; +import { CloudNotice } from '../../../features/qol'; +import { CardFace } from '../../../features/study'; +import { defineGallery } from '../registry'; + +const microsoft: ConnectorInfo = { + id: 'microsoft', + name: 'Microsoft', + group: 'microsoft', + auth: 'oauth', + features: ['importOneNote', 'outlookCalendar', 'microsoftToDo'], + access: [ + { capability: 'account', writes: false }, + { capability: 'onenoteRead', writes: false }, + { capability: 'tasksWrite', writes: true }, + ], + hosts: ['login.microsoftonline.com', 'graph.microsoft.com'], + state: { kind: 'notConnected' }, + baseUrl: null, + pending: false, + lastUsedUnix: null, +}; + +const none = () => Promise.resolve(false); +const noop = () => undefined; + +function Card(props: { info: ConnectorInfo; offline?: boolean }) { + return ( + + ); +} + +/** The cards are list items, so they sit in a list that adds no look of its own. */ +const LIST = { display: 'contents' } as const; + +export default defineGallery([ + { + id: 'connectors.card.states', + title: 'Connector cards', + group: 'Connectors', + description: 'Not connected, needing setup, connected, and needing to connect again.', + render: () => ( +
    + + + + +
+ ), + }, + { + id: 'connectors.card.offline', + title: 'Connector card, Work offline', + group: 'Connectors', + description: 'Connect waits while Work offline is on, and the card says why.', + render: () => ( + <> + +
    + +
+ + ), + }, + { + id: 'study.cardface.states', + title: 'Flashcard faces', + group: 'Study', + description: 'A basic card, a cloze card, and a multiple choice card, each with its answer hidden and shown.', + render: () => ( + <> + + + + + + ), + }, + { + id: 'qol.cloudnotice.states', + title: 'Cloud folder notice', + group: 'Storage', + description: + 'The notice for a notebook in a synced folder, where the app can keep it on the device and where it cannot.', + render: () => ( + <> + + + + ), + }, +]); diff --git a/app/src/dev/gallery/entries/ui.gallery.tsx b/app/src/dev/gallery/entries/ui.gallery.tsx new file mode 100644 index 00000000..cbb043a3 --- /dev/null +++ b/app/src/dev/gallery/entries/ui.gallery.tsx @@ -0,0 +1,62 @@ +// The interface primitives in the states that matter for their look and for accessibility. Later packages add their +// own files beside this one, one per area, and never edit this one. + +import { useState } from 'react'; +import { Button, Switch, TextField } from '../../../ui'; +import { defineGallery } from '../registry'; + +function Fields() { + const [name, setName] = useState('Biology 101'); + return ( + <> + + {}} error="A section needs a name." /> + + ); +} + +function Switches() { + const [on, setOn] = useState(true); + const [off, setOff] = useState(false); + return ( + <> + + + {}} disabled="aria" /> + + ); +} + +export default defineGallery([ + { + id: 'ui.button.variants', + title: 'Buttons', + group: 'Primitives', + description: 'The four variants, enabled and disabled.', + render: () => ( + <> + + + + + + + ), + }, + { + id: 'ui.switch.states', + title: 'Switches', + group: 'Primitives', + description: 'On, off, and disabled with an explanation.', + render: () => , + }, + { + id: 'ui.textfield.states', + title: 'Text fields', + group: 'Primitives', + description: 'With help text, and with an error.', + render: () => , + }, +]); diff --git a/app/src/dev/gallery/main.tsx b/app/src/dev/gallery/main.tsx new file mode 100644 index 00000000..45667a23 --- /dev/null +++ b/app/src/dev/gallery/main.tsx @@ -0,0 +1,32 @@ +// The component gallery's entry point (gallery.html). It starts nothing of the app: no platform, no notes, no +// settings. It reads the theme, the density, and the entry from the address, so each picture is one address. +// The test build includes it. The production build does not. + +import '../../styles/layers.css'; +import '../../theme/fonts'; +import '../../theme/tokens.css'; +import '../../styles/base.css'; +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; +import { setDensity } from '../../state/layout'; +import { Announcer } from '../../ui'; +import { Gallery } from './Gallery'; +import type { GalleryDensity, GalleryTheme } from './Gallery'; +import { entries } from './entries'; + +const root = document.getElementById('root'); +if (!root) throw new Error('Missing #root element'); + +const params = new URLSearchParams(window.location.search); +const theme: GalleryTheme = params.get('theme') === 'dark' ? 'dark' : 'light'; +const density: GalleryDensity = params.get('density') === 'touch' ? 'touch' : 'mouse'; +document.documentElement.setAttribute('data-theme', theme); +setDensity(density); +window.__OPENNOTE_GALLERY__ = entries.map(({ id, title, group }) => ({ id, title, group })); + +createRoot(root).render( + + + + , +); diff --git a/app/src/dev/gallery/registry.test.ts b/app/src/dev/gallery/registry.test.ts new file mode 100644 index 00000000..9dfe64c7 --- /dev/null +++ b/app/src/dev/gallery/registry.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; +import { collectEntries, defineGallery, groupEntries } from './registry'; +import type { GalleryEntry } from './registry'; + +const entry = (id: string, group: string, title: string): GalleryEntry => ({ id, group, title, render: () => null }); + +describe('collecting entries', () => { + it('joins every module and sorts by group, then title', () => { + const entries = collectEntries({ + './entries/b.gallery.tsx': { default: defineGallery([entry('b.one', 'Menus', 'Context menu')]) }, + './entries/a.gallery.tsx': { + default: defineGallery([entry('a.two', 'Buttons', 'Quiet'), entry('a.one', 'Buttons', 'Primary')]), + }, + }); + expect(entries.map((e) => e.id)).toEqual(['a.one', 'a.two', 'b.one']); + }); + + it('is empty with no modules, which is how a new package starts', () => { + expect(collectEntries({})).toEqual([]); + }); + + it('refuses two entries with the same id, since the id names a picture and a link', () => { + expect(() => + collectEntries({ + './entries/a.gallery.tsx': { default: [entry('same.id', 'A', 'One')] }, + './entries/b.gallery.tsx': { default: [entry('same.id', 'B', 'Two')] }, + }), + ).toThrow(/unique: same\.id/); + }); + + it('refuses an id that is not lowercase words joined by dots', () => { + for (const bad of ['Upper.case', 'has space', 'trailing.', '.leading', 'a..b', '1starts']) { + expect(() => collectEntries({ './x.gallery.tsx': { default: [entry(bad, 'A', 'One')] } })).toThrow( + /lowercase words/, + ); + } + }); +}); + +describe('grouping', () => { + it('lists the entries under their group, in order', () => { + const entries = collectEntries({ + './x.gallery.tsx': { + default: [entry('a.one', 'Buttons', 'B'), entry('a.two', 'Buttons', 'A'), entry('c.one', 'Fields', 'C')], + }, + }); + expect(groupEntries(entries).map((g) => [g.group, g.entries.map((e) => e.title)])).toEqual([ + ['Buttons', ['A', 'B']], + ['Fields', ['C']], + ]); + }); +}); diff --git a/app/src/dev/gallery/registry.ts b/app/src/dev/gallery/registry.ts new file mode 100644 index 00000000..09f08e50 --- /dev/null +++ b/app/src/dev/gallery/registry.ts @@ -0,0 +1,54 @@ +// The component gallery's registry. Each package lists the components and states it wants shown in a file named +// entries/.gallery.tsx, whose default export is a list from defineGallery. collectEntries finds them all, so +// no package edits a shared list. The gallery is for development and for screenshot tests: it is not part of the +// shipped app (see README.md). + +import type { ReactNode } from 'react'; + +/** One component in one state. */ +export interface GalleryEntry { + /** Unique across every file, lowercase words joined by dots, such as 'ui.button.primary'. */ + readonly id: string; + readonly title: string; + /** The heading it is listed under, such as 'Buttons'. */ + readonly group: string; + /** One sentence for the reader: what the state shows and why it matters. */ + readonly description?: string; + /** The entry's content. It may call hooks, because the gallery renders it as a component. */ + render(): ReactNode; +} + +export const ID_PATTERN = /^[a-z][a-z0-9]*(\.[a-z][a-z0-9]*)*$/; + +/** Type-checks a list of entries. */ +export function defineGallery(entries: readonly GalleryEntry[]): readonly GalleryEntry[] { + return entries; +} + +type Module = { default: readonly GalleryEntry[] }; + +/** + * Joins the lists of every module, sorted by group and then title. It throws when two entries share an id, or an + * id is not lowercase words joined by dots, because the id names a screenshot baseline and a link. + */ +export function collectEntries(modules: Readonly>): GalleryEntry[] { + const entries = Object.values(modules).flatMap((module) => [...module.default]); + const bad = entries.filter((entry) => !ID_PATTERN.test(entry.id)); + if (bad.length > 0) + throw new Error(`Gallery ids must be lowercase words joined by dots: ${bad.map((e) => e.id).join(', ')}`); + const ids = entries.map((entry) => entry.id); + const repeated = ids.filter((id, i) => ids.indexOf(id) !== i); + if (repeated.length > 0) throw new Error(`Gallery ids must be unique: ${[...new Set(repeated)].join(', ')}`); + return entries.sort((a, b) => a.group.localeCompare(b.group) || a.title.localeCompare(b.title)); +} + +/** The entries by group, in order. */ +export function groupEntries(entries: readonly GalleryEntry[]): { group: string; entries: GalleryEntry[] }[] { + const groups: { group: string; entries: GalleryEntry[] }[] = []; + for (const entry of entries) { + const last = groups.at(-1); + if (last?.group === entry.group) last.entries.push(entry); + else groups.push({ group: entry.group, entries: [entry] }); + } + return groups; +} diff --git a/app/src/editor/commands/fold.ts b/app/src/editor/commands/fold.ts index 2f7a254c..26a9146b 100644 --- a/app/src/editor/commands/fold.ts +++ b/app/src/editor/commands/fold.ts @@ -152,6 +152,22 @@ export function unfoldTo(editor: Editor, pos: number): boolean { return true; } +/** + * Unfolds whatever hides a DOM node, in the editor that shows it. For code that reveals something by its element: a + * link, a search result, a spoken paragraph. Returns whether anything unfolded; a node outside an editor does nothing. + */ +export function unfoldAroundDom(node: Node): boolean { + const element = node instanceof Element ? node : node.parentElement; + const root = element?.closest('.ProseMirror') as (HTMLElement & { editor?: Editor }) | null | undefined; + const editor = root?.editor; + if (!editor || editor.isDestroyed || !root?.contains(node)) return false; + try { + return unfoldTo(editor, editor.view.posAtDOM(node, 0)); + } catch { + return false; + } +} + /** The foldable heading or list item the selection is in: the innermost item with children, else its heading. */ export function foldTargetAt(state: EditorState): { pos: number; kind: FoldKind } | null { const { $from } = state.selection; diff --git a/app/src/editor/commands/tables.ts b/app/src/editor/commands/tables.ts index 76853bb8..8318c7a5 100644 --- a/app/src/editor/commands/tables.ts +++ b/app/src/editor/commands/tables.ts @@ -76,6 +76,25 @@ export function canApply(data: TableData, at: CellAt, op: TableOp): boolean { } /** The data after `op`, or null when it doesn't apply (moving the top row up, deleting the last column). */ +/** The narrowest a new column is made to keep a table inside its page. */ +const MIN_FITTED_COLUMN = 80; + +/** + * A column just added, narrowed so a table that fitted its page still does: four columns of 160 px in a 623 px page + * would hide the last column's right-aligned digits behind the scroll (Beta 5 walk W2-13). A table already wider than + * its page, or with too little room left for a usable column, keeps the width the column got. + */ +export function fitAddedColumn(before: TableData, change: TableChange, room: number): TableChange { + const total = (data: TableData) => data.columns.reduce((sum, column) => sum + column.width, 0); + const left = room - total(before); + if (room <= 0 || total(change.data) <= room || left < 0 || left < MIN_FITTED_COLUMN) return change; + const index = change.caret?.column ?? -1; + const columns = change.data.columns.map((column, i) => + i === index ? { ...column, width: clampColumnWidth(Math.floor(left)) } : column, + ); + return { ...change, data: { ...change.data, columns } }; +} + export function applyTableOp(input: TableData, at: CellAt, op: TableOp): TableChange | null { const data = normalizeTableData(input); const { row, column } = clampCell(data, at); diff --git a/app/src/editor/extensions/autocorrect.test.ts b/app/src/editor/extensions/autocorrect.test.ts index 144847d0..51f9830d 100644 --- a/app/src/editor/extensions/autocorrect.test.ts +++ b/app/src/editor/extensions/autocorrect.test.ts @@ -44,6 +44,12 @@ describe('AutoCorrect', () => { expect(auto.filter(Boolean)).toHaveLength(3); }); + it('leaves "-->" and "---" alone so diagram source survives', () => { + const page = mount('[]'); + typeInto(page.editor, 'A[Start] --> B[Finish] --- C '); + expect(page.editor.state.doc.textContent).toBe('A[Start] --> B[Finish] --- C '); + }); + it('reverts on Backspace right after', () => { const page = mount('[]'); typeInto(page.editor, 'teh '); diff --git a/app/src/editor/extensions/autocorrect.ts b/app/src/editor/extensions/autocorrect.ts index 80af4962..48d4338c 100644 --- a/app/src/editor/extensions/autocorrect.ts +++ b/app/src/editor/extensions/autocorrect.ts @@ -104,6 +104,8 @@ function correctionAt(state: EditorState, pos: number, list: readonly Replacemen const piece = token.text.slice(start); const to = correctionFor(piece, list); if (to === null || (start > 0 && /\w/.test(token.text[start - 1]) && /^\w/.test(piece))) continue; + // Arrows and dashes extend what was typed before them: "-->" and "---" stay, not "-→" and "-–". + if (start > 0 && /^[-=<>]/.test(piece) && /[-=<>]/.test(token.text[start - 1])) continue; const from = token.from + start; const tr = state.tr.insertText(to, from, pos); return { tr, meta: { from, to: pos, text: to } }; diff --git a/app/src/editor/extensions/content.module.css b/app/src/editor/extensions/content.module.css index d8761806..ed691cde 100644 --- a/app/src/editor/extensions/content.module.css +++ b/app/src/editor/extensions/content.module.css @@ -32,6 +32,13 @@ outline-offset: var(--size-focus-ring-offset); } + /* On ruled paper the line's baseline is one line less the lift below the item's top (blocks.module.css): the box is centered on + the capitals above it. Its own line height is 0 there, so it is sized in ems. */ + :global([data-ruled]) .taskBox { + inset-block-start: calc(var(--rule-line) - var(--rule-lift) - 0.5cap - 0.5em); + block-size: 1em; + } + .taskBody > p { margin-block: var(--space-1); } diff --git a/app/src/editor/extensions/crossBlock.ts b/app/src/editor/extensions/crossBlock.ts index 2b5f68f4..898a035f 100644 --- a/app/src/editor/extensions/crossBlock.ts +++ b/app/src/editor/extensions/crossBlock.ts @@ -35,7 +35,9 @@ const wrapperOf = (view: EditorView): HTMLElement | null => view.dom.closest(); + const note = (pos: number) => + foldsHiding(state, Math.min(Math.max(pos, 0), state.doc.content.size)).forEach((f) => hiding.add(f)); + for (const tr of transactions) { + if (!tr.docChanged || !tr.getMeta(META_REMOTE)) continue; + tr.mapping.maps.forEach((map, index) => { + map.forEach((_from, _to, start, end) => { + const rest = tr.mapping.slice(index + 1); + note(rest.map(start, 1)); + if (end > start) note(rest.map(end, -1) - 1); + }); + }); + note(state.selection.head); + } + if (hiding.size === 0) return null; + return setFolds( + state.tr, + foldsOf(state).filter((fold) => !hiding.has(fold)), + ); +} + function foldingPlugin(host: EditorHost): Plugin { return new Plugin({ key: foldKey as never, + appendTransaction: (transactions, _old, state) => openFoldsAfterUndo(transactions, state), state: { init: (_config, state) => { rememberFolds(state.doc, []); diff --git a/app/src/editor/extensions/graphFence.ts b/app/src/editor/extensions/graphFence.ts new file mode 100644 index 00000000..8d413035 --- /dev/null +++ b/app/src/editor/extensions/graphFence.ts @@ -0,0 +1,154 @@ +// Graphs in text (Phase 10). A code block whose language is "graph" holds the graph's functions as text. This keeps +// a live graph after each one, drawn by the page's renderer (EditorHost.graph), which loads on first use. The graph +// edits the block's text and nothing else, so undo, copy, search, and export treat it as the code it is. +import { Extension } from '@tiptap/core'; +import type { Editor, Extensions } from '@tiptap/core'; +import type { Node as PMNode } from '@tiptap/pm/model'; +import { Plugin, PluginKey } from '@tiptap/pm/state'; +import type { EditorState } from '@tiptap/pm/state'; +import { Decoration, DecorationSet } from '@tiptap/pm/view'; +import type { EditorView } from '@tiptap/pm/view'; +import type { EditorHost, GraphHandle, GraphRenderer } from '../host'; +import { META_COMMAND } from '../meta'; + +export const GRAPH_LANGUAGE = 'graph'; +const key = new PluginKey('opennote.graphFence'); + +const isGraph = (node: PMNode) => node.type.name === 'codeBlock' && node.attrs.language === GRAPH_LANGUAGE; + +/** The position and node of each graph code block, in order. */ +function graphs(doc: PMNode): { pos: number; node: PMNode }[] { + const found: { pos: number; node: PMNode }[] = []; + doc.forEach((node, offset) => { + if (isGraph(node)) found.push({ pos: offset, node }); + }); + return found; +} + +/** Replaces the text of the code block at `pos` as one command. */ +function setSource(view: EditorView, index: number, next: string): void { + const target = graphs(view.state.doc)[index]; + if (!target) return; + const from = target.pos + 1; + const to = target.pos + target.node.nodeSize - 1; + const text = next === '' ? [] : [view.state.schema.text(next)]; + view.dispatch(view.state.tr.replaceWith(from, to, text).setMeta(META_COMMAND, true)); +} + +class Widgets { + private renderer: GraphRenderer | null = null; + private handles = new Map(); + private containers = new Map(); + + constructor( + private view: EditorView, + host: EditorHost, + ) { + host.graph?.()?.then( + (renderer) => { + this.renderer = renderer; + this.sync(this.view.state); + }, + () => undefined, + ); + } + + container(index: number): HTMLElement { + let element = this.containers.get(index); + if (!element) { + element = document.createElement('div'); + element.setAttribute('data-graph', ''); + element.contentEditable = 'false'; + this.containers.set(index, element); + } + return element; + } + + sync(state: EditorState): void { + if (!this.renderer) return; + const present = graphs(state.doc); + present.forEach(({ node }, index) => { + const props = { + source: node.textContent, + editable: this.view.editable, + onSource: (next: string) => setSource(this.view, index, next), + }; + const existing = this.handles.get(index); + if (existing) existing.update(props); + else this.handles.set(index, this.renderer!.mount(this.container(index), props)); + }); + for (const [index, handle] of [...this.handles]) { + if (index >= present.length) { + handle.destroy(); + this.handles.delete(index); + this.containers.delete(index); + } + } + } + + destroy(): void { + for (const handle of this.handles.values()) handle.destroy(); + this.handles.clear(); + } +} + +function plugin(host: EditorHost): Plugin { + const widgets = new WeakMap(); + return new Plugin({ + key, + view(view) { + const mine = new Widgets(view, host); + widgets.set(view, mine); + return { + update: (updated) => mine.sync(updated.state), + destroy: () => mine.destroy(), + }; + }, + props: { + decorations(state) { + const found = graphs(state.doc); + if (found.length === 0) return DecorationSet.empty; + return DecorationSet.create( + state.doc, + found.map(({ pos, node }, index) => + Decoration.widget(pos + node.nodeSize, (view) => widgets.get(view)!.container(index), { + side: -1, + key: `graph-${index}`, + stopEvent: () => true, + ignoreSelection: true, + }), + ), + ); + }, + }, + }); +} + +/** The graph extension, for editors that hold text. */ +export function graphFenceExtensions(host: EditorHost): Extensions { + return [ + Extension.create({ + name: 'graphFence', + addProseMirrorPlugins() { + return host.graph ? [plugin(host)] : []; + }, + }), + ]; +} + +/** Puts a graph code block at the caret and returns its position, or null when this editor has no code blocks. */ +export function insertGraph(editor: Editor, source = 'y = sin(x)'): number | null { + const type = editor.schema.nodes.codeBlock; + if (!type || !editor.isEditable) return null; + const node = type.create({ language: GRAPH_LANGUAGE }, editor.schema.text(source)); + const tr = editor.state.tr.replaceSelectionWith(node, false).setMeta(META_COMMAND, true); + let end: number | null = null; + tr.mapping.maps[tr.steps.length - 1]?.forEach((_from, _to, _newFrom, newTo) => { + end ??= newTo; + }); + const pos = (end ?? tr.selection.to) - node.nodeSize; + if (tr.doc.nodeAt(pos)?.type !== type) return null; + editor.view.dispatch(tr); + editor.view.focus(); + return pos; +} diff --git a/app/src/editor/extensions/kit.ts b/app/src/editor/extensions/kit.ts index cf4cc069..629ae026 100644 --- a/app/src/editor/extensions/kit.ts +++ b/app/src/editor/extensions/kit.ts @@ -23,6 +23,7 @@ import { highlightExtensions } from './highlight'; import { imageExtensions } from './image'; import { keysExtensions } from './keys'; import { listItemExtensions } from './listItem'; +import { graphFenceExtensions } from './graphFence'; import { mathBlockExtensions, mathInlineExtensions } from './math'; import { outlineExtensions } from './outline'; import { slashExtensions } from './slash'; @@ -94,6 +95,7 @@ function textKit(host: EditorHost): Extensions { ...codeBlockExtensions(host), ...mathBlockExtensions(host), ...mathInlineExtensions(host), + ...graphFenceExtensions(host), ...imageExtensions(host), ...markKit(host), ...behaviorKit(host), diff --git a/app/src/editor/extensions/math.ts b/app/src/editor/extensions/math.ts index 266d1944..58bf5ef5 100644 --- a/app/src/editor/extensions/math.ts +++ b/app/src/editor/extensions/math.ts @@ -1,67 +1,327 @@ -// Math kept as source until Phase 10 renders it (ARCHITECTURE.md section 8.2; owner: WP4). Each atom shows its -// source as written, `$...$` inline and `$$` fences for display math, and opens a small source field on Enter or -// a double-click. Enter or leaving the field saves it as one undo step, Shift+Enter starts a new line of display -// math, and Escape puts the old source back. -import type { Editor, Extensions } from '@tiptap/core'; +// Math in text (ARCHITECTURE.md section 8.2; owners: WP4, then the Phase 10 lane). Each atom shows its source as +// written, `$...$` inline and `$$` fences for display math, until the page's host can draw LaTeX. Then it shows the +// drawn equation, with MathML behind it for screen readers. Enter or a double-click opens a source field with a +// live preview and the LaTeX problem, if any. Enter or leaving the field saves it as one undo step, Shift+Enter +// starts a new line of display math, and Escape puts the old source back. +import { InputRule, type Editor, type Extensions } from '@tiptap/core'; import type { Node as PMNode } from '@tiptap/pm/model'; import { NodeSelection, TextSelection } from '@tiptap/pm/state'; import type { NodeView } from '@tiptap/pm/view'; import { t } from '../../strings/t'; -import type { EditorHost } from '../host'; +import type { EditorHost, MathActionResult, MathRenderer, VisualMathField } from '../host'; import { META_COMMAND } from '../meta'; +import { createPenPad } from './mathPen'; +import type { PenPad } from './mathPen'; import { MathBlock, MathInline } from '../schema/nodes'; import styles from './content.module.css'; +import drawn from './mathDrawn.module.css'; const views = new WeakMap(); +const MODE_KEY = 'opennote.math.visual'; + +/** Whether the person last edited equations visually. A choice kept in the browser, and visual until they pick LaTeX. */ +function prefersVisual(): boolean { + try { + return localStorage.getItem(MODE_KEY) !== 'latex'; + } catch { + return true; + } +} + +function rememberMode(visual: boolean): void { + try { + localStorage.setItem(MODE_KEY, visual ? 'visual' : 'latex'); + } catch { + // The choice lasts until the page is closed. + } +} + +type Field = HTMLInputElement | HTMLTextAreaElement | (HTMLElement & { value: string }); class MathView implements NodeView { dom: HTMLElement; private shown: HTMLElement; - private field: HTMLInputElement | HTMLTextAreaElement | null = null; + private field: Field | null = null; + private visual: VisualMathField | null = null; + private toggle: HTMLButtonElement | null = null; + private penButton: HTMLButtonElement | null = null; + private pad: PenPad | null = null; + private swapping = false; + /** Handwriting is being read, so the field keeps its place while a dialog may take focus. */ + private recognizing = false; + private preview: HTMLElement | null = null; + private problem: HTMLElement | null = null; + private holder: HTMLElement | null = null; + private renderer: MathRenderer | null = null; + private gone = false; constructor( private node: PMNode, private editor: Editor, private getPos: () => number | undefined, private display: boolean, + private host: EditorHost, ) { this.dom = document.createElement(display ? 'div' : 'span'); this.dom.className = display ? styles.mathBlock : styles.mathInline; this.dom.setAttribute(display ? 'data-math-block' : 'data-math', ''); - this.shown = document.createElement('code'); + this.shown = document.createElement('span'); this.dom.append(this.shown); this.dom.addEventListener('dblclick', () => this.open()); views.set(this.dom, this); this.render(); + host.math?.()?.then( + (renderer) => { + if (this.gone) return; + this.renderer = renderer; + this.render(); + if (this.field && !this.preview) this.addPreview(this.field); + }, + () => undefined, + ); } private source(): string { return this.node.attrs.source as string; } + /** The drawing of some source, or null when the renderer isn't loaded or there is nothing to draw. */ + private draw(source: string) { + return this.renderer && source.trim() !== '' ? this.renderer.render(source, this.display) : null; + } + private render(): void { const source = this.source(); - this.shown.textContent = this.display ? `$$\n${source}\n$$` : `$${source}$`; + const drawing = this.draw(source); this.dom.setAttribute(this.display ? 'data-math-block' : 'data-math', source); this.dom.setAttribute('aria-label', t(this.display ? 'editor.math.block' : 'editor.math.inline', { source })); + this.dom.classList.toggle(drawn.invalid, drawing !== null && !drawing.ok); + if (drawing && !drawing.ok) this.dom.title = drawing.message; + else this.dom.removeAttribute('title'); + if (drawing?.ok) { + this.dom.setAttribute('role', 'math'); + this.shown.className = drawn.drawn; + this.shown.innerHTML = drawing.html; + return; + } + this.dom.removeAttribute('role'); + this.shown.className = ''; + this.shown.textContent = this.display ? `$$\n${source}\n$$` : `$${source}$`; } /** Opens the source field. */ open(): void { if (this.field || !this.editor.isEditable) return; - const field = document.createElement(this.display ? 'textarea' : 'input'); - field.className = styles.mathField; - field.value = this.source(); - field.setAttribute('aria-label', t('editor.math.field')); - field.addEventListener('keydown', (event) => this.onKey(event as KeyboardEvent)); - field.addEventListener('blur', () => this.close(true)); + const field = this.plainField(this.source()); this.field = field; + // The paragraph stops clipping its drawing while the equation is open, so the preview and actions that float under + // the equation are seen and can be pressed (blocks.module.css). + this.dom.setAttribute('data-math-editing', ''); this.shown.hidden = true; this.dom.append(field); + if (this.renderer) this.addPreview(field); field.focus(); field.select(); } + private plainField(value: string): HTMLInputElement | HTMLTextAreaElement { + const field = document.createElement(this.display ? 'textarea' : 'input'); + field.className = styles.mathField; + field.value = value; + field.setAttribute('aria-label', t('editor.math.field')); + this.wire(field); + return field; + } + + private wire(field: Field): void { + field.addEventListener('keydown', (event) => this.onKey(event as KeyboardEvent)); + field.addEventListener('input', () => this.showPreview()); + // Tab moves on to Simplify and Solve in the panel under the field; leaving the equation entirely saves it. + field.addEventListener('blur', (event) => { + if (this.swapping || this.recognizing) return; + if (!this.holder?.contains((event as FocusEvent).relatedTarget as Node | null)) this.close(true); + }); + } + + /** Swaps the open field for the visual editor or back, keeping what was typed. */ + private async swap(toVisual: boolean): Promise { + const old = this.field; + const make = this.renderer?.visual; + if (!old) return; + const value = this.visual ? this.visual.value : old.value; + let next: Field; + let visual: VisualMathField | null = null; + if (toVisual) { + if (!make) return; + try { + visual = await make.create({ value, label: t('editor.math.visualField'), display: this.display }); + } catch { + this.host.announce(t('editor.math.visualFailed')); + return; + } + if (this.field !== old) return; + next = visual.element as Field; + next.classList.add(styles.mathField); + this.wire(next); + } else { + next = this.plainField(value); + } + // The old field leaves without saving, because the new one holds the same equation. + this.swapping = true; + old.replaceWith(next); + this.swapping = false; + this.field = next; + this.visual = visual; + rememberMode(toVisual); + this.updateToggle(); + this.showPreview(); + next.focus(); + if (next instanceof HTMLInputElement || next instanceof HTMLTextAreaElement) next.select(); + } + + /** Opens the writing area under the equation. */ + private openPad(holder: HTMLElement): void { + const recognize = this.renderer?.pen?.recognize; + if (!recognize || this.pad || !this.visual) return; + const pad = createPenPad(() => { + const strokes = pad.strokes(); + const visual = this.visual; + if (strokes.length === 0 || !visual) return; + // Reading handwriting may ask the person to turn it on first, and that dialog takes focus from the field. The + // field stays open meanwhile, or the equation would close and save before the answer came back. + this.recognizing = true; + void recognize(strokes) + .finally(() => { + this.recognizing = false; + }) + .then((latex) => { + if (!latex || !this.visual) { + this.host.announce(t('intelMore.math.penNone')); + return; + } + visual.insert(latex); + pad.clear(); + this.showPreview(); + }); + }); + this.pad = pad; + holder.classList.add(drawn.holderPadOpen); + holder.append(pad.element); + } + + private closePad(): void { + this.pad?.element.parentElement?.classList.remove(drawn.holderPadOpen); + this.pad?.element.remove(); + this.pad = null; + } + + private updateToggle(): void { + if (this.toggle) this.toggle.textContent = t(this.visual ? 'editor.math.useLatex' : 'editor.math.useVisual'); + if (this.penButton) this.penButton.hidden = !this.visual; + if (!this.visual) this.closePad(); + } + + private addPreview(field: HTMLElement): void { + // The preview, the problem, and the actions sit together: under the field for display math, and floating + // under it for an inline equation, so nothing in them is covered by the others. + const holder = document.createElement('div'); + holder.className = this.display ? drawn.holderBlock : drawn.holderInline; + // The paragraph's paint containment would clip the floating panel; blocks.module.css lifts it while this is open. + holder.setAttribute('data-math-holder', ''); + holder.addEventListener('focusout', (event) => { + const to = event.relatedTarget as Node | null; + if (!holder.contains(to) && to !== this.field) this.close(true); + }); + const preview = document.createElement('div'); + preview.className = drawn.preview; + const problem = document.createElement('div'); + problem.className = drawn.problem; + problem.setAttribute('role', 'status'); + holder.append(preview, problem); + if (this.renderer?.visual) { + const toggle = document.createElement('button'); + toggle.type = 'button'; + toggle.className = drawn.toggle; + // A press must not take focus from the field, or the field would close and save. + toggle.addEventListener('mousedown', (event) => event.preventDefault()); + toggle.addEventListener('click', () => void this.swap(!this.visual)); + this.toggle = toggle; + holder.append(toggle); + if (this.renderer.pen) { + const pen = document.createElement('button'); + pen.type = 'button'; + pen.className = drawn.toggle; + pen.textContent = t('intelMore.math.penWrite'); + pen.addEventListener('mousedown', (event) => event.preventDefault()); + pen.addEventListener('click', () => (this.pad ? this.closePad() : this.openPad(holder))); + this.penButton = pen; + holder.append(pen); + } + this.updateToggle(); + } + const actions = this.renderer?.actions; + if (actions) { + const bar = document.createElement('div'); + bar.className = drawn.actions; + bar.setAttribute('role', 'group'); + bar.setAttribute('aria-label', t('smart.math.actions')); + for (const kind of ['simplify', 'solve'] as const) { + const button = document.createElement('button'); + button.type = 'button'; + button.textContent = t(`smart.math.${kind}`); + // A press must not take focus from the field, or the field would close and save. + button.addEventListener('mousedown', (event) => event.preventDefault()); + button.addEventListener('click', () => this.act(actions[kind])); + bar.append(button); + } + holder.append(bar); + } + this.preview = preview; + this.problem = problem; + this.holder = holder; + field.after(holder); + this.showPreview(); + if (this.renderer?.visual && prefersVisual() && !this.visual) void this.swap(true); + } + + private showPreview(): void { + const { field, preview, problem } = this; + if (!field || !preview || !problem) return; + const drawing = this.draw(this.visual ? this.visual.value : field.value); + field.setAttribute('aria-invalid', String(drawing !== null && !drawing.ok)); + preview.hidden = !drawing?.ok; + if (drawing?.ok) preview.innerHTML = drawing.html; + problem.textContent = drawing && !drawing.ok ? t('smart.math.problem', { message: drawing.message }) : ''; + } + + /** True while this equation is still on the page of that editor. */ + usableIn(editor: Editor): boolean { + return !this.gone && this.editor === editor && this.dom.isConnected; + } + + /** Opens the field if needed and runs Simplify or Solve on it, as the buttons in the panel do. */ + runAction(kind: 'simplify' | 'solve'): boolean { + this.open(); + const actions = this.renderer?.actions; + if (!this.field || !actions) return false; + this.act(actions[kind]); + return true; + } + + /** Runs Simplify or Solve on what is in the field, and says what happened. */ + private act(run: (latex: string) => MathActionResult): void { + const field = this.field; + if (!field) return; + const result = run(this.visual ? this.visual.value : field.value); + if (result.ok) { + if (this.visual) this.visual.value = result.latex; + else field.value = result.latex; + this.showPreview(); + this.host.announce(t('smart.math.done')); + } else this.host.announce(t(`smart.math.${result.reason}`)); + field.focus(); + } + private onKey(event: KeyboardEvent): void { if (event.key === 'Escape') { event.preventDefault(); @@ -77,8 +337,18 @@ class MathView implements NodeView { const field = this.field; if (!field) return; this.field = null; - const source = field.value; + this.dom.removeAttribute('data-math-editing'); + const source = this.visual ? this.visual.value : field.value; + rememberClosed(this); + if (save && this.visual && source.trim() !== '') + this.host.announce(t('editor.math.spoken', { text: this.visual.spoken() })); + this.closePad(); + this.visual = null; + this.toggle = null; + this.penButton = null; field.remove(); + this.holder?.remove(); + this.preview = this.problem = this.holder = null; this.shown.hidden = false; const pos = this.getPos(); if (pos === undefined) return; @@ -100,7 +370,11 @@ class MathView implements NodeView { } stopEvent(event: Event): boolean { - return this.field !== null && event.target === this.field; + if (this.field === null) return false; + // Keys and focus inside the field and its panel (Tab to Simplify and Solve, writing in the pen pad) are not the + // editor's: it would take focus and close the equation. + const target = event.target instanceof Node ? event.target : null; + return target === this.field || Boolean(target && this.holder?.contains(target)); } ignoreMutation(): boolean { @@ -108,6 +382,7 @@ class MathView implements NodeView { } destroy(): void { + this.gone = true; views.delete(this.dom); } } @@ -116,36 +391,98 @@ class MathView implements NodeView { function openSelected(editor: Editor): boolean { const { selection } = editor.state; if (!(selection instanceof NodeSelection) || !selection.node.type.name.startsWith('math')) return false; - const dom = editor.view.nodeDOM(selection.from); + return openMathAt(editor, selection.from); +} + +/** Opens the source field of the math atom at `pos`. False when there is none. */ +export function openMathAt(editor: Editor, pos: number): boolean { + const dom = editor.view.nodeDOM(pos); const view = dom instanceof Element ? views.get(dom) : undefined; view?.open(); return Boolean(view); } -export function mathBlockExtensions(_host: EditorHost): Extensions { +/** The equation whose field closed last: opening the palette closes the field and moves the caret, so Simplify and + * Solve from Ctrl+K act on this one when none is selected. */ +let lastClosed: MathView | null = null; +function rememberClosed(view: MathView): void { + lastClosed = view; +} + +/** Simplify or Solve for the selected equation, the one just before the caret, or the one edited last. */ +export function runMathAction(editor: Editor, kind: 'simplify' | 'solve'): boolean { + const { selection } = editor.state; + let pos: number | null = null; + if (selection instanceof NodeSelection && selection.node.type.name.startsWith('math')) pos = selection.from; + else { + const before = selection.$from.nodeBefore; + if (before?.type.name.startsWith('math')) pos = selection.from - before.nodeSize; + } + if (pos !== null) { + const dom = editor.view.nodeDOM(pos); + const view = dom instanceof Element ? views.get(dom) : undefined; + if (view) return view.runAction(kind); + } + return lastClosed?.usableIn(editor) ? lastClosed.runAction(kind) : false; +} + +export function mathBlockExtensions(host: EditorHost): Extensions { return [ MathBlock.extend({ addNodeView: () => ({ node, editor, getPos }) => - new MathView(node, editor, getPos, true), + new MathView(node, editor, getPos, true, host), addKeyboardShortcuts() { return { Enter: () => openSelected(this.editor) }; }, + // Typing $$x^2$$ alone on a line makes display math. + addInputRules() { + const type = this.type; + return [ + new InputRule({ + find: /^\$\$([^$]+)\$\$$/, + handler: ({ state, range, match }) => { + const $from = state.doc.resolve(range.from); + if ($from.parent.type.name !== 'paragraph' || match[1].trim() === '') return null; + const block = type.create({ source: match[1].trim() }); + const { tr } = state; + tr.replaceWith($from.before(), $from.after(), block); + // The caret goes on to a new line below, so what is typed next is text, not the equation, and Enter + // there does not open the equation editor (Beta 5 walk W2-11). + const after = $from.before() + block.nodeSize; + tr.insert(after, type.schema.nodes.paragraph.create()); + tr.setSelection(TextSelection.create(tr.doc, after + 1)); + }, + }), + ]; + }, }), ]; } -export function mathInlineExtensions(_host: EditorHost): Extensions { +export function mathInlineExtensions(host: EditorHost): Extensions { return [ MathInline.extend({ addNodeView: () => ({ node, editor, getPos }) => - new MathView(node, editor, getPos, false), + new MathView(node, editor, getPos, false, host), addKeyboardShortcuts() { return { Enter: () => openSelected(this.editor) }; }, + // Typing $x^2$ makes an equation; "$5 and $" stays text because the source can't end in a space. + addInputRules() { + const type = this.type; + return [ + new InputRule({ + find: /(? { + state.tr.replaceWith(range.from, range.to, type.create({ source: match[1] })); + }, + }), + ]; + }, }), ]; } diff --git a/app/src/editor/extensions/mathDrawn.module.css b/app/src/editor/extensions/mathDrawn.module.css new file mode 100644 index 00000000..95dfee49 --- /dev/null +++ b/app/src/editor/extensions/mathDrawn.module.css @@ -0,0 +1,106 @@ +/* + * Drawn equations and the live preview in the source field (Phase 10). KaTeX's own style sheet sizes and spaces the + * drawing; these rules only place it in the text and in the editing field, on tokens. + */ +@layer components { + .drawn { + font-family: var(--font-reading); + color: var(--color-text-primary); + } + + .invalid { + outline: 1px dashed var(--color-status-danger); + outline-offset: 2px; + } + + .toggle { + min-block-size: var(--size-target-min); + margin-block-start: var(--space-1); + margin-inline-end: var(--space-2); + padding: var(--space-1) var(--space-2); + font: inherit; + color: var(--color-text-primary); + background: transparent; + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-sm); + cursor: pointer; + } + + .holderBlock { + margin-block-start: var(--space-2); + } + + .holderInline { + position: absolute; + z-index: var(--layer-dropdown); + margin-block-start: var(--space-1); + min-inline-size: 16rem; + padding: var(--space-2); + background: var(--color-surface-raised); + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-sm); + } + + /* With the writing area open, the holder takes its place in the text, so the block grows to hold the area and + nothing clips it or lies over it. */ + .holderPadOpen { + position: static; + display: block; + inline-size: min(100%, 42rem); + } + + .preview { + padding: var(--space-2) var(--space-3); + color: var(--color-text-primary); + background: var(--color-surface-raised); + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-sm); + text-align: center; + overflow-x: auto; + } + + .problem { + color: var(--color-status-danger); + font-size: var(--type-small-size); + } + + .problem:empty { + display: none; + } + + .actions { + display: flex; + gap: var(--space-3); + margin-block-start: var(--space-2); + } + + .actions button { + min-block-size: var(--size-target-min); + padding-inline: var(--space-3); + font: inherit; + font-size: var(--type-small-size); + color: var(--color-text-primary); + background: var(--color-surface-raised); + border: 1px solid var(--color-border-control); + border-radius: var(--radius-sm); + cursor: pointer; + } +} + +@layer components { + .pad { + display: grid; + gap: var(--space-2); + margin-block-start: var(--space-2); + color: var(--color-text-primary); + } + + .padCanvas { + inline-size: 100%; + max-inline-size: 40rem; + aspect-ratio: 640 / 220; + background: var(--color-surface-raised); + border: 1px solid var(--color-border-control); + border-radius: var(--radius-sm); + } +} diff --git a/app/src/editor/extensions/mathPen.test.tsx b/app/src/editor/extensions/mathPen.test.tsx new file mode 100644 index 00000000..f807182c --- /dev/null +++ b/app/src/editor/extensions/mathPen.test.tsx @@ -0,0 +1,57 @@ +// The writing area for math: strokes from a pen, a finger, or a mouse, with a resting palm ignored once a pen is seen. +import { describe, expect, it } from 'vitest'; +import { createPenPad } from './mathPen'; + +function stroke(canvas: HTMLElement, type: string, from: [number, number], to: [number, number]) { + const make = (name: string, [x, y]: [number, number]) => + new PointerEvent(name, { + pointerType: type, + pointerId: 1, + clientX: x, + clientY: y, + bubbles: true, + cancelable: true, + }); + canvas.dispatchEvent(make('pointerdown', from)); + canvas.dispatchEvent(make('pointermove', [(from[0] + to[0]) / 2, (from[1] + to[1]) / 2])); + canvas.dispatchEvent(make('pointermove', to)); + canvas.dispatchEvent(make('pointerup', to)); +} + +describe('the math writing area', () => { + it('collects what is written and clears it', () => { + const done: number[] = []; + const pad = createPenPad(() => done.push(1)); + document.body.append(pad.element); + const canvas = pad.element.querySelector('canvas') as HTMLElement; + stroke(canvas, 'pen', [10, 10], [60, 40]); + stroke(canvas, 'mouse', [20, 50], [80, 50]); + expect(pad.strokes()).toHaveLength(2); + expect(pad.strokes()[0]?.length).toBeGreaterThanOrEqual(3); + const [use, clear] = [...pad.element.querySelectorAll('button')]; + clear?.click(); + expect(pad.strokes()).toHaveLength(0); + use?.click(); + expect(done).toEqual([1]); + pad.element.remove(); + }); + + it('ignores a palm touching the pad after the pen has been used', () => { + const pad = createPenPad(() => undefined); + document.body.append(pad.element); + const canvas = pad.element.querySelector('canvas') as HTMLElement; + stroke(canvas, 'touch', [5, 5], [30, 30]); + expect(pad.strokes()).toHaveLength(1); + stroke(canvas, 'pen', [10, 10], [60, 40]); + stroke(canvas, 'touch', [100, 100], [120, 120]); + expect(pad.strokes()).toHaveLength(2); + pad.element.remove(); + }); + + it('has names for assistive technology', () => { + const pad = createPenPad(() => undefined); + expect(pad.element.getAttribute('aria-label')).toBe('Writing area for math'); + expect(pad.element.querySelector('canvas')?.getAttribute('aria-label')).toContain('pen or finger'); + expect([...pad.element.querySelectorAll('button')].map((b) => b.textContent)).toEqual(['Use writing', 'Clear']); + }); +}); diff --git a/app/src/editor/extensions/mathPen.ts b/app/src/editor/extensions/mathPen.ts new file mode 100644 index 00000000..8d330593 --- /dev/null +++ b/app/src/editor/extensions/mathPen.ts @@ -0,0 +1,95 @@ +// A small writing area for math (A1-28): the person writes an equation with a pen, a finger, or a mouse, and the +// strokes go to the page's recognizer. It draws only what is written and keeps nothing; the strokes leave with the pad. +import { t } from '../../strings/t'; +import styles from './mathDrawn.module.css'; + +export interface PenPad { + element: HTMLElement; + /** The strokes written so far, as points in the pad's own units. */ + strokes(): [number, number][][]; + clear(): void; +} + +const MAX_POINTS = 4000; + +/** Makes the pad. `onDone` runs when the person chooses Use writing. */ +export function createPenPad(onDone: () => void): PenPad { + const element = document.createElement('div'); + element.className = styles.pad; + element.setAttribute('role', 'group'); + element.setAttribute('aria-label', t('intelMore.math.penPad')); + const canvas = document.createElement('canvas'); + canvas.className = styles.padCanvas; + canvas.width = 640; + canvas.height = 220; + canvas.setAttribute('aria-label', t('intelMore.math.penHelp')); + canvas.style.touchAction = 'none'; + const buttons = document.createElement('div'); + buttons.className = styles.actions; + const strokes: [number, number][][] = []; + let current: [number, number][] | null = null; + let sawPen = false; + const context = canvas.getContext('2d'); + const ink = () => getComputedStyle(element).color; + const point = (event: PointerEvent): [number, number] => { + const rect = canvas.getBoundingClientRect(); + return [ + ((event.clientX - rect.left) * canvas.width) / Math.max(1, rect.width), + ((event.clientY - rect.top) * canvas.height) / Math.max(1, rect.height), + ]; + }; + const draw = () => { + if (!context) return; + context.clearRect(0, 0, canvas.width, canvas.height); + context.strokeStyle = ink(); + context.lineWidth = 3; + context.lineCap = 'round'; + context.lineJoin = 'round'; + for (const stroke of strokes) { + context.beginPath(); + stroke.forEach(([x, y], i) => (i === 0 ? context.moveTo(x, y) : context.lineTo(x, y))); + context.stroke(); + } + }; + // Writing must not take focus from the equation field, or the field would close and save with the first stroke. + canvas.addEventListener('mousedown', (event) => event.preventDefault()); + canvas.addEventListener('pointerdown', (event) => { + if (event.pointerType === 'pen') sawPen = true; + // A palm resting beside a pen is not writing. + if (event.pointerType === 'touch' && sawPen) return; + try { + canvas.setPointerCapture(event.pointerId); + } catch { + // The pointer is already gone, which only means the stroke is short. + } + current = [point(event)]; + strokes.push(current); + }); + canvas.addEventListener('pointermove', (event) => { + if (!current || current.length >= MAX_POINTS) return; + current.push(point(event)); + draw(); + }); + const end = () => { + current = null; + }; + canvas.addEventListener('pointerup', end); + canvas.addEventListener('pointercancel', end); + const button = (label: string, run: () => void) => { + const one = document.createElement('button'); + one.type = 'button'; + one.textContent = label; + // A press must not take focus from the equation field, or the field would close and save. + one.addEventListener('mousedown', (event) => event.preventDefault()); + one.addEventListener('click', run); + buttons.append(one); + }; + const clear = () => { + strokes.length = 0; + draw(); + }; + button(t('intelMore.math.penDone'), onDone); + button(t('intelMore.math.penClear'), clear); + element.append(canvas, buttons); + return { element, strokes: () => strokes.map((stroke) => [...stroke]), clear }; +} diff --git a/app/src/editor/extensions/mathTyping.test.ts b/app/src/editor/extensions/mathTyping.test.ts new file mode 100644 index 00000000..3e7e2763 --- /dev/null +++ b/app/src/editor/extensions/mathTyping.test.ts @@ -0,0 +1,49 @@ +// @vitest-environment jsdom +// Typing $x^2$ makes an equation and $$x^2$$ on its own line makes display math. +import { afterEach, describe, expect, it } from 'vitest'; +import { mountEditor, typeInto } from '../commands/testing'; +import type { TestEditor } from '../commands/testing'; + +let mounted: TestEditor | null = null; +afterEach(() => { + mounted?.destroy(); + mounted = null; +}); + +const names = (page: TestEditor): string[] => { + const out: string[] = []; + page.editor.state.doc.descendants((node) => { + out.push(node.type.name); + }); + return out; +}; + +describe('typing math', () => { + it('turns $x^2$ into an inline equation', () => { + mounted = mountEditor('[]'); + typeInto(mounted.editor, 'Inline $x^2$'); + expect(names(mounted)).toContain('mathInline'); + }); + + it('leaves prices alone', () => { + mounted = mountEditor('[]'); + typeInto(mounted.editor, 'It costs $5 and $'); + expect(names(mounted)).not.toContain('mathInline'); + }); + + it('turns $$x^2$$ on its own line into display math', () => { + mounted = mountEditor('[]'); + typeInto(mounted.editor, '$$x^2$$'); + expect(names(mounted)).toContain('mathBlock'); + }); + + it('carries on with text on a new line after display math', () => { + mounted = mountEditor('[]'); + typeInto(mounted.editor, '$$x^2$$'); + typeInto(mounted.editor, 'Cost $5 and $10 stays'); + const doc = mounted.editor.state.doc; + expect(doc.child(0).type.name).toBe('mathBlock'); + expect(doc.child(0).attrs.source).toBe('x^2'); + expect(doc.child(1).textContent).toBe('Cost $5 and $10 stays'); + }); +}); diff --git a/app/src/editor/extensions/outline.test.ts b/app/src/editor/extensions/outline.test.ts index c8d9d59e..ddfe30f8 100644 --- a/app/src/editor/extensions/outline.test.ts +++ b/app/src/editor/extensions/outline.test.ts @@ -3,7 +3,8 @@ // edges. Levels refuse past heading 1 and 6. Fold buttons are real buttons with aria-expanded, and folded content // gets `hidden`. Fold keys survive a reload, and unfoldTo uncovers a position. import { afterEach, describe, expect, it } from 'vitest'; -import { foldKeysFor, foldsOf, positionsFor, setFolds, unfoldTo } from '../commands/fold'; +import { foldKeysFor, foldsOf, positionsFor, setFolds, unfoldAroundDom, unfoldTo } from '../commands/fold'; +import { META_REMOTE } from '../meta'; import { runOutline } from '../commands/outlineRun'; import { mountEditor } from '../commands/testing'; import type { TestEditor } from '../commands/testing'; @@ -143,3 +144,30 @@ describe('folding', () => { expect(unfoldTo(page.editor, inside)).toBe(false); }); }); + +describe('opening folds for what the person is taken to', () => { + it('unfolds around a DOM node, as a link, a search result, read aloud, and find do', () => { + const page = mount('## A\n\nHidden text\n\n## B\n\nMore'); + page.editor.view.dispatch(setFolds(page.editor.state.tr, [0])); + const hidden = [...page.root.querySelectorAll('p')].find((node) => node.textContent === 'Hidden text'); + expect(hidden).toBeDefined(); + expect(unfoldAroundDom(hidden!)).toBe(true); + expect(foldsOf(page.editor.state)).toEqual([]); + expect(unfoldAroundDom(hidden!)).toBe(false); + expect(unfoldAroundDom(document.createElement('p'))).toBe(false); + }); + + it('opens the fold an undo lands in, and leaves other folds shut', () => { + const page = mount('## A\n\nHidden text\n\n## B\n\nMore text'); + const inside = page.editor.state.doc.child(0).nodeSize + 2; + page.editor.view.dispatch(page.editor.state.tr.insertText('x', inside)); + const { doc } = page.editor.state; + const second = doc.child(0).nodeSize + doc.child(1).nodeSize; + page.editor.view.dispatch(setFolds(page.editor.state.tr, [0, second])); + expect(foldsOf(page.editor.state)).toEqual([0, second]); + // The core's undo puts the text back as a change marked as remote. + page.editor.view.dispatch(page.editor.state.tr.delete(inside, inside + 1).setMeta(META_REMOTE, true)); + expect(page.markdown()).toBe('## A\n\nHidden text\n\n## B\n\nMore text'); + expect(foldsOf(page.editor.state)).toEqual([second - 1]); + }); +}); diff --git a/app/src/editor/host.ts b/app/src/editor/host.ts index 16cd9ea7..cd38be0a 100644 --- a/app/src/editor/host.ts +++ b/app/src/editor/host.ts @@ -33,6 +33,69 @@ export interface EditorHost { spelling(): SpellingService | null; /** Selects blocks as objects: when a selection grows past one block, on the second Ctrl+A, or on Escape. */ selectBlocks(blocks: readonly BlockId[], reason: 'escalate' | 'selectAll' | 'escape'): void; + /** Loads the LaTeX drawing code (Phase 10) on first use. Absent or null shows math as its source. */ + math?(): Promise | null; + /** Loads the function grapher (Phase 10) on first use. Absent or null leaves a graph as its code block. */ + graph?(): Promise | null; +} + +/** What a graph is given: the text of its code block, and a way to replace that text. */ +export interface GraphProps { + source: string; + editable: boolean; + onSource(next: string): void; +} + +export interface GraphHandle { + update(props: GraphProps): void; + destroy(): void; +} + +/** Draws a graph into an element the editor owns. */ +export interface GraphRenderer { + mount(container: HTMLElement, props: GraphProps): GraphHandle; +} + +/** One drawn equation, or what is wrong with its LaTeX and where. */ +export type MathDrawing = { ok: true; html: string } | { ok: false; message: string; position: number; length: number }; + +/** What an action did to some LaTeX: new LaTeX, or why it could not. */ +export type MathActionResult = + { ok: true; latex: string } | { ok: false; reason: 'unsupported' | 'unchanged' | 'none' | 'letters' }; + +/** A visual equation field: the page's math editor with its own keyboard, which reads and writes LaTeX. */ +export interface VisualMathField { + /** The element to put in the page. It takes focus, key events, and blur like any field. */ + element: HTMLElement; + /** The LaTeX of what is in the field. */ + value: string; + /** The equation as words, for a screen reader. */ + spoken(): string; + /** Puts LaTeX in at the caret. */ + insert(latex: string): void; + focus(): void; +} + +export interface VisualMath { + create(options: { value: string; label: string; display: boolean }): Promise; +} + +/** Reads handwritten math. Each stroke is a list of points in one shared space. */ +export interface PenMath { + recognize(strokes: [number, number][][]): Promise; +} + +export interface MathRenderer { + /** Handwriting to LaTeX, when the page offers it. */ + pen?: PenMath; + /** The visual editor, when the page offers it. */ + visual?: VisualMath; + render(source: string, display: boolean): MathDrawing; + /** Simplify and Solve for the source field, when the page offers them. */ + actions?: { + simplify(latex: string): MathActionResult; + solve(latex: string): MathActionResult; + }; } /** The settings.editing defaults, for hosts in tests and before settings load. */ diff --git a/app/src/editor/table/mapping.test.ts b/app/src/editor/table/mapping.test.ts index a2976193..82788bfc 100644 --- a/app/src/editor/table/mapping.test.ts +++ b/app/src/editor/table/mapping.test.ts @@ -2,7 +2,7 @@ // and after random table commands every row still has the first row's columns, in order. import * as fc from 'fast-check'; import { describe, expect, it } from 'vitest'; -import { applyTableOp } from '../commands/tables'; +import { applyTableOp, fitAddedColumn } from '../commands/tables'; import type { TableOp } from '../commands/tables'; import { createMarkdownCache } from '../markdown'; import type { TableData } from '../schema/specs'; @@ -216,3 +216,25 @@ describe('the table commands on data', () => { expect(applyTableOp(one, { row: 0, column: 0 }, { op: 'deleteColumn' })).toBeNull(); }); }); + +describe('a column added to a table that fits its page (Beta 5 walk W2-13)', () => { + const widths = (data: TableData) => data.columns.map((column) => column.width); + + it('takes the room left, so the table still fits', () => { + const data = { ...newTableData(3, 3, true) }; + const each = data.columns[0].width; + const added = applyTableOp(data, { row: 0, column: 2 }, { op: 'columnRight' })!; + const room = each * 3 + 143; + expect(widths(fitAddedColumn(data, added, room).data)).toEqual([each, each, each, 143]); + }); + + it('keeps its width when there is no usable room, or the table was already wider than the page', () => { + const data = { ...newTableData(3, 3, true) }; + const each = data.columns[0].width; + const added = applyTableOp(data, { row: 0, column: 2 }, { op: 'columnRight' })!; + expect(fitAddedColumn(data, added, each * 3 + 40)).toBe(added); + expect(fitAddedColumn(data, added, each * 2)).toBe(added); + expect(fitAddedColumn(data, added, each * 10)).toBe(added); + expect(fitAddedColumn(data, added, 0)).toBe(added); + }); +}); diff --git a/app/src/features/a11yCheck/A11y.module.css b/app/src/features/a11yCheck/A11y.module.css new file mode 100644 index 00000000..09cc8f36 --- /dev/null +++ b/app/src/features/a11yCheck/A11y.module.css @@ -0,0 +1,31 @@ +@layer components { + .page h3 { + margin: var(--space-4) 0 var(--space-2); + font-weight: var(--font-weight-semibold); + } + + .issues { + display: grid; + gap: var(--space-3); + margin: 0; + padding: 0; + list-style: none; + } + + .issue { + padding: var(--space-3); + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-md); + } + + .issue p { + margin: 0 0 var(--space-2); + } + + .fixes { + display: flex; + flex-wrap: wrap; + align-items: end; + gap: var(--space-2); + } +} diff --git a/app/src/features/a11yCheck/AccessibilityReport.tsx b/app/src/features/a11yCheck/AccessibilityReport.tsx new file mode 100644 index 00000000..d852dd1d --- /dev/null +++ b/app/src/features/a11yCheck/AccessibilityReport.tsx @@ -0,0 +1,165 @@ +// The accessibility report: the problems of a page or of every page in a section, each with its fix. A fix is one +// edit to the page, so Ctrl+Z in the page undoes it. The report checks a page again after a fix. + +import { useCallback, useEffect, useState } from 'react'; +import { currentNotesService } from '../../services/notes'; +import type { NodeId, NodeSummary, NotesService } from '../../services/notes'; +import type { Edit, PageId } from '../../services/pages/types'; +import { t } from '../../strings/t'; +import { Button, Dialog, TextField, showToast } from '../../ui'; +import { pagesClient, showDialog } from '../qol'; +import { checkPage, imageAltEdits } from './check'; +import type { Issue } from './check'; +import styles from './A11y.module.css'; + +interface PageResult { + readonly id: NodeId; + readonly title: string; + readonly issues: readonly Issue[]; +} + +/** Opens a page, runs `work` on it, and closes it. */ +async function withPage( + id: NodeId, + work: (page: Awaited['open']>>) => Promise, +) { + const page = await pagesClient().open(id as unknown as PageId, { viewport: null }); + try { + return await work(page); + } finally { + await page.close(); + } +} + +async function checkNode(node: NodeSummary): Promise { + const issues = await withPage(node.id, async (page) => checkPage(page.initial)); + return { id: node.id, title: node.title.trim() || t('tree.untitled.page'), issues }; +} + +function IssueRow({ issue, onFix }: { issue: Issue; onFix(edits: readonly Edit[]): Promise }) { + const [alt, setAlt] = useState(''); + const [busy, setBusy] = useState(false); + const run = async (edits: readonly Edit[]) => { + setBusy(true); + try { + await onFix(edits); + } finally { + setBusy(false); + } + }; + const message = t(`qol.a11y.issues.${issue.kind}`, issue.params as never); + return ( +
  • +

    {message}

    +
    + {issue.fix.kind === 'edits' && ( + + )} + {issue.fix.kind === 'imageAlt' && ( + <> + + + + + )} +
    +
  • + ); +} + +function Report({ nodes, done }: { nodes: readonly NodeSummary[]; done(): void }) { + const [results, setResults] = useState(null); + useEffect(() => { + let current = true; + void (async () => { + const found: PageResult[] = []; + for (const node of nodes) { + try { + found.push(await checkNode(node)); + } catch { + found.push({ id: node.id, title: node.title, issues: [] }); + } + } + if (current) setResults(found); + })(); + return () => { + current = false; + }; + }, [nodes]); + const fix = useCallback( + async (result: PageResult, edits: readonly Edit[]) => { + try { + await withPage(result.id, (page) => page.send({ edits: [...edits] })); + const node = nodes.find((one) => one.id === result.id); + if (node) { + const next = await checkNode(node); + setResults((all) => all && all.map((one) => (one.id === next.id ? next : one))); + } + } catch { + showToast({ message: t('qol.a11y.fixFailed'), tone: 'danger' }); + } + }, + [nodes], + ); + const total = results?.reduce((sum, result) => sum + result.issues.length, 0) ?? 0; + return ( + + {results + ?.filter((result) => result.issues.length > 0) + .map((result) => ( +
    +

    {result.title}

    +
      + {result.issues.map((issue) => ( + fix(result, edits)} /> + ))} +
    +
    + ))} +
    + ); +} + +/** Checks one page, or every page of a section. */ +export async function openAccessibilityCheck(notes: NotesService, node: NodeSummary): Promise { + const pages = + node.kind === 'page' ? [node] : (await notes.listChildren(node.id)).filter((child) => child.kind === 'page'); + await showDialog((done) => done()} />); +} + +/** Opens the full report for pages by ID, as the check-first dialog's Review button does. */ +export async function reviewPages(ids: readonly string[]): Promise { + const notes = currentNotesService(); + if (!notes) return; + const nodes: NodeSummary[] = []; + for (const id of ids) { + const node = await notes.get(id as NodeId).catch(() => null); + if (node) nodes.push(node); + } + if (nodes.length > 0) await showDialog((done) => done()} />); +} diff --git a/app/src/features/a11yCheck/check.test.ts b/app/src/features/a11yCheck/check.test.ts new file mode 100644 index 00000000..1d929275 --- /dev/null +++ b/app/src/features/a11yCheck/check.test.ts @@ -0,0 +1,96 @@ +// checks-disable-file brand-consistency: the tests use hexadecimal text colors, the input the checker reads +import { describe, expect, it } from 'vitest'; +import type { BlockJson, PageJson } from '../../services/pages/types'; +import { checkPage, contrast, imageAltEdits } from './check'; + +let counter = 0; +function block(type: string, data: Record, extra: Partial = {}): BlockJson { + counter += 1; + return { + id: `b${counter}` as BlockJson['id'], + type, + order: `a${counter}`, + created: '2026-01-01T00:00:00Z', + modified: '2026-01-01T00:00:00Z', + data, + ...extra, + }; +} + +function page(blocks: BlockJson[], view: PageJson['view'] = {}): PageJson { + return { + id: 'p1' as PageJson['id'], + title: 'A page', + created: '2026-01-01T00:00:00Z', + modified: '2026-01-01T00:00:00Z', + tags: [], + view, + blocks, + assets: {}, + }; +} + +describe('accessibility checker', () => { + it('passes a plain page', () => { + const clean = page([block('text', { markdown: '## Intro\n\nHello.\n\n### Detail' })]); + expect(checkPage(clean)).toEqual([]); + }); + + it('finds an image with no description, and accepts a decorative one', () => { + const issues = checkPage(page([block('image', { alt: '' }), block('image', { alt: '', decorative: true })])); + expect(issues.map((issue) => issue.kind)).toEqual(['imageAlt']); + expect(imageAltEdits('b1', ' A graph ', false)[0]).toMatchObject({ data: { alt: 'A graph', decorative: false } }); + }); + + it('finds a skipped heading level and fixes it to the next level', () => { + const [issue] = checkPage(page([block('text', { markdown: '## One\n\n#### Four' })])); + expect(issue.kind).toBe('headingSkip'); + expect(issue.params).toMatchObject({ from: 2, to: 4, fixed: 3 }); + expect(issue.fix).toMatchObject({ kind: 'edits', edits: [{ edit: 'setText', markdown: '## One\n\n### Four' }] }); + }); + + it('wants a header row on a table with several rows', () => { + const rows = [{}, {}]; + expect(checkPage(page([block('table', { rows, header: false })])).map((i) => i.kind)).toEqual(['tableHeader']); + expect(checkPage(page([block('table', { rows, header: true })]))).toEqual([]); + expect(checkPage(page([block('table', { rows: [{}], header: false })]))).toEqual([]); + }); + + it('measures contrast against white and flags pale text colors', () => { + expect(contrast('#000000', '#ffffff')).toBeCloseTo(21, 0); + expect(contrast('red', '#ffffff')).toBeNull(); + const pale = block('text', { markdown: 'See this part' }); + const [issue] = checkPage(page([pale])); + expect(issue.kind).toBe('lowContrast'); + expect(issue.fix).toMatchObject({ edits: [{ markdown: 'See this part' }] }); + expect(checkPage(page([block('text', { markdown: 'See this' })]))).toEqual([]); + }); + + it('flags meaning carried by color alone and bolds the colored text as the fix', () => { + const text = 'Yes and No'; + const [issue] = checkPage(page([block('text', { markdown: text })])); + expect(issue.kind).toBe('colorOnly'); + const edit = issue.fix.kind === 'edits' ? issue.fix.edits[0] : null; + expect(edit).toMatchObject({ markdown: expect.stringContaining('**Yes**') }); + expect( + checkPage( + page([ + block('text', { + markdown: '**Yes** and **No**', + }), + ]), + ), + ).toEqual([]); + }); + + it('asks a free-form page for a reading order', () => { + const frame = (x: number, y: number) => ({ frame: { x, y, w: 10, h: 10 } }) as Partial; + const blocks = [block('text', { markdown: 'b' }, frame(0, 50)), block('text', { markdown: 'a' }, frame(0, 0))]; + const [issue] = checkPage(page(blocks, { layout: 'freeform' })); + expect(issue.kind).toBe('readingOrder'); + expect(issue.fix).toMatchObject({ + edits: [{ edit: 'setPage', view: { readingOrder: [blocks[1].id, blocks[0].id] } }], + }); + expect(checkPage(page(blocks, { layout: 'freeform', readingOrder: [blocks[1].id, blocks[0].id] }))).toEqual([]); + }); +}); diff --git a/app/src/features/a11yCheck/check.ts b/app/src/features/a11yCheck/check.ts new file mode 100644 index 00000000..3c7767e4 --- /dev/null +++ b/app/src/features/a11yCheck/check.ts @@ -0,0 +1,218 @@ +// The accessibility checker (docs/FEATURES.md, "Accessibility checker"). It reads a page as the page service gives it. +// It lists what makes the page hard to use with a screen reader, a keyboard, or low vision. The problems are images +// with no description, skipped heading levels, tables without a header row, text colors that are hard to read, +// meaning that only color carries, and free-form pages with no reading order. Each problem carries the edit that +// fixes it, so the report can apply the fix with one press. Nothing here changes a page. + +import type { BlockJson, Edit, PageJson } from '../../services/pages/types'; + +export type IssueKind = 'imageAlt' | 'headingSkip' | 'tableHeader' | 'lowContrast' | 'colorOnly' | 'readingOrder'; + +/** How an issue gets fixed: ready-made edits, or edits that need some text from the person. */ +export type Fix = + { readonly kind: 'edits'; readonly edits: readonly Edit[] } | { readonly kind: 'imageAlt'; readonly block: string }; + +export interface Issue { + readonly id: string; + readonly kind: IssueKind; + readonly block: string | null; + /** Message parameters: a heading level, a color, and so on. */ + readonly params: Readonly>; + readonly fix: Fix; +} + +const HEADING = /^(#{1,6})\s+\S/; +const COLOR_SPAN = /]*data-color="([^"]+)"[^>]*>(.*?)<\/span>/gs; +const EMPHASIS = /\*\*|__|||(^|\s)\*\S|(^|\s)_\S/; + +/** WCAG relative luminance of a color written as #rrggbb or #rgb. Null for anything else. */ +export function luminance(hex: string): number | null { + const match = /^#([0-9a-f]{3}|[0-9a-f]{6})$/i.exec(hex.trim()); + if (!match) return null; + const full = match[1].length === 3 ? [...match[1]].map((c) => c + c).join('') : match[1]; + const channels = [0, 2, 4].map((at) => parseInt(full.slice(at, at + 2), 16) / 255); + const [r, g, b] = channels.map((c) => (c <= 0.03928 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4)); + return 0.2126 * r + 0.7152 * g + 0.0722 * b; +} + +/** The contrast ratio of two colors, from 1 to 21, or null when either isn't a hex color. */ +export function contrast(a: string, b: string): number | null { + const one = luminance(a); + const two = luminance(b); + if (one === null || two === null) return null; + const [light, dark] = one > two ? [one, two] : [two, one]; + return (light + 0.05) / (dark + 0.05); +} + +/** The lowest contrast ratio that passes for body text (WCAG 2.2, 1.4.3). */ +export const MIN_CONTRAST = 4.5; +// checks-disable-next-line brand-consistency: the white of a light page, for the contrast ratio, not a style +const PAGE_WHITE = '#ffffff'; + +function textOf(block: BlockJson): string { + const markdown = block.data.markdown; + return typeof markdown === 'string' ? markdown : ''; +} + +/** The blocks in the order a reader meets them: the reading order of the page, else by position on the page. */ +export function readingBlocks(page: PageJson): BlockJson[] { + const preferred = page.view.readingOrder ?? []; + const rank = new Map(preferred.map((id, i) => [id as string, i])); + const place = (block: BlockJson) => [block.frame?.y ?? 0, block.frame?.x ?? 0] as const; + return [...page.blocks].sort((a, b) => { + const ra = rank.get(a.id); + const rb = rank.get(b.id); + if (ra !== undefined || rb !== undefined) return (ra ?? Infinity) - (rb ?? Infinity); + const [ay, ax] = place(a); + const [by, bx] = place(b); + return ay - by || ax - bx || a.order.localeCompare(b.order); + }); +} + +function headingIssues(blocks: readonly BlockJson[]): Issue[] { + const issues: Issue[] = []; + // The page title is the first level, so the first heading in the text may be level 2. + let previous = 1; + for (const block of blocks) { + if (block.type !== 'text') continue; + const lines = textOf(block).split('\n'); + let changed = false; + lines.forEach((line, index) => { + const level = HEADING.exec(line)?.[1].length; + if (!level) return; + if (level > previous + 1) { + const fixed = previous + 1; + lines[index] = line.replace(/^#{1,6}/, '#'.repeat(fixed)); + changed = true; + issues.push({ + id: `${block.id}:heading:${index}`, + kind: 'headingSkip', + block: block.id, + params: { from: previous, to: level, fixed }, + // Applied together below, so two skips in one block fix in one edit. + fix: { kind: 'edits', edits: [] }, + }); + previous = fixed; + } else { + previous = level; + } + }); + if (changed) { + const edit: Edit = { edit: 'setText', block: block.id, markdown: lines.join('\n') }; + for ( + let i = issues.length - 1; + i >= 0 && issues[i].block === block.id && issues[i].kind === 'headingSkip'; + i -= 1 + ) { + issues[i] = { ...issues[i], fix: { kind: 'edits', edits: [edit] } }; + } + } + } + return issues; +} + +function colorIssues(blocks: readonly BlockJson[]): Issue[] { + const issues: Issue[] = []; + for (const block of blocks) { + if (block.type !== 'text') continue; + const markdown = textOf(block); + const spans = [...markdown.matchAll(COLOR_SPAN)]; + const weak = spans.find((span) => { + const ratio = contrast(span[1], PAGE_WHITE); + return ratio !== null && ratio < MIN_CONTRAST; + }); + if (weak) { + const cleaned = markdown.replace(COLOR_SPAN, (all, color: string, inner: string) => + (contrast(color, PAGE_WHITE) ?? 21) < MIN_CONTRAST ? inner : all, + ); + issues.push({ + id: `${block.id}:contrast`, + kind: 'lowContrast', + block: block.id, + params: { color: weak[1], ratio: Number((contrast(weak[1], PAGE_WHITE) ?? 0).toFixed(1)) }, + fix: { kind: 'edits', edits: [{ edit: 'setText', block: block.id, markdown: cleaned }] }, + }); + } + const colors = new Set(spans.map((span) => span[1])); + if (colors.size >= 2 && !EMPHASIS.test(markdown.replace(COLOR_SPAN, '$2'))) { + const bolded = markdown.replace( + COLOR_SPAN, + (_all, color: string, inner: string) => `**${inner}**`, + ); + issues.push({ + id: `${block.id}:colorOnly`, + kind: 'colorOnly', + block: block.id, + params: { count: colors.size }, + fix: { kind: 'edits', edits: [{ edit: 'setText', block: block.id, markdown: bolded }] }, + }); + } + } + return issues; +} + +function imageIssue(block: BlockJson): Issue[] { + const alt = typeof block.data.alt === 'string' ? block.data.alt.trim() : ''; + if (alt || block.data.decorative === true) return []; + return [ + { + id: `${block.id}:alt`, + kind: 'imageAlt', + block: block.id, + params: {}, + fix: { kind: 'imageAlt', block: block.id }, + }, + ]; +} + +function tableIssue(block: BlockJson): Issue[] { + const rows = Array.isArray(block.data.rows) ? block.data.rows.length : 0; + if (rows <= 1 || block.data.header === true) return []; + return [ + { + id: `${block.id}:header`, + kind: 'tableHeader', + block: block.id, + params: {}, + fix: { kind: 'edits', edits: [{ edit: 'patchBlock', block: block.id, data: { header: true } }] }, + }, + ]; +} + +function objectIssues(blocks: readonly BlockJson[]): Issue[] { + return blocks.flatMap((block) => + block.type === 'image' ? imageIssue(block) : block.type === 'table' ? tableIssue(block) : [], + ); +} + +function orderIssues(page: PageJson): Issue[] { + const framed = page.blocks.filter((block) => block.frame); + if (page.view.layout !== 'freeform' || framed.length < 2 || (page.view.readingOrder?.length ?? 0) > 0) return []; + const order = readingBlocks({ ...page, view: { ...page.view, readingOrder: [] } }).map((block) => block.id); + return [ + { + id: 'page:readingOrder', + kind: 'readingOrder', + block: null, + params: { count: framed.length }, + fix: { kind: 'edits', edits: [{ edit: 'setPage', view: { readingOrder: order } }] }, + }, + ]; +} + +/** Every accessibility problem of a page, in the order a reader meets them. */ +export function checkPage(page: PageJson): Issue[] { + const blocks = readingBlocks(page); + return [...objectIssues(blocks), ...headingIssues(blocks), ...colorIssues(blocks), ...orderIssues(page)]; +} + +/** The edits for a description the person typed, or for marking the image as decorative. */ +export function imageAltEdits(block: string, alt: string, decorative: boolean): Edit[] { + return [ + { + edit: 'patchBlock', + block, + data: decorative ? { decorative: true, alt: '' } : { alt: alt.trim(), decorative: false }, + }, + ]; +} diff --git a/app/src/features/a11yCheck/gate.test.ts b/app/src/features/a11yCheck/gate.test.ts new file mode 100644 index 00000000..3d3b8b7c --- /dev/null +++ b/app/src/features/a11yCheck/gate.test.ts @@ -0,0 +1,31 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { checkBeforeOutput, checkFirstOn, setCheckFirst } from './gate'; + +describe('check accessibility first', () => { + beforeEach(() => { + const store = new Map(); + vi.stubGlobal('localStorage', { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => void store.set(key, value), + }); + }); + + it('is off until the person turns it on, and remembers the choice', () => { + expect(checkFirstOn()).toBe(false); + setCheckFirst(true); + expect(checkFirstOn()).toBe(true); + setCheckFirst(false); + expect(checkFirstOn()).toBe(false); + }); + + it('lets output go ahead without a check when it is off or nothing is written', async () => { + let reviewed = false; + const review = () => { + reviewed = true; + }; + expect(await checkBeforeOutput(['a'], review)).toBe(true); + setCheckFirst(true); + expect(await checkBeforeOutput([], review)).toBe(true); + expect(reviewed).toBe(false); + }); +}); diff --git a/app/src/features/a11yCheck/gate.tsx b/app/src/features/a11yCheck/gate.tsx new file mode 100644 index 00000000..8cb172ab --- /dev/null +++ b/app/src/features/a11yCheck/gate.tsx @@ -0,0 +1,117 @@ +// "Check accessibility first" for Export, Print/PDF, and Send. The choice is remembered. When it is on, the checker +// runs on the pages about to be written and the report shows before anything is: the person can go on anyway, +// open the full report to fix things, or cancel. Pages with no problems go straight through. +import { useState } from 'react'; +import { isEnabled } from '../../app/flags'; +import { t } from '../../strings/t'; +import { Dialog, Switch } from '../../ui'; +import { pagesClient, showDialog } from '../qol'; +import type { PageId } from '../../services/pages/types'; +import { checkPage } from './check'; +import type { Issue } from './check'; +import styles from './A11y.module.css'; + +const KEY = 'opennote.a11y.checkFirst'; + +/** Whether the person asked for the check before output. */ +export function checkFirstOn(): boolean { + if (!isEnabled('qol.a11yGate')) return false; + try { + return localStorage.getItem(KEY) === 'true'; + } catch { + return false; + } +} + +export function setCheckFirst(on: boolean): void { + try { + localStorage.setItem(KEY, String(on)); + } catch { + // The choice just is not remembered. + } +} + +/** The switch for Export, Print/PDF, and Send dialogs. It shows nothing when the feature is off. */ +export function CheckFirstSwitch() { + const [on, setOn] = useState(checkFirstOn); + if (!isEnabled('qol.a11yGate')) return null; + return ( + { + setCheckFirst(value); + setOn(value); + }} + /> + ); +} + +interface Found { + readonly id: string; + readonly issues: readonly Issue[]; +} + +/** Checks the pages, in order. A page that cannot be read counts as having no problems. */ +export async function findProblems(pageIds: readonly string[]): Promise { + const found: Found[] = []; + for (const id of pageIds) { + try { + const open = await pagesClient().open(id as PageId, { viewport: null }); + try { + found.push({ id, issues: checkPage(open.initial) }); + } finally { + await open.close(); + } + } catch { + found.push({ id, issues: [] }); + } + } + return found; +} + +type Choice = 'continue' | 'review' | 'cancel'; + +function GateDialog(props: { problems: readonly Issue[]; done(choice: Choice): void }) { + const { problems, done } = props; + return ( + done('cancel')} + actions={[ + { + id: 'cancel', + label: t('common.cancel'), + variant: 'secondary', + leastDestructive: true, + onPress: () => done('cancel'), + }, + { id: 'review', label: t('qol.a11y.gateReview'), variant: 'secondary', onPress: () => done('review') }, + { id: 'go', label: t('qol.a11y.gateContinue'), variant: 'primary', onPress: () => done('continue') }, + ]} + > +
      + {problems.slice(0, 8).map((issue) => ( +
    • {t(`qol.a11y.issues.${issue.kind}`, issue.params as never)}
    • + ))} +
    +
    + ); +} + +/** + * Runs the check when the person turned it on. Resolves true when the output may go ahead: the check is off, found + * nothing, or the person chose to go on. Choosing to review opens the full report and resolves false. + */ +export async function checkBeforeOutput( + pageIds: readonly string[], + review: () => void | Promise, +): Promise { + if (!checkFirstOn() || pageIds.length === 0) return true; + const problems = (await findProblems(pageIds)).flatMap((found) => found.issues); + if (problems.length === 0) return true; + const choice = await showDialog((done) => ); + if (choice === 'review') await review(); + return choice === 'continue'; +} diff --git a/app/src/features/a11yCheck/index.ts b/app/src/features/a11yCheck/index.ts new file mode 100644 index 00000000..e8a6cb28 --- /dev/null +++ b/app/src/features/a11yCheck/index.ts @@ -0,0 +1,4 @@ +// The accessibility checker: the report for a page or a section, and the check before output. +export { openAccessibilityCheck, reviewPages } from './AccessibilityReport'; +export { checkPage } from './check'; +export { checkBeforeOutput, CheckFirstSwitch, checkFirstOn } from './gate'; diff --git a/app/src/features/accounts/fakeHost.ts b/app/src/features/accounts/fakeHost.ts new file mode 100644 index 00000000..2aac2b8a --- /dev/null +++ b/app/src/features/accounts/fakeHost.ts @@ -0,0 +1,109 @@ +// An in-memory AccountsHost for the web platform and the tests. State and "files" live in maps. A download asks the +// connectors client for the answer and keeps its text as the file; an upload joins the parts and asks the client to +// run the request with the joined body, so a mock server sees exactly the bytes a service would. + +import type { ConnectorsClient } from '../connectors'; +import { ConnectorError } from '../connectors'; +import type { AccountsHost, UploadPart } from './host'; + +const encoder = new TextEncoder(); + +const fromBase64 = (base64: string): Uint8Array => Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)); + +export interface FakeAccountsHost extends AccountsHost { + /** The saved state by name, for a test to read or seed. */ + readonly state: Map; + /** The files of the transfer folder by name. */ + readonly files: Map; + /** The bodies the uploads sent, as text. */ + readonly uploads: { connector: string; url: string; contentType: string; body: string }[]; +} + +const FOLDER = '/fake-accounts/tmp'; + +export interface FakeHostOptions { + /** The answer to a preview address (a title or a picture). Default: not found. */ + preview?(url: string): { status: number; contentType: string | null; bytes: Uint8Array }; +} + +export function createFakeAccountsHost( + connectors: () => ConnectorsClient, + options: FakeHostOptions = {}, +): FakeAccountsHost { + const state = new Map(); + const files = new Map(); + const uploads: FakeAccountsHost['uploads'] = []; + const name = (file: string): string => (file.startsWith(`${FOLDER}/`) ? file.slice(FOLDER.length + 1) : file); + const partBytes = (part: UploadPart): Uint8Array => { + if (part.kind === 'text') return encoder.encode(part.text); + if (part.kind === 'base64') return fromBase64(part.base64); + const bytes = files.get(name(part.file)); + if (!bytes) throw new ConnectorError('badInput'); + const start = part.offset ?? 0; + return bytes.slice(start, part.length === undefined ? undefined : start + part.length); + }; + return { + state, + files, + uploads, + stateGet: (key: string) => Promise.resolve(structuredClone((state.get(key) as T | undefined) ?? null)), + stateSet: (key, value) => (state.set(key, structuredClone(value)), Promise.resolve()), + stateDelete: (key) => (state.delete(key), Promise.resolve()), + async setClient(connector, clientId) { + const list = await connectors().list(); + const info = list.find((item) => item.id === connector); + if (!info || clientId.trim() === '' || /\s/.test(clientId)) throw new ConnectorError('badInput'); + return info.state.kind === 'needsSetup' ? { ...info, state: { kind: 'notConnected' } } : info; + }, + tempDir: () => Promise.resolve(FOLDER), + tempWrite(file, part) { + const bytes = partBytes(part); + files.set(name(file), bytes); + return Promise.resolve({ path: `${FOLDER}/${name(file)}`, bytes: bytes.length }); + }, + tempRead(file) { + const bytes = files.get(name(file)); + return bytes ? Promise.resolve(bytes) : Promise.reject(new ConnectorError('badInput')); + }, + tempRemove: (file) => (files.delete(name(file)), Promise.resolve()), + async download(connector, access, request, file) { + const response = await connectors().request(connector, access, request); + const ok = response.status >= 200 && response.status < 300; + const bytes = ok ? encoder.encode(response.body) : new Uint8Array(); + if (ok) files.set(name(file), bytes); + return { + status: response.status, + contentType: response.contentType, + bytes: bytes.length, + path: `${FOLDER}/${name(file)}`, + body: ok ? '' : response.body, + }; + }, + async upload(connector, access, request, parts, contentType) { + const joined = parts.map(partBytes); + const all = new Uint8Array(joined.reduce((sum, one) => sum + one.length, 0)); + let at = 0; + for (const one of joined) { + all.set(one, at); + at += one.length; + } + const body = new TextDecoder().decode(all); + uploads.push({ connector, url: request.url, contentType, body }); + return connectors().request(connector, access, { ...request, body, contentType }); + }, + fetchPreview: (url) => + Promise.resolve(options.preview?.(url) ?? { status: 404, contentType: null, bytes: new Uint8Array() }), + async uploadPublic(connector, url, parts, contentType) { + const joined = parts.map(partBytes); + const all = new Uint8Array(joined.reduce((sum, one) => sum + one.length, 0)); + let at = 0; + for (const one of joined) { + all.set(one, at); + at += one.length; + } + const body = new TextDecoder().decode(all); + uploads.push({ connector, url, contentType, body }); + return connectors().request(connector, [], { method: 'POST', url, body, contentType }); + }, + }; +} diff --git a/app/src/features/accounts/flags.ts b/app/src/features/accounts/flags.ts new file mode 100644 index 00000000..00c68c5d --- /dev/null +++ b/app/src/features/accounts/flags.ts @@ -0,0 +1,38 @@ +// The flags of the account and media features: what a connected account unlocks (sync, share, send, import, meeting +// notes) and the media features that came with them (players, live embeds, a narrated video). Each is on in +// development, nightly, and Beta builds, and off in Stable until it has had its time in Beta. app/flags.ts joins +// this list to the others. +import type { FlagDef, FlagId } from '../../app/flags'; + +export type AccountsFlagId = Extract< + FlagId, + `accounts.${string}` | 'page.mediaEmbeds' | 'page.liveEmbeds' | 'audio.exportVideo' +>; + +const ISSUES = 'https://github.com/XrxcGH/OpenNote/issues?q=label%3Aflag%3A'; +const beta = { dev: true, nightly: true, beta: true, stable: false }; + +const flag = (id: AccountsFlagId, description: string): FlagDef => ({ + id, + description, + issue: `${ISSUES}${encodeURIComponent(id)}`, + enabled: beta, +}); + +export const ACCOUNTS_FLAGS: readonly FlagDef[] = [ + flag('accounts.readwise', 'Sync Readwise: bring highlights into a notebook.'), + flag('accounts.meetings', 'New meeting note from a calendar event or an .ics file.'), + flag('accounts.share', 'Share a page to Slack or Teams.'), + flag('accounts.lms', 'Canvas, Moodle, and Classroom courses and assignments, and Submit page as PDF.'), + flag('accounts.sendTo', 'Send to Google Drive or OneDrive, and Update the copy.'), + flag('accounts.googleOffice', 'Import and export Google Docs, Sheets, and Slides.'), + flag('accounts.onenoteGraph', 'Import OneNote notebooks through Microsoft Graph.'), + flag('accounts.tasksSync', 'Two-way sync with Microsoft To Do and Google Tasks.'), + flag('accounts.cloudSync', 'Dropbox, Box, and WebDAV: send pages and sync a folder.'), + flag('accounts.penSync', 'Pen sets, colors, and toolbar layout synced across devices.'), + flag('accounts.youtubeUpload', 'Upload a recording video to YouTube as private or unlisted.'), + flag('accounts.captions', 'Transcripts from YouTube and Vimeo captions.'), + flag('page.mediaEmbeds', 'Player blocks for YouTube, Vimeo, and podcast links, with notes stamped to the playhead.'), + flag('page.liveEmbeds', 'Live embeds for Desmos, GeoGebra, Figma, Miro, and Lucidchart.'), + flag('audio.exportVideo', 'Export a recording as a narrated video.'), +]; diff --git a/app/src/features/accounts/googleOffice/commands.ts b/app/src/features/accounts/googleOffice/commands.ts new file mode 100644 index 00000000..abe468f3 --- /dev/null +++ b/app/src/features/accounts/googleOffice/commands.ts @@ -0,0 +1,65 @@ +// Import from Google Docs, Sheets, or Slides, and Export to Google Docs, Sheets, or Slides. Import asks which kind and +// which file, has Drive export it, and opens Import notes at the review. Export is Send to Google Drive with the type +// already chosen, so the file is converted to Google's own type as Drive takes it. +import { t } from '../../../strings/t'; +import { announce } from '../../../ui'; +import { connectors } from '../../connectors'; +import { importFile } from '../../interop'; +import { accountsHost } from '../host'; +import { attempt, isConnected, tellNotConnected } from '../run'; +import { pickOne } from '../ui/prompts'; +import type { Choice } from '../ui/prompts'; +import { sendTo } from '../sendto/commands'; +import type { SendFormat } from '../sendto/types'; +import { fetchDoc, recentDocs } from './import'; +import type { DriveDoc, OfficeKind } from './import'; + +export async function importFromGoogle(): Promise { + if (!(await isConnected('google'))) return tellNotConnected('Google'); + const kind = await pickOne({ + title: t('accounts.googleOffice.kind.title'), + description: t('accounts.googleOffice.kind.description'), + confirmLabel: t('accounts.googleOffice.kind.confirm'), + choices: [ + { id: 'docs', label: t('accounts.googleOffice.kind.docs'), value: 'docs' }, + { + id: 'sheets', + label: t('accounts.googleOffice.kind.sheets'), + detail: t('accounts.googleOffice.kind.sheetsDetail'), + value: 'sheets', + }, + { + id: 'slides', + label: t('accounts.googleOffice.kind.slides'), + detail: t('accounts.googleOffice.kind.slidesDetail'), + value: 'slides', + }, + ], + }); + if (!kind) return; + await attempt('Google', async () => { + const docs = await recentDocs(connectors(), kind); + const choices: Choice[] = docs.map((doc) => ({ + id: doc.id, + label: doc.name || t('accounts.googleOffice.untitled'), + detail: doc.modified ? new Date(doc.modified).toLocaleDateString() : undefined, + value: doc, + })); + const doc = await pickOne({ + title: t('accounts.googleOffice.file.title'), + description: t('accounts.googleOffice.file.description'), + confirmLabel: t('accounts.googleOffice.file.confirm'), + choices, + empty: t('accounts.googleOffice.file.none'), + }); + if (!doc) return; + announce(t('accounts.googleOffice.fetching', { name: doc.name })); + const path = await fetchDoc(accountsHost(), kind, doc); + importFile(path); + }); +} + +/** Sends the open page, section, or notebook to Google Drive as a Google Doc, Sheet, or Slides file. */ +export function exportToGoogle(format: Extract): Promise { + return sendTo('drive', { format }); +} diff --git a/app/src/features/accounts/googleOffice/googleOffice.test.ts b/app/src/features/accounts/googleOffice/googleOffice.test.ts new file mode 100644 index 00000000..10021515 --- /dev/null +++ b/app/src/features/accounts/googleOffice/googleOffice.test.ts @@ -0,0 +1,106 @@ +// Google Docs, Sheets, and Slides against a mock Drive: the recent files of each kind, the export Drive makes of one, a +// refusal said, and an Office file sent to be converted to a Google Sheet or Slides file. +import { describe, expect, it } from 'vitest'; +import { addDrive } from '../../../../../tests/mock-servers/storage'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { ConnectorHttpError, createFakeConnectors } from '../../connectors'; +import { createFakeAccountsHost } from '../fakeHost'; +import { driveDestination } from '../sendto/drive'; +import { sendFile } from '../sendto/commands'; +import { formatsFor } from '../sendto/commands'; +import { OFFICE, officeOf, XLSX } from '../sendto/types'; +import { fetchDoc, KINDS, recentDocs } from './import'; + +const docs = [ + { + id: 'd1', + name: 'Cell lecture', + mimeType: KINDS.docs.google, + modified: '2026-10-02T10:00:00Z', + exported: 'DOCX-BYTES', + }, + { id: 'd2', name: 'Old essay', mimeType: KINDS.docs.google, modified: '2026-09-01T10:00:00Z', exported: 'DOCX-OLD' }, + { + id: 's1', + name: 'Lab results', + mimeType: KINDS.sheets.google, + modified: '2026-10-03T10:00:00Z', + exported: 'XLSX-BYTES', + }, + { + id: 'p1', + name: 'Mitosis / Meiosis', + mimeType: KINDS.slides.google, + modified: '2026-10-01T10:00:00Z', + exported: 'PPTX-BYTES', + }, +]; + +async function rig(server: MockServer) { + const connectors = createFakeConnectors({ connected: { google: 'sam@example.com' }, respond: server.respond }); + return { client: connectors.client, host: createFakeAccountsHost(() => connectors.client) }; +} + +describe('importing from Google', () => { + it('lists the newest files of one kind', async () => { + const server = new MockServer(); + addDrive(server, [], 100, docs); + const { client } = await rig(server); + expect((await recentDocs(client, 'docs')).map((one) => one.name)).toEqual(['Cell lecture', 'Old essay']); + expect((await recentDocs(client, 'sheets')).map((one) => one.name)).toEqual(['Lab results']); + expect(await recentDocs(client, 'docs', 1)).toHaveLength(1); + }); + + it('has Drive export the file as an Office file and keeps it in the transfer folder', async () => { + const server = new MockServer(); + addDrive(server, [], 100, docs); + const { client, host } = await rig(server); + const [sheet] = await recentDocs(client, 'sheets'); + if (!sheet) throw new Error('no sheet'); + const path = await fetchDoc(host, 'sheets', sheet); + expect(path.endsWith('Lab results.xlsx')).toBe(true); + expect(new TextDecoder().decode(host.files.get('Lab results.xlsx'))).toBe('XLSX-BYTES'); + const asked = server.requests.find((one) => one.path.endsWith('/export')); + expect(asked?.query.mimeType).toBe(KINDS.sheets.exportAs); + }); + + it('makes a safe file name from a title with a slash', async () => { + const server = new MockServer(); + addDrive(server, [], 100, docs); + const { client, host } = await rig(server); + const [slides] = await recentDocs(client, 'slides'); + if (!slides) throw new Error('no slides'); + expect((await fetchDoc(host, 'slides', slides)).endsWith('Mitosis Meiosis.pptx')).toBe(true); + }); + + it('says so when Drive refuses the export', async () => { + const server = new MockServer(); + addDrive(server, [], 100, []); + const { host } = await rig(server); + await expect(fetchDoc(host, 'docs', { id: 'gone', name: 'Gone', modified: '' })).rejects.toBeInstanceOf( + ConnectorHttpError, + ); + }); +}); + +describe('exporting to Google', () => { + it('converts an Excel file to a Google Sheet and drops the extension from the name', async () => { + const server = new MockServer(); + const { files } = addDrive(server, []); + const { client, host } = await rig(server); + await host.tempWrite('Tables.xlsx', { kind: 'text', text: 'XLSX-FILE' }); + await sendFile( + driveDestination({ client, host }), + { file: 'Tables.xlsx', name: 'Tables.xlsx', mime: XLSX }, + { node: 'n', scope: 'page', format: 'gsheet', folder: null, folderName: 'My Drive', host }, + ); + expect(files.get('file1')).toMatchObject({ name: 'Tables', mimeType: OFFICE.xlsx.convertTo }); + }); + + it('knows which Office file each Google type is made from', () => { + expect(officeOf('gslides')).toEqual({ type: 'pptx', convert: true }); + expect(officeOf('pdf')).toBeNull(); + expect(formatsFor('drive', 'section')).toEqual(['docx', 'xlsx', 'pptx', 'gdoc', 'gsheet', 'gslides']); + expect(formatsFor('onedrive', 'page')).toEqual(['pdf', 'docx', 'xlsx', 'pptx']); + }); +}); diff --git a/app/src/features/accounts/googleOffice/import.ts b/app/src/features/accounts/googleOffice/import.ts new file mode 100644 index 00000000..5df3c3b3 --- /dev/null +++ b/app/src/features/accounts/googleOffice/import.ts @@ -0,0 +1,69 @@ +// Google Docs, Sheets, and Slides into OpenNote: the recent files of a kind in the person's Drive, and one of them +// exported by Drive as a Word, Excel, or PowerPoint file in the transfer folder. The importers OpenNote already has for +// those files do the rest (Sheets become table pages, Slides become a page for each slide to write on), and the person +// sees their review and loss report before anything is added. +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import { fileNameOf } from '../sendto/render'; + +export type OfficeKind = 'docs' | 'sheets' | 'slides'; + +export const KINDS: Record = { + docs: { + google: 'application/vnd.google-apps.document', + exportAs: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + extension: 'docx', + }, + sheets: { + google: 'application/vnd.google-apps.spreadsheet', + exportAs: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + extension: 'xlsx', + }, + slides: { + google: 'application/vnd.google-apps.presentation', + exportAs: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + extension: 'pptx', + }, +}; + +const FILES = 'https://www.googleapis.com/drive/v3/files'; + +export interface DriveDoc { + id: string; + name: string; + modified: string; +} + +interface FileList { + files?: { id: string; name: string; modifiedTime?: string }[]; +} + +/** The files of a kind that changed most recently, newest first. */ +export async function recentDocs(client: ConnectorsClient, kind: OfficeKind, limit = 50): Promise { + const reply = await requestConnector( + 'google', + ['driveRead'], + { + url: FILES, + query: { + q: `mimeType='${KINDS[kind].google}' and trashed=false`, + fields: 'files(id,name,modifiedTime)', + orderBy: 'modifiedTime desc', + pageSize: limit, + }, + }, + client, + ); + return (reply.data?.files ?? []).map((one) => ({ id: one.id, name: one.name, modified: one.modifiedTime ?? '' })); +} + +/** Has Drive export the file and keeps it in the transfer folder. Answers the file's path. */ +export async function fetchDoc(host: AccountsHost, kind: OfficeKind, doc: DriveDoc): Promise { + const { extension, exportAs } = KINDS[kind]; + const url = `${FILES}/${encodeURIComponent(doc.id)}/export?mimeType=${encodeURIComponent(exportAs)}`; + const name = fileNameOf(doc.name, extension); + const got = await host.download('google', ['driveRead'], { method: 'GET', url }, name); + if (got.status < 200 || got.status >= 300) throw new ConnectorHttpError(got.status, got.body); + return got.path; +} diff --git a/app/src/features/accounts/host.ts b/app/src/features/accounts/host.ts new file mode 100644 index 00000000..5f6dbe53 --- /dev/null +++ b/app/src/features/accounts/host.ts @@ -0,0 +1,79 @@ +// What the account features need from the host beyond a request: small saved state, files in the transfer folder, +// and a file moved to or from a service without its bytes crossing into the interface. The shell side is +// `accounts_call` (app/src-tauri/src/connectors/accounts.rs). The web platform and the tests use `fakeHost.ts`. + +import { commandContext } from '../../commands/registry'; +import type { ConnectorInfo, ConnectorRequest, ConnectorResponse } from '../connectors'; + +/** A piece of an upload's body. The pieces are joined in order. */ +export type UploadPart = + | { kind: 'text'; text: string } + | { kind: 'base64'; base64: string } + /** A file in the transfer folder, or a slice of one. */ + | { kind: 'file'; file: string; offset?: number; length?: number }; + +export interface Downloaded { + status: number; + contentType: string | null; + bytes: number; + /** Where the file is, for a step that reads it (an import). */ + path: string; + /** The service's words when the answer was a failure; the file is written only on success. */ + body: string; +} + +export interface AccountsHost { + stateGet(name: string): Promise; + stateSet(name: string, value: unknown): Promise; + stateDelete(name: string): Promise; + /** Saves the client ID (and secret) typed on a card. Resolves to the connector with its new state. */ + setClient(connector: string, clientId: string, clientSecret?: string): Promise; + /** The transfer folder, made if it is missing. */ + tempDir(): Promise; + tempWrite(file: string, part: UploadPart): Promise<{ path: string; bytes: number }>; + tempRead(file: string): Promise; + tempRemove(file: string): Promise; + /** Fetches the address and writes the body to `file` in the transfer folder. */ + download(connector: string, access: readonly string[], request: ConnectorRequest, file: string): Promise; + /** Sends the joined parts as the body of the request. */ + upload( + connector: string, + access: readonly string[], + request: ConnectorRequest, + parts: readonly UploadPart[], + contentType: string, + ): Promise; + /** + * Sends the joined parts as a POST to an address that the service itself gave in an earlier answer (an upload + * address on its storage), with no token. The connector must be connected. Only a plain https address on a name + * (not a number, and not this computer) is sent to. + */ + uploadPublic( + connector: string, + url: string, + parts: readonly UploadPart[], + contentType: string, + ): Promise; + /** + * Reads a link's title or picture from a short list of public hosts (the players' and embeds' own), with no token. + * Work offline refuses it. + */ + fetchPreview(url: string): Promise<{ status: number; contentType: string | null; bytes: Uint8Array }>; +} + +/** The host's account calls. */ +export function accountsHost(): AccountsHost { + return commandContext('menu').platform.accounts; +} + +/** Bytes as base64 text. */ +export function base64Of(bytes: Uint8Array): string { + let binary = ''; + for (let at = 0; at < bytes.length; at += 0x8000) binary += String.fromCharCode(...bytes.subarray(at, at + 0x8000)); + return btoa(binary); +} + +/** Bytes as an upload part. */ +export function bytesPart(bytes: Uint8Array): UploadPart { + return { kind: 'base64', base64: base64Of(bytes) }; +} diff --git a/app/src/features/accounts/index.ts b/app/src/features/accounts/index.ts new file mode 100644 index 00000000..c46cda1e --- /dev/null +++ b/app/src/features/accounts/index.ts @@ -0,0 +1,13 @@ +// The account features: what a connected account unlocks (meeting notes, sharing, sending, syncing, imports) and the +// media features that came with them. The platforms import the host's types and the in-memory host from here. + +export { createFakeAccountsHost } from './fakeHost'; +export type { FakeAccountsHost, FakeHostOptions } from './fakeHost'; +export { accountsHost, base64Of, bytesPart } from './host'; +export type { AccountsHost, Downloaded, UploadPart } from './host'; +export { askText, pickOne } from './ui/prompts'; +export type { UploadRequest } from './youtube/commands'; +/** Uploads a recording's video to YouTube. The code loads when it is first called. */ +export const uploadRecordingToYoutube = (...args: Parameters) => + import('./youtube/commands').then((module) => module.uploadRecording(...args)); +export { failureText, isConnected, tellNotConnected } from './run'; diff --git a/app/src/features/accounts/lms/bring.ts b/app/src/features/accounts/lms/bring.ts new file mode 100644 index 00000000..19bc29be --- /dev/null +++ b/app/src/features/accounts/lms/bring.ts @@ -0,0 +1,127 @@ +// Bringing a course in: the course becomes a section of a notebook, each assignment a page in it (with the due date, +// the link, and the instructions at the top), and the assignments with a due date go to Upcoming. Doing it again +// updates what the school changed and leaves everything the person wrote on a page alone. + +import type { PagesClient } from '../../../platform/types'; +import type { NodeId, NotesService } from '../../../services/notes/types'; +import { t } from '../../../strings/t'; +import { dueAt } from '../../tools'; +import type { UpcomingItem } from '../../tools'; +import type { AccountsHost } from '../host'; +import { cleanTitle, findOrCreate, replaceHeader, appendMarkdown, createPage } from '../notebook'; +import { loadState, saveState } from '../state'; +import type { Assignment, Course } from './types'; + +export const STATE_NAME = 'lms.courses'; + +interface AssignmentState { + page: string; + /** The blocks the sync wrote at the top of the page. */ + header: string[]; +} + +interface CourseState { + section: string; + assignments: Record; +} + +interface LmsState { + courses: Record; +} + +export interface BringDeps { + notes: NotesService; + pages: PagesClient; + host: AccountsHost; + /** Puts items in Upcoming, replacing those of the same source. */ + setUpcoming(source: string, items: UpcomingItem[]): { added: number; removed: number; changed: number }; + zone?: string; +} + +export interface BringResult { + pages: number; + dated: number; + section: NodeId; +} + +const keyOf = (course: Course): string => `${course.service}:${course.id}`; + +/** The words for when something is due, in the viewer's zone. */ +export function dueText(iso: string | null, zone: string): string { + if (!iso) return t('accounts.lms.page.noDue'); + const when = new Date(iso); + return t('accounts.lms.page.due', { + when: when.toLocaleString(undefined, { + weekday: 'short', + month: 'short', + day: 'numeric', + hour: 'numeric', + minute: '2-digit', + timeZone: zone, + }), + }); +} + +function header(assignment: Assignment, serviceLabel: string, zone: string): string[] { + const blocks = [dueText(assignment.due, zone)]; + if (assignment.submitted) blocks.push(t('accounts.lms.page.submitted')); + if (assignment.url) blocks.push(t('accounts.lms.page.open', { service: serviceLabel, link: assignment.url })); + if (assignment.text) blocks.push(assignment.text); + return blocks; +} + +/** Writes the course and its assignments into the notebook and Upcoming. */ +export async function bringCourse( + deps: BringDeps, + notebook: NodeId, + course: Course, + serviceLabel: string, + assignments: readonly Assignment[], +): Promise { + const zone = deps.zone ?? Intl.DateTimeFormat().resolvedOptions().timeZone; + const state = await loadState(STATE_NAME, { courses: {} }, deps.host); + const key = keyOf(course); + const known = state.courses[key]; + const knownSection = known ? await deps.notes.get(known.section as NodeId) : null; + const section = + knownSection ?? (await findOrCreate(deps.notes, notebook, 'section', cleanTitle(course.name, serviceLabel))); + const courseState: CourseState = { section: section.id, assignments: knownSection ? (known?.assignments ?? {}) : {} }; + state.courses[key] = courseState; + + for (const assignment of assignments) { + const blocks = header(assignment, serviceLabel, zone); + const earlier = courseState.assignments[assignment.id]; + const existing = earlier ? await deps.notes.get(earlier.page as NodeId) : null; + if (existing && earlier) { + earlier.header = await replaceHeader(deps.pages, earlier.page, earlier.header, blocks); + // The heading stays in step with the school when the assignment is renamed. + if (existing.title !== cleanTitle(assignment.title, assignment.title)) { + await deps.notes.rename(existing.id, cleanTitle(assignment.title, t('accounts.lms.untitled'))); + } + } else { + const page = await createPage( + deps.notes, + deps.pages, + section.id, + assignment.title || t('accounts.lms.untitled'), + [], + ); + const made = await appendMarkdown(deps.pages, page.id, blocks); + courseState.assignments[assignment.id] = { page: page.id, header: made }; + } + await saveState(STATE_NAME, state, deps.host); + } + + // Upcoming has the ones that are due and not yet handed in. + const items: UpcomingItem[] = assignments + .filter((one) => one.due !== null) + .map((one) => ({ + id: `${keyOf(course)}:${one.id}`, + title: `${one.title || t('accounts.lms.untitled')} (${course.name})`, + due: dueAt(new Date(one.due ?? 0).getTime(), zone), + done: one.submitted, + kind: 'task', + })); + deps.setUpcoming(keyOf(course), items); + return { pages: assignments.length, dated: items.length, section: section.id }; +} diff --git a/app/src/features/accounts/lms/canvas.ts b/app/src/features/accounts/lms/canvas.ts new file mode 100644 index 00000000..00470c0c --- /dev/null +++ b/app/src/features/accounts/lms/canvas.ts @@ -0,0 +1,156 @@ +// Canvas: the courses the person is in, their assignments, and handing in a file. The person's token is the host's. +// Canvas pages its lists with a Link header, which the interface does not see, so each list asks for a page number +// until a page comes back short. + +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import { bytesPart } from '../host'; +import { agendaText } from '../meetings/text'; +import type { Assignment, Course, HandIn, HandInResult, LmsApi, LmsDeps } from './types'; + +const PAGE = 50; +const MAX_PAGES = 40; + +/** The school's address the person connected with. */ +export async function schoolAddress(client: ConnectorsClient, connector: string): Promise { + const info = (await client.list()).find((item) => item.id === connector); + if (!info?.baseUrl) throw new ConnectorHttpError(400, JSON.stringify({ message: 'The school address is missing.' })); + return info.baseUrl.replace(/\/+$/, ''); +} + +interface CanvasCourse { + id: number; + name?: string; + course_code?: string; +} + +interface CanvasAssignment { + id: number; + name?: string; + due_at?: string | null; + html_url?: string; + description?: string | null; + submission?: { workflow_state?: string; submitted_at?: string | null } | null; +} + +async function listPages( + client: ConnectorsClient, + access: string, + url: string, + query: Record, +): Promise { + const all: T[] = []; + for (let page = 1; page <= MAX_PAGES; page += 1) { + const reply = await requestConnector( + 'canvas', + [access], + { url, query: { ...query, per_page: PAGE, page } }, + client, + ); + const items = Array.isArray(reply.data) ? reply.data : []; + all.push(...items); + if (items.length < PAGE) break; + } + return all; +} + +export function canvasApi({ client, host }: LmsDeps): LmsApi { + return { + service: 'canvas', + label: 'Canvas', + async courses() { + const base = await schoolAddress(client, 'canvas'); + const found = await listPages(client, 'canvasRead', `${base}/api/v1/courses`, { + enrollment_state: 'active', + }); + return found + .filter((course) => course.name) + .map((course) => ({ service: 'canvas' as const, id: String(course.id), name: course.name ?? '' })); + }, + async assignments(course: Course) { + const base = await schoolAddress(client, 'canvas'); + const found = await listPages( + client, + 'canvasRead', + `${base}/api/v1/courses/${encodeURIComponent(course.id)}/assignments`, + { 'include[]': 'submission', order_by: 'due_at' }, + ); + return found.map((one): Assignment => ({ + service: 'canvas', + courseId: course.id, + id: String(one.id), + title: one.name ?? '', + due: one.due_at ?? null, + url: one.html_url ?? null, + text: agendaText(one.description ?? '', true), + submitted: ['submitted', 'graded', 'pending_review'].includes(one.submission?.workflow_state ?? ''), + })); + }, + async handIn(assignment: Assignment, file: HandIn): Promise { + const base = await schoolAddress(client, 'canvas'); + const root = `${base}/api/v1/courses/${encodeURIComponent(assignment.courseId)}/assignments/${encodeURIComponent(assignment.id)}`; + // 1. Canvas says where the file goes. + const slot = ( + await requestConnector<{ upload_url?: string; upload_params?: Record }>( + 'canvas', + ['canvasSubmit'], + { + url: `${root}/submissions/self/files`, + form: { name: file.name, size: String(file.bytes.length), content_type: file.mime }, + }, + client, + ) + ).data; + if (!slot?.upload_url) throw new ConnectorHttpError(502, ''); + // 2. The file goes to that address, as a form with the fields Canvas listed and the file last. + const boundary = `----opennote${Math.random().toString(36).slice(2)}`; + const parts = Object.entries(slot.upload_params ?? {}).map( + ([name, value]) => + ({ + kind: 'text', + text: `--${boundary}\r\nContent-Disposition: form-data; name="${name.replace(/"/g, '')}"\r\n\r\n${value}\r\n`, + }) as const, + ); + const upload = await host.uploadPublic( + 'canvas', + slot.upload_url, + [ + ...parts, + { + kind: 'text', + // checks-disable-next-line length: one multipart body, with its line breaks kept visible + text: `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${file.name.replace(/"/g, '')}"\r\nContent-Type: ${file.mime}\r\n\r\n`, + }, + bytesPart(file.bytes), + { kind: 'text', text: `\r\n--${boundary}--\r\n` }, + ], + `multipart/form-data; boundary=${boundary}`, + ); + // Canvas answers with the file, or with a redirect that the app confirms with the token. + let uploaded: { id?: number } | undefined; + if (upload.status >= 300 && upload.status < 400 && upload.location) { + uploaded = ( + await requestConnector<{ id?: number }>('canvas', ['canvasSubmit'], { url: upload.location }, client) + ).data; + } else if (upload.status >= 200 && upload.status < 300) { + try { + uploaded = JSON.parse(upload.body) as { id?: number }; + } catch { + uploaded = undefined; + } + } else throw new ConnectorHttpError(upload.status, upload.body); + if (!uploaded?.id) throw new ConnectorHttpError(502, ''); + // 3. The file is handed in. + await requestConnector( + 'canvas', + ['canvasSubmit'], + { + url: `${root}/submissions`, + form: { 'submission[submission_type]': 'online_upload', 'submission[file_ids][]': String(uploaded.id) }, + }, + client, + ); + return { kind: 'submitted' }; + }, + }; +} diff --git a/app/src/features/accounts/lms/classroom.ts b/app/src/features/accounts/lms/classroom.ts new file mode 100644 index 00000000..d7d2b15c --- /dev/null +++ b/app/src/features/accounts/lms/classroom.ts @@ -0,0 +1,141 @@ +// Google Classroom: the courses the person is in and their assignments. Classroom lets a student attach a file only +// to work that the same app made, so handing in saves the PDF in the person's Drive and gives its link, and the +// person attaches it in Classroom and turns it in there. + +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import { bytesPart } from '../host'; +import { agendaText } from '../meetings/text'; +import type { Assignment, Course, HandIn, HandInResult, LmsApi, LmsDeps } from './types'; + +const CLASSROOM = 'https://classroom.googleapis.com/v1'; +const MAX_PAGES = 40; + +interface Work { + id: string; + title?: string; + description?: string; + alternateLink?: string; + state?: string; + dueDate?: { year: number; month: number; day: number }; + dueTime?: { hours?: number; minutes?: number }; +} + +interface Submission { + courseWorkId: string; + state?: string; +} + +async function pages( + client: ConnectorsClient, + access: string, + url: string, + key: string, + query: Record = {}, +): Promise { + const all: T[] = []; + let token = ''; + for (let page = 0; page < MAX_PAGES; page += 1) { + const reply = await requestConnector>( + 'google', + [access], + { url, query: { ...query, pageSize: 100, pageToken: token || null } }, + client, + ); + all.push(...((reply.data?.[key] as T[] | undefined) ?? [])); + token = (reply.data?.nextPageToken as string | undefined) ?? ''; + if (!token) break; + } + return all; +} + +/** Classroom's due date and time (UTC) as an ISO time, or null. */ +export function dueOf(work: Work): string | null { + const day = work.dueDate; + if (!day) return null; + const time = work.dueTime ?? {}; + return new Date(Date.UTC(day.year, day.month - 1, day.day, time.hours ?? 0, time.minutes ?? 0)).toISOString(); +} + +function webLink(body: string): string { + try { + return (JSON.parse(body) as { webViewLink?: string }).webViewLink ?? ''; + } catch { + return ''; + } +} + +export function classroomApi({ client, host }: LmsDeps): LmsApi { + return { + service: 'classroom', + label: 'Google Classroom', + async courses() { + const found = await pages<{ id: string; name?: string; section?: string }>( + client, + 'classroomRead', + `${CLASSROOM}/courses`, + 'courses', + { courseStates: 'ACTIVE', studentId: 'me' }, + ); + return found.map((course) => ({ + service: 'classroom' as const, + id: course.id, + name: course.section ? `${course.name ?? ''} (${course.section})` : (course.name ?? ''), + })); + }, + async assignments(course: Course) { + const base = `${CLASSROOM}/courses/${encodeURIComponent(course.id)}`; + const work = await pages(client, 'classroomRead', `${base}/courseWork`, 'courseWork'); + let handedIn = new Set(); + try { + const mine = await pages( + client, + 'classroomRead', + `${base}/courseWork/-/studentSubmissions`, + 'studentSubmissions', + { userId: 'me' }, + ); + handedIn = new Set( + mine.filter((one) => one.state === 'TURNED_IN' || one.state === 'RETURNED').map((one) => one.courseWorkId), + ); + } catch { + // Without the submissions, nothing is marked as handed in. + } + return work + .filter((one) => one.state === undefined || one.state === 'PUBLISHED') + .map((one): Assignment => ({ + service: 'classroom', + courseId: course.id, + id: one.id, + title: one.title ?? '', + due: dueOf(one), + url: one.alternateLink ?? null, + text: agendaText(one.description ?? '', false), + submitted: handedIn.has(one.id), + })); + }, + async handIn(_assignment: Assignment, file: HandIn): Promise { + const boundary = `opennote${Math.random().toString(36).slice(2)}`; + const sent = await host.upload( + 'google', + ['driveFiles'], + { + method: 'POST', + url: 'https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&fields=id,webViewLink', + }, + [ + { + kind: 'text', + // checks-disable-next-line length: one multipart body, with its line breaks kept visible + text: `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${JSON.stringify({ name: file.name })}\r\n--${boundary}\r\nContent-Type: ${file.mime}\r\n\r\n`, + }, + bytesPart(file.bytes), + { kind: 'text', text: `\r\n--${boundary}--` }, + ], + `multipart/related; boundary=${boundary}`, + ); + if (sent.status < 200 || sent.status >= 300) throw new ConnectorHttpError(sent.status, sent.body); + return { kind: 'saved', link: webLink(sent.body) }; + }, + }; +} diff --git a/app/src/features/accounts/lms/commands.ts b/app/src/features/accounts/lms/commands.ts new file mode 100644 index 00000000..eb510e4c --- /dev/null +++ b/app/src/features/accounts/lms/commands.ts @@ -0,0 +1,176 @@ +// Bring in course assignments and Submit page as PDF: choose the school account, the course, and (to hand in) the +// assignment. Nothing is handed in until the last button. + +import { getLocation } from '../../../app/location'; +import { commandContext } from '../../../commands/registry'; +import type { NodeId } from '../../../services/notes/types'; +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { navigate } from '../../../app/location'; +import { shownMounted } from '../../page'; +import { renderShownPage } from '../../pages'; +import { setSourceItems } from '../../tools'; +import { accountsHost } from '../host'; +import { services } from '../notebook'; +import { attempt, isConnected } from '../run'; +import { pickOne } from '../ui/prompts'; +import type { Choice } from '../ui/prompts'; +import { bringCourse } from './bring'; +import type { BringDeps } from './bring'; +import { canvasApi } from './canvas'; +import { classroomApi } from './classroom'; +import { moodleApi } from './moodle'; +import { CONNECTOR_OF } from './types'; +import type { Assignment, Course, LmsApi, LmsService } from './types'; + +const ORDER: readonly LmsService[] = ['canvas', 'moodle', 'classroom']; + +function apiFor(service: LmsService): LmsApi { + const deps = { client: commandContext('palette').platform.connectors, host: accountsHost() }; + return service === 'canvas' ? canvasApi(deps) : service === 'moodle' ? moodleApi(deps) : classroomApi(deps); +} + +/** The connected services, and the one the person picks when there are several. */ +async function chooseService(): Promise { + const connected: LmsService[] = []; + for (const service of ORDER) if (await isConnected(CONNECTOR_OF[service])) connected.push(service); + if (connected.length === 0) { + showToast({ + message: t('accounts.lms.notConnected'), + action: { + label: t('accounts.common.openSettings'), + run: () => navigate({ view: 'settings', section: 'connectors' }), + }, + }); + return null; + } + const only = connected.length === 1 ? connected[0] : undefined; + if (only) return apiFor(only); + const choices: Choice[] = connected.map((service) => ({ + id: service, + label: apiFor(service).label, + value: service, + })); + const picked = await pickOne({ + title: t('accounts.lms.service.title'), + description: t('accounts.lms.service.description'), + choices, + confirmLabel: t('accounts.lms.service.confirm'), + }); + return picked ? apiFor(picked) : null; +} + +async function notebookHere(): Promise { + const here = getLocation(); + if (here.view === 'workspace' && here.notebookId) return here.notebookId; + return (await services().notes.listNotebooks())[0]?.id ?? null; +} + +export async function bringAssignments(): Promise { + const api = await chooseService(); + if (!api) return; + await attempt(api.label, async () => { + announce(t('accounts.lms.working')); + const courses = await api.courses(); + const choice = await pickOne({ + title: t('accounts.lms.course.title'), + description: t('accounts.lms.course.description'), + confirmLabel: t('accounts.lms.course.confirm'), + empty: t('accounts.lms.course.none'), + choices: [ + ...(courses.length > 1 ? [{ id: 'all', label: t('accounts.lms.course.all'), value: 'all' as const }] : []), + ...courses.map((course) => ({ id: course.id, label: course.name, value: course })), + ], + }); + if (!choice) return; + const notebook = await notebookHere(); + if (!notebook) return; + const { notes, pages } = services(); + const deps: BringDeps = { notes, pages, host: accountsHost(), setUpcoming: setSourceItems }; + const chosen = choice === 'all' ? courses : [choice]; + let pageCount = 0; + let dated = 0; + for (const course of chosen) { + const result = await bringCourse(deps, notebook, course, api.label, await api.assignments(course)); + pageCount += result.pages; + dated += result.dated; + } + const message = t('accounts.lms.done', { courses: chosen.length, pages: pageCount, dated }); + showToast({ message }); + announce(message); + }); +} + +function detailOf(assignment: Assignment): string { + const when = assignment.due + ? new Date(assignment.due).toLocaleString(undefined, { + weekday: 'short', + month: 'short', + day: 'numeric', + hour: 'numeric', + minute: '2-digit', + }) + : ''; + if (assignment.submitted) return t('accounts.lms.assignment.detailSubmitted', { when }).trim(); + return assignment.due ? t('accounts.lms.assignment.detailDue', { when }) : t('accounts.lms.assignment.detailNoDue'); +} + +export async function submitPage(): Promise { + if (!shownMounted.get()) return void showToast({ message: t('accounts.lms.nothing') }); + const api = await chooseService(); + if (!api) return; + await attempt(api.label, async () => { + const courses = await api.courses(); + const course = await pickOne({ + title: t('accounts.lms.course.title'), + confirmLabel: t('accounts.lms.course.submitConfirm'), + empty: t('accounts.lms.course.none'), + choices: courses.map((one) => ({ id: one.id, label: one.name, value: one })), + }); + if (!course) return; + // Work still to hand in comes first, soonest due first. + const list = (await api.assignments(course)).sort( + (a, b) => + Number(a.submitted) - Number(b.submitted) || + (a.due ?? '9999').localeCompare(b.due ?? '9999') || + a.title.localeCompare(b.title), + ); + const assignment = await pickOne({ + title: t('accounts.lms.assignment.title'), + description: t('accounts.lms.assignment.description'), + confirmLabel: t('accounts.lms.assignment.confirm'), + empty: t('accounts.lms.assignment.none'), + choices: list.map((one) => ({ + id: one.id, + label: one.title || t('accounts.lms.untitled'), + detail: detailOf(one), + value: one, + })), + }); + if (!assignment) return; + const rendered = await renderShownPage(commandContext('palette'), 'pdf'); + if (!rendered) return void showToast({ message: t('accounts.lms.failedRender'), tone: 'danger' }); + announce(t('accounts.lms.submitting', { title: assignment.title })); + const result = await api.handIn(assignment, { name: rendered.name, mime: rendered.mime, bytes: rendered.bytes }); + if (result.kind === 'submitted') { + const message = t('accounts.lms.submitted', { title: assignment.title, course: course.name }); + showToast({ message }); + announce(message); + return; + } + let copied = false; + try { + if (result.link) { + await navigator.clipboard.writeText(result.link); + copied = true; + } + } catch { + copied = false; + } + showToast({ + message: copied + ? t('accounts.lms.saved', { title: assignment.title }) + : t('accounts.lms.savedNoLink', { title: assignment.title }), + }); + }); +} diff --git a/app/src/features/accounts/lms/lms.test.ts b/app/src/features/accounts/lms/lms.test.ts new file mode 100644 index 00000000..25e9528e --- /dev/null +++ b/app/src/features/accounts/lms/lms.test.ts @@ -0,0 +1,266 @@ +// Canvas, Moodle, and Classroom against mock services: the course and assignment lists (across pages), the section and +// pages a course becomes, the items for Upcoming, and handing in a file the way each service takes it. +import { describe, expect, it } from 'vitest'; +import { addCanvas, addClassroom, addMoodle } from '../../../../../tests/mock-servers/lms'; +import type { LmsCourse } from '../../../../../tests/mock-servers/lms'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { createMemoryNotesService } from '../../../services/notes/memory'; +import { createMemoryPageService } from '../../../services/pages/memory'; +import type { UpcomingItem } from '../../tools'; +import { ConnectorHttpError, createFakeConnectors } from '../../connectors'; +import { createFakeAccountsHost } from '../fakeHost'; +import { bringCourse } from './bring'; +import { canvasApi } from './canvas'; +import { classroomApi, dueOf } from './classroom'; +import { moodleApi } from './moodle'; + +const bytes = new TextEncoder().encode('%PDF-1.7 homework'); +const file = { name: 'Lab 3.pdf', mime: 'application/pdf', bytes }; + +const courses: LmsCourse[] = [ + { + id: 11, + name: 'Biology 101', + assignments: [ + { id: 1, name: 'Lab 3', due: '2026-10-14T21:00:00Z', description: '

    Write up the results.

    ' }, + { id: 2, name: 'Reading quiz', due: '2026-10-09T15:30:00Z', submitted: true }, + { id: 3, name: 'Extra credit' }, + ], + }, + { id: 12, name: 'Chemistry', assignments: [{ id: 4, name: 'Problem set', due: '2026-10-20T04:00:00Z' }] }, +]; + +async function rig(connector: 'canvas' | 'moodle' | 'google', server: MockServer) { + const connectors = createFakeConnectors({ respond: server.respond }); + if (connector === 'google') { + const fresh = createFakeConnectors({ connected: { google: 'sam@example.com' }, respond: server.respond }); + return { client: fresh.client, host: createFakeAccountsHost(() => fresh.client) }; + } + await connectors.client.connect(connector, { token: 'token-12345678', baseUrl: 'school.example.edu' }); + return { client: connectors.client, host: createFakeAccountsHost(() => connectors.client) }; +} + +function storage() { + const notes = createMemoryNotesService({ seed: 'empty' }); + const pages = createMemoryPageService([], { + missing: (id) => ({ id, title: '', created: '', modified: '', tags: [], view: {}, blocks: [], assets: {} }), + }); + return { notes, pages }; +} + +describe('Canvas', () => { + it('lists courses and assignments across pages, and knows what is handed in', async () => { + const many: LmsCourse = { + id: 20, + name: 'Big course', + assignments: Array.from({ length: 60 }, (_, at) => ({ id: 100 + at, name: `Task ${at}` })), + }; + const server = new MockServer(); + addCanvas(server, 'school.example.edu', [...courses, many]); + const deps = await rig('canvas', server); + const api = canvasApi(deps); + expect((await api.courses()).map((one) => one.name)).toEqual(['Biology 101', 'Chemistry', 'Big course']); + const list = await api.assignments({ service: 'canvas', id: '11', name: 'Biology 101' }); + expect(list.map((one) => [one.title, one.submitted])).toEqual([ + ['Lab 3', false], + ['Reading quiz', true], + ['Extra credit', false], + ]); + expect(list[0]).toMatchObject({ due: '2026-10-14T21:00:00Z', text: 'Write up the results.' }); + const all = await api.assignments({ service: 'canvas', id: '20', name: 'Big course' }); + expect(all).toHaveLength(60); + expect(server.requestsTo('GET', '/api/v1/courses/20/assignments')).toHaveLength(2); + }); + + it('hands a file in with the three steps, sending no token to the storage address', async () => { + const server = new MockServer(); + const canvas = addCanvas(server, 'school.example.edu', courses); + const deps = await rig('canvas', server); + const api = canvasApi(deps); + const result = await api.handIn( + { service: 'canvas', courseId: '11', id: '1', title: 'Lab 3', due: null, url: null, text: '', submitted: false }, + file, + ); + expect(result).toEqual({ kind: 'submitted' }); + expect(canvas.uploads).toEqual([ + { fields: ['filename', 'content_type', 'key', 'file'], fileName: 'Lab 3.pdf', bytes: bytes.length, token: false }, + ]); + expect(canvas.submissions).toEqual([{ course: '11', assignment: '1', fileId: '701' }]); + }); + + it('says a refusal of the file as a failure', async () => { + const server = new MockServer(); + server.route('POST school.example.edu/api/v1/courses/:c/assignments/:a/submissions/self/files', () => ({ + status: 403, + json: { message: 'The assignment is closed.' }, + })); + const api = canvasApi(await rig('canvas', server)); + const error = await api + .handIn( + { service: 'canvas', courseId: '11', id: '1', title: '', due: null, url: null, text: '', submitted: false }, + file, + ) + .catch((one) => one); + expect(error).toBeInstanceOf(ConnectorHttpError); + expect((error as ConnectorHttpError).status).toBe(403); + }); +}); + +describe('Moodle', () => { + it('lists courses and assignments with their hand-in state, and hands a file in', async () => { + const server = new MockServer(); + const moodle = addMoodle(server, 'school.example.edu', courses); + const api = moodleApi(await rig('moodle', server)); + expect((await api.courses()).map((one) => one.name)).toEqual(['Biology 101', 'Chemistry']); + const list = await api.assignments({ service: 'moodle', id: '11', name: 'Biology 101' }); + expect(list.map((one) => [one.title, one.submitted, one.due !== null])).toEqual([ + ['Lab 3', false, true], + ['Reading quiz', true, true], + ['Extra credit', false, false], + ]); + expect(list[0]?.url).toBe('https://school.example.edu/mod/assign/view.php?id=1001'); + const first = list[0]; + if (!first) throw new Error('no assignment'); + await api.handIn(first, file); + const upload = moodle.calls.find((call) => call.fn === 'core_files_upload'); + expect(upload?.fields.filename).toBe('Lab 3.pdf'); + expect(atob(upload?.fields.filecontent ?? '')).toBe('%PDF-1.7 homework'); + expect(moodle.calls.map((call) => call.fn).slice(-2)).toEqual([ + 'mod_assign_save_submission', + 'mod_assign_submit_for_grading', + ]); + expect(moodle.submitted.has('1')).toBe(true); + }); + + it('reads Moodle’s refusal inside a 200 as a failure', async () => { + const server = new MockServer(); + addMoodle(server, 'school.example.edu', courses, { badToken: true }); + const api = moodleApi(await rig('moodle', server)); + const error = await api.courses().catch((one) => one); + expect(error).toBeInstanceOf(ConnectorHttpError); + expect((error as ConnectorHttpError).status).toBe(401); + }); +}); + +describe('Google Classroom', () => { + it('lists courses and assignments across pages with due times in UTC', async () => { + const server = new MockServer(); + addClassroom(server, courses); + const api = classroomApi(await rig('google', server)); + expect((await api.courses()).map((one) => one.name)).toEqual(['Biology 101 (Period 2)', 'Chemistry (Period 2)']); + const list = await api.assignments({ service: 'classroom', id: '11', name: 'Biology 101' }); + expect(list.map((one) => [one.title, one.due, one.submitted])).toEqual([ + ['Lab 3', '2026-10-14T21:00:00.000Z', false], + ['Reading quiz', '2026-10-09T15:30:00.000Z', true], + ['Extra credit', null, false], + ]); + expect(dueOf({ id: '1', dueDate: { year: 2026, month: 1, day: 5 } })).toBe('2026-01-05T00:00:00.000Z'); + }); + + it('saves the PDF in Drive and gives its link, because Classroom takes no file from another app', async () => { + const server = new MockServer(); + const classroom = addClassroom(server, courses); + const api = classroomApi(await rig('google', server)); + const result = await api.handIn( + { + service: 'classroom', + courseId: '11', + id: '1', + title: 'Lab 3', + due: null, + url: null, + text: '', + submitted: false, + }, + file, + ); + expect(result).toEqual({ kind: 'saved', link: 'https://drive.google.com/file/d/drive1/view' }); + expect(classroom.driveFiles).toEqual([{ name: 'Lab 3.pdf', type: expect.stringContaining('multipart/related') }]); + }); +}); + +describe('bringing a course in', () => { + it('makes a section with a page for each assignment, and gives the dated ones to Upcoming', async () => { + const server = new MockServer(); + addCanvas(server, 'school.example.edu', courses); + const deps = await rig('canvas', server); + const api = canvasApi(deps); + const { notes, pages } = storage(); + const notebook = await notes.create({ + kind: 'notebook', + placement: { parentId: null, beforeId: null }, + title: 'School', + }); + const upcoming: { source: string; items: UpcomingItem[] }[] = []; + const bring = { + notes, + pages, + host: deps.host, + zone: 'UTC', + setUpcoming: (source: string, items: UpcomingItem[]) => { + upcoming.push({ source, items }); + return { added: items.length, removed: 0, changed: 0 }; + }, + }; + const course = { service: 'canvas' as const, id: '11', name: 'Biology 101' }; + const result = await bringCourse(bring, notebook.id, course, 'Canvas', await api.assignments(course)); + expect(result).toMatchObject({ pages: 3, dated: 2 }); + const [section] = await notes.listChildren(notebook.id); + expect(section?.title).toBe('Biology 101'); + const made = await notes.listChildren(section?.id ?? ''); + expect(made.map((one) => one.title)).toEqual(['Lab 3', 'Reading quiz', 'Extra credit']); + const lab = pages.held(made[0]?.id ?? '')?.blocks.map((block) => String(block.data.markdown)); + expect(lab?.[0]).toMatch(/^Due Wed, Oct 14/); + expect(lab?.[1]).toBe('Open in Canvas: https://school.example.edu/courses/11/assignments/1'); + expect(lab?.[2]).toBe('Write up the results.'); + const quiz = pages.held(made[1]?.id ?? '')?.blocks.map((block) => String(block.data.markdown)); + expect(quiz?.[1]).toBe('Handed in.'); + expect(upcoming).toHaveLength(1); + expect(upcoming[0]?.source).toBe('canvas:11'); + expect(upcoming[0]?.items.map((one) => [one.title, one.done, one.due?.time])).toEqual([ + ['Lab 3 (Biology 101)', false, { hour: 21, minute: 0 }], + ['Reading quiz (Biology 101)', true, { hour: 15, minute: 30 }], + ]); + }); + + it('updates in place when the school changes things, and leaves the person’s own lines alone', async () => { + const server = new MockServer(); + addCanvas(server, 'school.example.edu', courses); + const deps = await rig('canvas', server); + const api = canvasApi(deps); + const { notes, pages } = storage(); + const notebook = await notes.create({ + kind: 'notebook', + placement: { parentId: null, beforeId: null }, + title: 'School', + }); + const bring = { + notes, + pages, + host: deps.host, + zone: 'UTC', + setUpcoming: () => ({ added: 0, removed: 0, changed: 0 }), + }; + const course = { service: 'canvas' as const, id: '11', name: 'Biology 101' }; + await bringCourse(bring, notebook.id, course, 'Canvas', await api.assignments(course)); + const [section] = await notes.listChildren(notebook.id); + const [lab] = await notes.listChildren(section?.id ?? ''); + const open = await pages.open(lab?.id ?? '', { viewport: null }); + await open.send({ + edits: [{ edit: 'insertBlock', block: { id: 'mine', type: 'text', data: { markdown: 'My plan.' } } }], + }); + await open.close(); + + const changed = (await api.assignments(course)).map((one) => + one.id === '1' ? { ...one, title: 'Lab 3 (revised)', due: '2026-10-16T21:00:00Z' } : one, + ); + await bringCourse(bring, notebook.id, course, 'Canvas', changed); + const children = await notes.listChildren(section?.id ?? ''); + expect(children).toHaveLength(3); + expect(children[0]?.title).toBe('Lab 3 (revised)'); + const blocks = pages.held(lab?.id ?? '')?.blocks.map((block) => String(block.data.markdown)); + expect(blocks?.[0]).toMatch(/^Due Fri, Oct 16/); + expect(blocks).toContain('My plan.'); + expect(blocks?.filter((one) => one.startsWith('Due '))).toHaveLength(1); + }); +}); diff --git a/app/src/features/accounts/lms/moodle.ts b/app/src/features/accounts/lms/moodle.ts new file mode 100644 index 00000000..07e8fad7 --- /dev/null +++ b/app/src/features/accounts/lms/moodle.ts @@ -0,0 +1,134 @@ +// Moodle: the courses the person is in, their assignments, and handing in a file, all through the school's web service. +// Every call is a form posted to one address; Moodle answers 200 even when it refuses, with an exception in the body. +// The token is the host's (Moodle takes it as a form field, which the connector adds). + +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import { base64Of } from '../host'; +import { agendaText } from '../meetings/text'; +import { schoolAddress } from './canvas'; +import type { Assignment, Course, HandIn, HandInResult, LmsApi, LmsDeps } from './types'; + +/** How many assignments get their hand-in status asked for. */ +const STATUS_LIMIT = 60; + +async function moodleCall( + client: ConnectorsClient, + access: string, + base: string, + fn: string, + fields: Record = {}, +): Promise { + const reply = await requestConnector( + 'moodle', + [access], + { url: `${base}/webservice/rest/server.php`, form: { wsfunction: fn, moodlewsrestformat: 'json', ...fields } }, + client, + ); + const data = reply.data; + if (data && typeof data === 'object' && !Array.isArray(data) && 'exception' in data && data.exception) { + throw new ConnectorHttpError( + data.errorcode === 'invalidtoken' ? 401 : 400, + JSON.stringify({ message: data.message }), + ); + } + return data as T; +} + +interface MoodleAssignment { + id: number; + cmid?: number; + name: string; + duedate?: number; + intro?: string; +} + +// checks-disable-next-line modifiability: nested callbacks that walk one structure; flatten them when this changes +export function moodleApi({ client }: LmsDeps): LmsApi { + const userId = async (base: string): Promise => + (await moodleCall<{ userid: number }>(client, 'moodleRead', base, 'core_webservice_get_site_info')).userid; + return { + service: 'moodle', + label: 'Moodle', + async courses() { + const base = await schoolAddress(client, 'moodle'); + const list = await moodleCall<{ id: number; fullname?: string; shortname?: string }[]>( + client, + 'moodleRead', + base, + 'core_enrol_get_users_courses', + { userid: String(await userId(base)) }, + ); + return (Array.isArray(list) ? list : []).map((course) => ({ + service: 'moodle' as const, + id: String(course.id), + name: course.fullname || course.shortname || '', + })); + }, + async assignments(course: Course) { + const base = await schoolAddress(client, 'moodle'); + const answer = await moodleCall<{ courses?: { id: number; assignments?: MoodleAssignment[] }[] }>( + client, + 'moodleRead', + base, + 'mod_assign_get_assignments', + { 'courseids[0]': course.id }, + ); + const found = (answer.courses ?? []).flatMap((one) => one.assignments ?? []); + const result: Assignment[] = []; + for (const [at, one] of found.entries()) { + let submitted = false; + if (at < STATUS_LIMIT) { + try { + const status = await moodleCall<{ lastattempt?: { submission?: { status?: string } } }>( + client, + 'moodleRead', + base, + 'mod_assign_get_submission_status', + { assignid: String(one.id) }, + ); + submitted = status.lastattempt?.submission?.status === 'submitted'; + } catch { + // The status is a convenience; an assignment without it is shown as not handed in. + } + } + result.push({ + service: 'moodle', + courseId: course.id, + id: String(one.id), + title: one.name, + due: one.duedate ? new Date(one.duedate * 1000).toISOString() : null, + url: one.cmid ? `${base}/mod/assign/view.php?id=${one.cmid}` : null, + text: agendaText(one.intro ?? '', true), + submitted, + }); + } + return result; + }, + async handIn(assignment: Assignment, file: HandIn): Promise { + const base = await schoolAddress(client, 'moodle'); + const user = await userId(base); + // The file goes to the person's draft area first (as base64 in the call), then into the submission. + const draft = await moodleCall<{ itemid?: number }>(client, 'moodleSubmit', base, 'core_files_upload', { + component: 'user', + filearea: 'draft', + itemid: '0', + filepath: '/', + filename: file.name, + filecontent: base64Of(file.bytes), + contextlevel: 'user', + instanceid: String(user), + }); + if (draft.itemid === undefined) throw new ConnectorHttpError(502, ''); + await moodleCall(client, 'moodleSubmit', base, 'mod_assign_save_submission', { + assignmentid: assignment.id, + 'plugindata[files_filemanager]': String(draft.itemid), + }); + await moodleCall(client, 'moodleSubmit', base, 'mod_assign_submit_for_grading', { + assignmentid: assignment.id, + acceptsubmissionstatement: '1', + }); + return { kind: 'submitted' }; + }, + }; +} diff --git a/app/src/features/accounts/lms/types.ts b/app/src/features/accounts/lms/types.ts new file mode 100644 index 00000000..4db4b978 --- /dev/null +++ b/app/src/features/accounts/lms/types.ts @@ -0,0 +1,60 @@ +// The courses and assignments of a school's learning system, in one shape for Canvas, Moodle, and Classroom. + +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; + +export type LmsService = 'canvas' | 'moodle' | 'classroom'; + +export interface Course { + service: LmsService; + id: string; + name: string; +} + +export interface Assignment { + service: LmsService; + courseId: string; + id: string; + title: string; + /** When it is due, as an ISO time in UTC, or null. */ + due: string | null; + /** Where the assignment is on the web, or null. */ + url: string | null; + /** The instructions as plain text. */ + text: string; + /** Whether the person has already handed it in. */ + submitted: boolean; +} + +/** A file to hand in. */ +export interface HandIn { + name: string; + mime: string; + bytes: Uint8Array; +} + +/** What handing in did: the work is in, or it is saved where the person finishes it. */ +export type HandInResult = { kind: 'submitted' } | { kind: 'saved'; link: string }; + +export interface LmsDeps { + client: ConnectorsClient; + host: AccountsHost; +} + +export interface LmsApi { + readonly service: LmsService; + /** The name shown in sentences. */ + readonly label: string; + courses(): Promise; + assignments(course: Course): Promise; + handIn(assignment: Assignment, file: HandIn): Promise; +} + +export const SERVICE_LABEL: Record = { + canvas: 'Canvas', + moodle: 'Moodle', + classroom: 'Google Classroom', +}; + +/** The connector that holds the sign-in for each service. */ +export const CONNECTOR_OF: Record = { canvas: 'canvas', moodle: 'moodle', classroom: 'google' }; diff --git a/app/src/features/accounts/meetings/commands.ts b/app/src/features/accounts/meetings/commands.ts new file mode 100644 index 00000000..6b717355 --- /dev/null +++ b/app/src/features/accounts/meetings/commands.ts @@ -0,0 +1,108 @@ +// The New meeting note command: choose where the meeting is (Outlook, Google Calendar, or a calendar file), choose the +// meeting, and open the note that fills in. With no account connected, the file is still there. + +import { getLocation } from '../../../app/location'; +import { commandContext } from '../../../commands/registry'; +import type { NodeId } from '../../../services/notes/types'; +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { isConnectorConnected, refreshConnectors } from '../../connectors'; +import { openPage } from '../../search'; +import { services } from '../notebook'; +import { attempt } from '../run'; +import { pickOne } from '../ui/prompts'; +import type { Choice } from '../ui/prompts'; +import { pickFile } from '../ui/pickFile'; +import { listGoogleEvents } from './google'; +import { eventsFromIcs } from './ics'; +import { makeMeetingNote, whenText } from './note'; +import { listOutlookEvents } from './outlook'; +import { windowAround } from './types'; +import type { MeetingEvent, MeetingSource } from './types'; + +const SERVICE: Record = { outlook: 'Outlook', google: 'Google', ics: '' }; + +async function chooseSource(): Promise { + await refreshConnectors(); + const choices: Choice[] = []; + if (isConnectorConnected('microsoft')) { + choices.push({ id: 'outlook', label: t('accounts.meetings.source.outlook'), value: 'outlook' }); + } + if (isConnectorConnected('google')) { + choices.push({ id: 'google', label: t('accounts.meetings.source.google'), value: 'google' }); + } + choices.push({ id: 'ics', label: t('accounts.meetings.source.ics'), value: 'ics' }); + if (choices.length === 1) { + showToast({ message: t('accounts.meetings.source.connectHint') }); + return 'ics'; + } + return pickOne({ + title: t('accounts.meetings.source.title'), + description: t('accounts.meetings.source.description'), + choices, + confirmLabel: t('accounts.meetings.source.confirm'), + }); +} + +async function readEvents(source: MeetingSource): Promise { + const { from, to } = windowAround(new Date()); + const client = commandContext('palette').platform.connectors; + if (source === 'outlook') return listOutlookEvents(client, from, to); + if (source === 'google') return listGoogleEvents(client, from, to); + const file = await pickFile('.ics,text/calendar'); + if (!file) return null; + const zone = Intl.DateTimeFormat().resolvedOptions().timeZone; + try { + return eventsFromIcs(await file.text(), new Date(), zone); + } catch { + showToast({ message: t('accounts.meetings.icsFailed'), tone: 'danger' }); + return null; + } +} + +function chooseEvent(events: readonly MeetingEvent[]): Promise { + const choices = events.map((event, index) => ({ + id: String(index), + label: event.title || t('accounts.meetings.event.untitled'), + detail: event.location + ? t('accounts.meetings.event.where', { when: whenText(event), place: event.location }) + : whenText(event), + value: event, + })); + return pickOne({ + title: t('accounts.meetings.event.title'), + description: t('accounts.meetings.event.description'), + choices, + confirmLabel: t('accounts.meetings.event.confirm'), + empty: t('accounts.meetings.event.none'), + }); +} + +/** The notebook the note goes in: the open one, else the first. */ +async function targetNotebook(): Promise { + const here = getLocation(); + if (here.view === 'workspace' && here.notebookId) return here.notebookId; + return (await services().notes.listNotebooks())[0]?.id ?? null; +} + +export async function newMeetingNote(): Promise { + const source = await chooseSource(); + if (!source) return; + const service = SERVICE[source] || t('accounts.meetings.source.ics'); + await attempt(service, async () => { + announce(t('accounts.meetings.fetching')); + const events = await readEvents(source); + if (!events) return; + if (source === 'ics' && events.length === 0) return void showToast({ message: t('accounts.meetings.icsEmpty') }); + const event = await chooseEvent(events); + if (!event) return; + const notebook = await targetNotebook(); + if (!notebook) return; + const { notes, pages } = services(); + const made = await makeMeetingNote(notes, pages, notebook, event); + const title = event.title || t('accounts.meetings.event.untitled'); + const message = made.existing ? t('accounts.meetings.opened', { title }) : t('accounts.meetings.done', { title }); + showToast({ message }); + await openPage(notes, made.page.id); + }); +} diff --git a/app/src/features/accounts/meetings/google.ts b/app/src/features/accounts/meetings/google.ts new file mode 100644 index 00000000..7eb78d68 --- /dev/null +++ b/app/src/features/accounts/meetings/google.ts @@ -0,0 +1,92 @@ +// Meetings from Google Calendar (events.list on the primary calendar). Repeating events come out one meeting at a time +// (`singleEvents`), and canceled ones are left out. + +import { requestConnector } from '../../connectors'; +import type { ConnectorReply, ConnectorsClient } from '../../connectors'; +import { agendaText } from './text'; +import type { MeetingEvent } from './types'; + +const BASE = 'https://www.googleapis.com/calendar/v3/calendars/primary/events'; +const MAX_PAGES = 4; + +interface GoogleWhen { + dateTime?: string; + date?: string; +} + +export interface GoogleEvent { + id: string; + status?: string; + summary?: string; + description?: string; + location?: string; + start?: GoogleWhen; + end?: GoogleWhen; + organizer?: { email?: string; displayName?: string }; + attendees?: { email?: string; displayName?: string; resource?: boolean }[]; + hangoutLink?: string; +} + +interface GooglePage { + items?: GoogleEvent[]; + nextPageToken?: string; +} + +function instant(when: GoogleWhen | undefined): string | null { + const text = when?.dateTime ?? (when?.date ? `${when.date}T00:00:00` : undefined); + if (!text) return null; + const date = new Date(text); + return Number.isNaN(date.getTime()) ? null : date.toISOString(); +} + +export function fromGoogle(event: GoogleEvent): MeetingEvent | null { + const start = instant(event.start); + if (!start || event.status === 'cancelled') return null; + const people = [ + event.organizer?.displayName?.trim() || event.organizer?.email, + ...(event.attendees ?? []).filter((one) => !one.resource).map((one) => one.displayName?.trim() || one.email), + ].filter((name): name is string => Boolean(name)); + const html = /<[a-z][\s\S]*>/i.test(event.description ?? ''); + return { + source: 'google', + id: event.id, + title: event.summary?.trim() ?? '', + start, + end: instant(event.end), + allDay: Boolean(event.start?.date), + location: event.location?.trim() ?? '', + attendees: [...new Set(people)], + agenda: agendaText(event.description ?? '', html), + joinLink: event.hangoutLink ?? '', + }; +} + +export async function listGoogleEvents(client: ConnectorsClient, from: Date, to: Date): Promise { + const events: MeetingEvent[] = []; + let token: string | null = null; + for (let page = 0; page < MAX_PAGES; page += 1) { + const reply: ConnectorReply = await requestConnector( + 'google', + ['calendarRead'], + { + url: BASE, + query: { + timeMin: from.toISOString(), + timeMax: to.toISOString(), + singleEvents: 'true', + orderBy: 'startTime', + maxResults: 50, + pageToken: token, + }, + }, + client, + ); + for (const raw of reply.data?.items ?? []) { + const event = fromGoogle(raw); + if (event) events.push(event); + } + token = reply.data?.nextPageToken ?? null; + if (!token) break; + } + return events; +} diff --git a/app/src/features/accounts/meetings/ics.ts b/app/src/features/accounts/meetings/ics.ts new file mode 100644 index 00000000..98d530f1 --- /dev/null +++ b/app/src/features/accounts/meetings/ics.ts @@ -0,0 +1,105 @@ +// Meetings from a calendar file (.ics), the way in when no account is connected. The Upcoming tool's reader finds the +// events and expands the daily and weekly repeats in a window of dates; this adds what a meeting note needs and that +// reader leaves out: the description, who is invited, and the link. + +import { dateIn, icsToItems, parseIcs, toInstant } from '../../tools'; +import type { IcsComponent, IcsTime } from '../../tools'; +import { agendaText } from './text'; +import type { MeetingEvent } from './types'; +import { windowAround } from './types'; + +interface Details { + description: string; + attendees: string[]; + link: string; +} + +const unfold = (text: string): string[] => + text + .replace(/\r\n|\r/g, '\n') + .replace(/\n[ \t]/g, '') + .split('\n') + .filter((line) => line.trim() !== ''); + +const unescapeText = (value: string): string => + value.replace(/\\([nN,;\\])/g, (_, char: string) => (char === 'n' || char === 'N' ? '\n' : char)); + +/** The name of a person line: the CN parameter, else the address after `mailto:`. */ +function personOf(head: string, value: string): string { + const common = /;CN=("([^"]*)"|[^;:]*)/i.exec(head); + const name = common?.[2] ?? common?.[1] ?? ''; + return name.trim() || value.replace(/^mailto:/i, '').trim(); +} + +/** The description, the people, and the link of each event, by UID. */ +function detailsByUid(text: string): Map { + const found = new Map(); + let open: (Details & { uid: string }) | null = null; + for (const line of unfold(text)) { + const at = line.indexOf(':'); + if (at < 0) continue; + const head = line.slice(0, at); + const value = line.slice(at + 1); + const name = head.split(';')[0]?.toUpperCase() ?? ''; + if (name === 'BEGIN' && value.trim().toUpperCase() === 'VEVENT') + open = { uid: '', description: '', attendees: [], link: '' }; + else if (name === 'END' && value.trim().toUpperCase() === 'VEVENT' && open) { + if (open.uid) found.set(open.uid, open); + open = null; + } else if (open && name === 'UID') open.uid = value.trim(); + else if (open && name === 'DESCRIPTION') open.description = unescapeText(value); + else if (open && name === 'URL') open.link = value.trim(); + else if (open && (name === 'ORGANIZER' || name === 'ATTENDEE')) { + const person = personOf(head, value); + if (person) open.attendees[name === 'ORGANIZER' ? 'unshift' : 'push'](person); + } + } + return found; +} + +/** The moment of an iCalendar time in the viewer's zone, as milliseconds. */ +function instantOf(time: IcsTime, viewerZone: string): number { + return toInstant(time.date, time.time ?? { hour: 0, minute: 0 }, time.zone ?? viewerZone); +} + +function idOf(itemId: string): string { + return itemId.split('#')[0] ?? itemId; +} + +function lengthOf(component: IcsComponent, viewerZone: string): number | null { + if (!component.when || !component.end) return null; + const length = instantOf(component.end, viewerZone) - instantOf(component.when, viewerZone); + return length >= 0 ? length : null; +} + +/** The meetings in a calendar file for the last day and the next week. A file with none gives an empty list. */ +export function eventsFromIcs(text: string, now: Date, zone: string): MeetingEvent[] { + const calendar = parseIcs(text); + const events = calendar.components.filter((component) => component.kind === 'event'); + const window = windowAround(now); + const range = { from: dateIn(window.from.getTime(), zone), to: dateIn(window.to.getTime(), zone) }; + const details = detailsByUid(text); + const meetings: MeetingEvent[] = []; + for (const item of icsToItems({ components: events, skipped: 0 }, range, zone)) { + const uid = idOf(item.id); + const component = events.find((one) => one.uid === uid && !one.replaces) ?? events.find((one) => one.uid === uid); + if (!component || !item.due) continue; + const start = toInstant(item.due.date, item.due.time ?? { hour: 0, minute: 0 }, zone); + if (start < window.from.getTime() || start > window.to.getTime()) continue; + const length = lengthOf(component, zone); + const extra = details.get(uid); + meetings.push({ + source: 'ics', + id: item.id, + title: component.title, + start: new Date(start).toISOString(), + end: length === null ? null : new Date(start + length).toISOString(), + allDay: component.when?.time === null, + location: component.location ?? '', + attendees: [...new Set(extra?.attendees ?? [])], + agenda: agendaText(extra?.description ?? '', false), + joinLink: extra?.link ?? '', + }); + } + return meetings.sort((a, b) => a.start.localeCompare(b.start)); +} diff --git a/app/src/features/accounts/meetings/meetings.test.ts b/app/src/features/accounts/meetings/meetings.test.ts new file mode 100644 index 00000000..3707a931 --- /dev/null +++ b/app/src/features/accounts/meetings/meetings.test.ts @@ -0,0 +1,207 @@ +// New meeting note: the invitation text, the three calendars (Outlook and Google through mock services, and an .ics file), +// the note that fills in, and the event ID that the note and its recordings keep. +import { describe, expect, it } from 'vitest'; +import { addGoogleCalendar, addGraphCalendar } from '../../../../../tests/mock-servers/calendars'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { createMemoryNotesService } from '../../../services/notes/memory'; +import { createMemoryPageService } from '../../../services/pages/memory'; +import { createFakeConnectors } from '../../connectors'; +import { listGoogleEvents } from './google'; +import { eventsFromIcs } from './ics'; +import { makeMeetingNote, noteBlocks, VIEW_KEY } from './note'; +import { listOutlookEvents } from './outlook'; +import { agendaText } from './text'; +import { refOf, windowAround } from './types'; +import type { MeetingEvent } from './types'; + +const NOW = new Date('2026-10-07T12:00:00Z'); +const { from, to } = windowAround(NOW); + +describe('the invitation text', () => { + it('turns HTML into lines and cuts the conferencing boilerplate', () => { + const html = + '

    Review the draft & the budget

    • Item one
    • Item two
    ' + + '
    ________________________________________________________________________________
    Microsoft Teams meeting
    '; + expect(agendaText(html, true)).toBe('Review the draft & the budget\n- Item one\n- Item two'); + expect(agendaText('Plan\r\n\r\n\r\n\r\nGo\n---------------\nJoin Zoom Meeting', false)).toBe('Plan\n\nGo'); + expect(agendaText('x'.repeat(30), false, 10)).toBe(`${'x'.repeat(10)}…`); + }); +}); + +describe('Outlook through Graph', () => { + const graph = [ + { id: 'g1', subject: 'Standup', start: '2026-10-07T15:00:00', end: '2026-10-07T15:15:00', location: 'Room 4' }, + { + id: 'g2', + subject: 'Design review', + start: '2026-10-08T17:00:00', + end: '2026-10-08T18:00:00', + organizer: 'Ada Writer', + attendees: [{ name: 'Sam Student', address: 'sam@example.com' }], + html: '

    Agenda: the new cover

    Microsoft Teams meeting
    ', + joinUrl: 'https://teams.example/join/1', + }, + { id: 'g3', subject: 'Called off', start: '2026-10-09T09:00:00', end: '2026-10-09T10:00:00', isCancelled: true }, + { id: 'g4', subject: 'Offsite', start: '2026-10-10T00:00:00', end: '2026-10-11T00:00:00', isAllDay: true }, + { id: 'g5', subject: 'Too far away', start: '2026-12-01T09:00:00', end: '2026-12-01T10:00:00' }, + ]; + + it('reads every page, asks for UTC, and leaves out cancelled and faraway meetings', async () => { + const server = addGraphCalendar(new MockServer(), graph); + const { client } = createFakeConnectors({ connected: { microsoft: 'sam@example.com' }, respond: server.respond }); + const events = await listOutlookEvents(client, from, to); + expect(events.map((event) => event.title)).toEqual(['Standup', 'Design review', 'Offsite']); + expect(server.requests).toHaveLength(2); + expect(server.requests[0]?.query.$orderby).toBe('start/dateTime'); + const review = events[1]; + expect(review).toMatchObject({ + source: 'outlook', + id: 'g2', + start: '2026-10-08T17:00:00.000Z', + attendees: ['Ada Writer', 'Sam Student'], + agenda: 'Agenda: the new cover', + joinLink: 'https://teams.example/join/1', + }); + expect(events[2]?.allDay).toBe(true); + }); +}); + +describe('Google Calendar', () => { + it('reads every page of single events in order and keeps the people and the link', async () => { + const server = addGoogleCalendar(new MockServer(), [ + { + id: 'e1', + summary: 'Lab', + start: '2026-10-07T14:00:00.000Z', + end: '2026-10-07T15:00:00.000Z', + meet: 'https://meet.example/abc', + }, + { + id: 'e2', + summary: 'Study group', + start: '2026-10-08T20:00:00.000Z', + end: '2026-10-08T21:00:00.000Z', + description: 'Chapters 3 and 4', + attendees: [{ name: 'Sam', email: 'sam@example.com' }, { email: 'kim@example.com' }], + }, + { + id: 'e3', + summary: 'Dropped', + start: '2026-10-09T20:00:00.000Z', + end: '2026-10-09T21:00:00.000Z', + status: 'cancelled', + }, + ]); + const { client } = createFakeConnectors({ connected: { google: 'sam@example.com' }, respond: server.respond }); + const events = await listGoogleEvents(client, from, to); + expect(events.map((event) => event.id)).toEqual(['e1', 'e2']); + expect(server.requests).toHaveLength(2); + expect(events[0]?.joinLink).toBe('https://meet.example/abc'); + expect(events[1]).toMatchObject({ agenda: 'Chapters 3 and 4', attendees: ['Organizer', 'Sam', 'kim@example.com'] }); + }); +}); + +const ICS = [ + 'BEGIN:VCALENDAR', + 'BEGIN:VEVENT', + 'UID:one@example.com', + 'SUMMARY:Advising', + 'DTSTART;TZID=America/New_York:20261008T100000', + 'DTEND;TZID=America/New_York:20261008T103000', + 'LOCATION:Room 12', + 'DESCRIPTION:Bring the plan\\nand the transcript', + 'ORGANIZER;CN=Dr. Patel:mailto:patel@example.edu', + 'ATTENDEE;CN="Sam Student":mailto:sam@example.edu', + 'ATTENDEE:mailto:kim@example.edu', + 'URL:https://school.example/advising', + 'END:VEVENT', + 'BEGIN:VEVENT', + 'UID:weekly@example.com', + 'SUMMARY:Seminar', + 'DTSTART:20261006T140000Z', + 'DTEND:20261006T150000Z', + 'RRULE:FREQ=WEEKLY', + 'END:VEVENT', + 'BEGIN:VEVENT', + 'UID:far@example.com', + 'SUMMARY:Next year', + 'DTSTART:20270601T140000Z', + 'END:VEVENT', + 'END:VCALENDAR', +].join('\r\n'); + +describe('a calendar file', () => { + it('reads the meetings in the window, expands repeats, and keeps the description and the people', () => { + const events = eventsFromIcs(ICS, NOW, 'UTC'); + expect(events.map((event) => `${event.title} ${event.start}`)).toEqual([ + 'Seminar 2026-10-06T14:00:00.000Z', + 'Advising 2026-10-08T14:00:00.000Z', + 'Seminar 2026-10-13T14:00:00.000Z', + ]); + const advising = events.find((event) => event.title === 'Advising'); + expect(advising).toMatchObject({ + source: 'ics', + id: 'one@example.com', + end: '2026-10-08T14:30:00.000Z', + location: 'Room 12', + attendees: ['Dr. Patel', 'Sam Student', 'kim@example.edu'], + agenda: 'Bring the plan\nand the transcript', + joinLink: 'https://school.example/advising', + }); + expect(events.some((event) => event.title === 'Next year')).toBe(false); + expect(events.find((event) => event.title === 'Seminar')?.end).toBe('2026-10-06T15:00:00.000Z'); + }); + + it('gives an empty list for a file with no meetings', () => { + expect(eventsFromIcs('BEGIN:VCALENDAR\r\nEND:VCALENDAR', NOW, 'UTC')).toEqual([]); + }); +}); + +describe('the note', () => { + const event: MeetingEvent = { + source: 'outlook', + id: 'g2', + title: 'Design review', + start: '2026-10-08T17:00:00.000Z', + end: '2026-10-08T18:00:00.000Z', + allDay: false, + location: 'Room 4', + attendees: ['Ada Writer', 'Sam Student'], + agenda: 'Agenda: the new cover', + joinLink: '', + }; + + it('fills in the time, place, people, and agenda', () => { + const [details, agenda, notes] = noteBlocks(event); + expect(details).toContain('When: '); + expect(details).toContain('Where: Room 4'); + expect(details).toContain('With: Ada Writer, Sam Student'); + expect(agenda).toBe('## Agenda\n\nAgenda: the new cover'); + expect(notes.startsWith('## Notes')).toBe(true); + expect(noteBlocks({ ...event, agenda: '' })[1]).toContain('The invitation has no agenda.'); + const crowd = noteBlocks({ ...event, attendees: Array.from({ length: 25 }, (_, i) => `Person ${i}`) })[0]; + expect(crowd).toContain('and 5 more'); + }); + + it('is made once, keeps the event ID in the page, and is found again for the same meeting', async () => { + const notes = createMemoryNotesService({ seed: 'empty' }); + const pages = createMemoryPageService([], { + missing: (id) => ({ id, title: '', created: '', modified: '', tags: [], view: {}, blocks: [], assets: {} }), + }); + const notebook = await notes.create({ + kind: 'notebook', + placement: { parentId: null, beforeId: null }, + title: 'School', + }); + const first = await makeMeetingNote(notes, pages, notebook.id, event); + expect(first.existing).toBe(false); + expect(first.page.title).toContain('Design review'); + expect(pages.held(first.page.id)?.view[VIEW_KEY]).toEqual(refOf(event)); + expect(pages.held(first.page.id)?.blocks).toHaveLength(3); + const again = await makeMeetingNote(notes, pages, notebook.id, event); + expect(again).toMatchObject({ existing: true, page: { id: first.page.id } }); + const section = (await notes.listChildren(notebook.id))[0]; + expect(section?.title).toBe('Meetings'); + expect(await notes.listChildren(section?.id ?? '')).toHaveLength(1); + }); +}); diff --git a/app/src/features/accounts/meetings/note.ts b/app/src/features/accounts/meetings/note.ts new file mode 100644 index 00000000..cb9fc725 --- /dev/null +++ b/app/src/features/accounts/meetings/note.ts @@ -0,0 +1,88 @@ +// The meeting note: its title, its blocks, and where it goes. The note keeps the meeting's calendar ID in the page's +// view (`meetingEvent`), so asking for the same meeting again opens the note that is there instead of making a second. +// A recording made on the page is stamped with the same ID (page/audio/controller.ts). + +import type { PagesClient } from '../../../platform/types'; +import type { NodeId, NodeSummary, NotesService } from '../../../services/notes/types'; +import { formatDate, formatTime } from '../../../strings/format'; +import { t } from '../../../strings/t'; +import { cleanTitle, createPage, findOrCreate, readPageKey, writePageKey } from '../notebook'; +import { refOf } from './types'; +import type { MeetingEvent, MeetingRef } from './types'; + +export const VIEW_KEY = 'meetingEvent'; + +/** The people shown on a note before it says how many more were invited. */ +const PEOPLE_SHOWN = 20; + +/** "Oct 7, 2026, 10:00 AM to 10:30 AM", or "Oct 7, 2026, all day". */ +export function whenText(event: Pick): string { + const date = formatDate(event.start); + if (event.allDay) return t('accounts.meetings.event.allDay', { date }); + const start = formatTime(event.start); + return event.end + ? t('accounts.meetings.event.span', { date, start, end: formatTime(event.end) }) + : t('accounts.meetings.event.at', { date, start }); +} + +export function noteTitle(event: MeetingEvent): string { + const title = event.title.trim() || t('accounts.meetings.event.untitled'); + return cleanTitle(t('accounts.meetings.note.title', { title, date: formatDate(event.start) }), title); +} + +function peopleLine(attendees: readonly string[]): string { + const shown = attendees.slice(0, PEOPLE_SHOWN).join(', '); + const more = attendees.length - PEOPLE_SHOWN; + const people = more > 0 ? t('accounts.meetings.note.withMore', { people: shown, count: more }) : shown; + return t('accounts.meetings.note.with', { people }); +} + +/** The blocks of the note: the details, the agenda, and an empty place for notes. */ +export function noteBlocks(event: MeetingEvent): string[] { + const details = [t('accounts.meetings.note.when', { when: whenText(event) })]; + if (event.location) details.push(t('accounts.meetings.note.where', { place: event.location })); + if (event.attendees.length > 0) details.push(peopleLine(event.attendees)); + if (event.joinLink) details.push(t('accounts.meetings.note.join', { link: event.joinLink })); + const agenda = event.agenda || t('accounts.meetings.note.noAgenda'); + return [ + details.join('\n\n'), + `## ${t('accounts.meetings.note.agenda')}\n\n${agenda}`, + `## ${t('accounts.meetings.note.notes')}\n\n`, + ]; +} + +export interface MadeNote { + page: NodeSummary; + /** True when the note was there already and was not written again. */ + existing: boolean; +} + +/** The note for this meeting in the section, if one was made before. */ +async function findExisting( + notes: NotesService, + pages: PagesClient, + section: NodeId, + event: MeetingEvent, +): Promise { + const pagesHere = (await notes.listChildren(section)).filter((node) => node.kind === 'page'); + for (const page of pagesHere) { + const ref = await readPageKey(pages, page.id, VIEW_KEY); + if (ref?.id === event.id && ref.source === event.source && ref.start === event.start) return page; + } + return null; +} + +/** Makes the note for the meeting in the Meetings section of the notebook, or finds the one made before. */ +export async function makeMeetingNote( + notes: NotesService, + pages: PagesClient, + notebook: NodeId, + event: MeetingEvent, +): Promise { + const section = await findOrCreate(notes, notebook, 'section', t('accounts.meetings.note.section')); + const existing = await findExisting(notes, pages, section.id, event); + if (existing) return { page: existing, existing: true }; + const page = await createPage(notes, pages, section.id, noteTitle(event), noteBlocks(event)); + await writePageKey(pages, page.id, VIEW_KEY, refOf(event)); + return { page, existing: false }; +} diff --git a/app/src/features/accounts/meetings/outlook.ts b/app/src/features/accounts/meetings/outlook.ts new file mode 100644 index 00000000..8953ec00 --- /dev/null +++ b/app/src/features/accounts/meetings/outlook.ts @@ -0,0 +1,86 @@ +// Meetings from the Outlook calendar, through Microsoft Graph (calendarView). Times are asked for in UTC, so a +// meeting means the same moment wherever the PC is. Canceled meetings are left out. + +import { requestConnector } from '../../connectors'; +import type { ConnectorReply, ConnectorsClient } from '../../connectors'; +import { agendaText } from './text'; +import type { MeetingEvent } from './types'; + +const BASE = 'https://graph.microsoft.com/v1.0'; +const MAX_PAGES = 4; + +interface Person { + emailAddress?: { name?: string; address?: string }; +} + +export interface GraphEvent { + id: string; + subject?: string; + isCancelled?: boolean; + isAllDay?: boolean; + start?: { dateTime?: string }; + end?: { dateTime?: string }; + location?: { displayName?: string }; + attendees?: Person[]; + organizer?: Person; + body?: { contentType?: string; content?: string }; + bodyPreview?: string; + onlineMeeting?: { joinUrl?: string } | null; +} + +interface GraphPage { + value?: GraphEvent[]; + '@odata.nextLink'?: string; +} + +const named = (person: Person | undefined): string => + person?.emailAddress?.name?.trim() || person?.emailAddress?.address?.trim() || ''; + +/** A Graph time in UTC: it has no zone letter, so one is added. */ +function utc(value: string | undefined): string | null { + if (!value) return null; + const date = new Date(/(Z|[+-]\d\d:?\d\d)$/.test(value) ? value : `${value}Z`); + return Number.isNaN(date.getTime()) ? null : date.toISOString(); +} + +export function fromGraph(event: GraphEvent): MeetingEvent | null { + const start = utc(event.start?.dateTime); + if (!start || event.isCancelled) return null; + const people = [named(event.organizer), ...(event.attendees ?? []).map(named)].filter(Boolean); + const html = event.body?.contentType?.toLowerCase() === 'html'; + return { + source: 'outlook', + id: event.id, + title: event.subject?.trim() ?? '', + start, + end: utc(event.end?.dateTime), + allDay: event.isAllDay === true, + location: event.location?.displayName?.trim() ?? '', + attendees: [...new Set(people)], + agenda: agendaText(event.body?.content ?? event.bodyPreview ?? '', html), + joinLink: event.onlineMeeting?.joinUrl ?? '', + }; +} + +export async function listOutlookEvents(client: ConnectorsClient, from: Date, to: Date): Promise { + const events: MeetingEvent[] = []; + let url: string | null = `${BASE}/me/calendarView`; + for (let page = 0; page < MAX_PAGES && url; page += 1) { + const query = + page === 0 + ? { startDateTime: from.toISOString(), endDateTime: to.toISOString(), $top: 50, $orderby: 'start/dateTime' } + : undefined; + const reply: ConnectorReply = await requestConnector( + 'microsoft', + ['calendarRead'], + { url, query, headers: { Prefer: 'outlook.timezone="UTC", outlook.body-content-type="text"' } }, + client, + ); + for (const raw of reply.data?.value ?? []) { + const event = fromGraph(raw); + if (event) events.push(event); + } + url = reply.data?.['@odata.nextLink'] ?? null; + } + return events; +} diff --git a/app/src/features/accounts/meetings/text.ts b/app/src/features/accounts/meetings/text.ts new file mode 100644 index 00000000..71d725e3 --- /dev/null +++ b/app/src/features/accounts/meetings/text.ts @@ -0,0 +1,33 @@ +// Plain text from the parts of an invitation: HTML turned into lines, and the boilerplate that Teams, Meet, and Zoom +// add after the real agenda taken off. + +const ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' }; + +/** Text from HTML: line breaks and paragraphs become new lines, tags go, and the common entities are decoded. */ +export function htmlToText(html: string): string { + return html + .replace(/<(script|style)[\s\S]*?<\/\1>/gi, '') + .replace(/|<\/(p|div|li|tr|h[1-6])>/gi, '\n') + .replace(/]*>/gi, '- ') + .replace(/<[^>]+>/g, '') + .replace(/&(#\d+|#x[0-9a-f]+|[a-z]+);/gi, (whole, name: string) => { + if (name.startsWith('#x')) return String.fromCodePoint(Number.parseInt(name.slice(2), 16)); + if (name.startsWith('#')) return String.fromCodePoint(Number(name.slice(1))); + return ENTITIES[name.toLowerCase()] ?? whole; + }); +} + +/** The lines where a conferencing service begins its own text. Everything after the first of them is dropped. */ +const BOILERPLATE = [/^_{8,}\s*$/, /^-{8,}\s*$/, /^Microsoft Teams (meeting|Need help)/i, /^Join (Zoom|Google Meet)/i]; + +/** The agenda in an invitation: plain text, trimmed, with the conferencing boilerplate cut and long text shortened. */ +export function agendaText(body: string, isHtml: boolean, limit = 2000): string { + const lines = (isHtml ? htmlToText(body) : body).replace(/\r\n?/g, '\n').split('\n'); + const cut = lines.findIndex((line) => BOILERPLATE.some((pattern) => pattern.test(line.trim()))); + const kept = (cut < 0 ? lines : lines.slice(0, cut)).map((line) => line.replace(/[ \t]+$/g, '')); + const text = kept + .join('\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); + return text.length > limit ? `${text.slice(0, limit).trimEnd()}…` : text; +} diff --git a/app/src/features/accounts/meetings/types.ts b/app/src/features/accounts/meetings/types.ts new file mode 100644 index 00000000..cb5d822a --- /dev/null +++ b/app/src/features/accounts/meetings/types.ts @@ -0,0 +1,42 @@ +// A meeting as the note needs it, whichever calendar it came from. + +export type MeetingSource = 'outlook' | 'google' | 'ics'; + +export interface MeetingEvent { + source: MeetingSource; + /** The calendar's own ID for the event. For a file, the event's UID. */ + id: string; + title: string; + /** ISO 8601, in UTC. */ + start: string; + end: string | null; + allDay: boolean; + location: string; + /** Names, or addresses for people with no name. The organizer comes first. */ + attendees: string[]; + /** The invitation's text, without HTML or the join boilerplate. */ + agenda: string; + joinLink: string; +} + +/** What a meeting note and its recordings keep to name the meeting (page view key `meetingEvent`). */ +export interface MeetingRef { + source: MeetingSource; + id: string; + title: string; + start: string; + end: string | null; +} + +export const refOf = (event: MeetingEvent): MeetingRef => ({ + source: event.source, + id: event.id, + title: event.title, + start: event.start, + end: event.end, +}); + +/** The time window the pickers read: the last day and the next seven. */ +export function windowAround(now: Date): { from: Date; to: Date } { + return { from: new Date(now.getTime() - 86_400_000), to: new Date(now.getTime() + 7 * 86_400_000) }; +} diff --git a/app/src/features/accounts/mockServer.test.ts b/app/src/features/accounts/mockServer.test.ts new file mode 100644 index 00000000..8cd7b635 --- /dev/null +++ b/app/src/features/accounts/mockServer.test.ts @@ -0,0 +1,52 @@ +// The mock service itself: route matching by method, host, and path, parameters, wildcards, recording, and paging. +import { describe, expect, it } from 'vitest'; +import { MockServer, paged } from '../../../../tests/mock-servers/server'; + +describe('MockServer', () => { + const server = new MockServer() + .route('GET api.example/v1/items/:id', (_request, params) => ({ json: { id: params.id } })) + .route('POST api.example/v1/items', (request) => ({ status: 201, json: request.json() })) + .route('GET api.example/files/*', () => ({ text: 'file' })) + .route('GET api.example/v1/none', () => undefined); + + it('matches the method, host, and path and fills the parameters', () => { + expect(server.answer('x', { method: 'GET', url: 'https://api.example/v1/items/a%20b' })).toMatchObject({ + status: 200, + body: '{"id":"a b"}', + }); + const made = server.answer('x', { method: 'post', url: 'https://api.example/v1/items', body: '{"n":1}' }); + expect(made).toMatchObject({ status: 201, body: '{"n":1}' }); + }); + + it('answers 404 for another host, a longer path, or a handler with nothing to say', () => { + for (const [method, url] of [ + ['GET', 'https://other.example/v1/items/1'], + ['GET', 'https://api.example/v1/items/1/more'], + ['GET', 'https://api.example/v1/none'], + ['DELETE', 'https://api.example/v1/items/1'], + ]) { + expect(server.answer('x', { method: method ?? 'GET', url: url ?? '' }).status).toBe(404); + } + expect(server.answer('x', { method: 'GET', url: 'https://api.example/files/a/b/c' }).body).toBe('file'); + }); + + it('records what it was sent, with the form decoded', () => { + const recording = new MockServer().route('POST t.example/token', () => ({ json: {} })); + recording.answer('google', { + method: 'POST', + url: 'https://t.example/token?x=1', + form: { grant_type: 'refresh_token' }, + }); + const seen = recording.requests[0]; + expect(seen?.connector).toBe('google'); + expect(seen?.query).toEqual({ x: '1' }); + expect(seen?.form).toEqual({ grant_type: 'refresh_token' }); + expect(seen?.contentType).toBe('application/x-www-form-urlencoded'); + }); + + it('pages a list with a cursor', () => { + const first = paged([1, 2, 3, 4, 5], { query: {} } as never, 2); + expect(first).toEqual({ items: [1, 2], next: '2' }); + expect(paged([1, 2, 3, 4, 5], { query: { cursor: '4' } } as never, 2)).toEqual({ items: [5], next: null }); + }); +}); diff --git a/app/src/features/accounts/notebook.ts b/app/src/features/accounts/notebook.ts new file mode 100644 index 00000000..84cd0c98 --- /dev/null +++ b/app/src/features/accounts/notebook.ts @@ -0,0 +1,159 @@ +// Writing into the notes for the account features: find or make a notebook, a section, or a page by its title, and +// put Markdown blocks on a page. Everything goes through the notes service and the page service, so the notes +// folder and the page's undo and versions stay the way the rest of the app keeps them. + +import { commandContext } from '../../commands/registry'; +import { newId } from '../../editor/ids'; +import type { PagesClient } from '../../platform/types'; +import type { NodeId, NodeKind, NodeSummary, NotesService } from '../../services/notes/types'; +import { NOTES_LIMITS } from '../../services/notes/types'; +import type { JsonPatch, NewBlock, OpenPage, PageViewJson } from '../../services/pages/types'; + +/** A title the notes service accepts. */ +export function cleanTitle(title: string, fallback: string): string { + const trimmed = title.replace(/\s+/g, ' ').trim().slice(0, NOTES_LIMITS.titleLength).trim(); + return trimmed === '' ? fallback : trimmed; +} + +/** The context the account commands run in. */ +export function services(): { notes: NotesService; pages: PagesClient } { + const context = commandContext('palette'); + return { notes: context.notes, pages: context.platform.pages }; +} + +async function childrenOf(notes: NotesService, parent: NodeId | null): Promise { + return parent === null ? notes.listNotebooks() : notes.listChildren(parent); +} + +/** The child of `parent` of this kind and title, made when it is missing. A null parent means a notebook. */ +export async function findOrCreate( + notes: NotesService, + parent: NodeId | null, + kind: NodeKind, + title: string, +): Promise { + const wanted = cleanTitle(title, title); + const found = (await childrenOf(notes, parent)).find((node) => node.kind === kind && node.title === wanted); + if (found) return found; + return notes.create({ kind, placement: { parentId: parent, beforeId: null }, title: wanted }); +} + +/** Adds a text block for each Markdown string after the block `after`, and answers the IDs it made. */ +async function insertText(open: OpenPage, markdown: readonly string[], after: string | undefined): Promise { + const made: string[] = []; + let previous = after; + for (const text of markdown) { + const id = newId(); + const block: NewBlock = { id, type: 'text', data: { markdown: text } }; + await open.send({ edits: [{ edit: 'insertBlock', block, ...(previous ? { after: previous } : {}) }] }); + made.push(id); + previous = id; + } + return made; +} + +/** Puts a text block for each Markdown string at the end of a page. Answers the IDs of the blocks it made. */ +export async function appendMarkdown( + pages: PagesClient, + pageId: string, + markdown: readonly string[], +): Promise { + const open = await pages.open(pageId, { viewport: null }); + try { + const made = await insertText(open, markdown, open.initial.blocks.at(-1)?.id); + await open.saveNow(); + return made; + } finally { + await open.close(); + } +} + +/** Makes a page in a section with this title and these Markdown blocks. */ +export async function createPage( + notes: NotesService, + pages: PagesClient, + section: NodeId, + title: string, + markdown: readonly string[], +): Promise { + const page = await notes.create({ + kind: 'page', + placement: { parentId: section, beforeId: null }, + title: cleanTitle(title, 'Untitled page'), + }); + if (markdown.length > 0) await appendMarkdown(pages, page.id, markdown); + return page; +} + +/** Replaces blocks that this code wrote earlier with new text blocks; other blocks stay. Answers the new IDs. */ +export async function replaceMarkdown( + pages: PagesClient, + pageId: string, + blockIds: readonly string[], + markdown: readonly string[], +): Promise { + const open = await pages.open(pageId, { viewport: null }); + try { + const present = blockIds.filter((id) => open.initial.blocks.some((block) => block.id === id)); + if (present.length > 0) await open.send({ edits: [{ edit: 'deleteBlocks', blocks: present }] }); + const kept = open.initial.blocks.filter((block) => !present.includes(block.id)); + const made = await insertText(open, markdown, kept.at(-1)?.id); + await open.saveNow(); + return made; + } finally { + await open.close(); + } +} + +/** + * Replaces blocks that this code wrote at the top of a page with new text blocks, which go to the top again. The + * blocks the person wrote below stay where they are. Answers the new IDs. + */ +export async function replaceHeader( + pages: PagesClient, + pageId: string, + blockIds: readonly string[], + markdown: readonly string[], +): Promise { + const open = await pages.open(pageId, { viewport: null }); + try { + const present = blockIds.filter((id) => open.initial.blocks.some((block) => block.id === id)); + if (present.length > 0) await open.send({ edits: [{ edit: 'deleteBlocks', blocks: present }] }); + const first = open.initial.blocks.find((block) => !present.includes(block.id)); + const made: string[] = []; + let previous: string | undefined; + for (const text of markdown) { + const id = newId(); + const block: NewBlock = { id, type: 'text', data: { markdown: text } }; + const place = previous ? { after: previous } : first ? { before: first.id } : {}; + await open.send({ edits: [{ edit: 'insertBlock', block, ...place }] }); + made.push(id); + previous = id; + } + await open.saveNow(); + return made; + } finally { + await open.close(); + } +} + +/** What a page keeps in its view under one key, such as the event a meeting note came from. */ +export async function readPageKey(pages: PagesClient, pageId: string, key: string): Promise { + const open = await pages.open(pageId, { viewport: null }); + try { + return ((open.initial.view as PageViewJson)[key] as T | undefined) ?? null; + } finally { + await open.close(); + } +} + +/** Saves a value in the page's view. The page keeps it with its other settings, and undo and versions cover it. */ +export async function writePageKey(pages: PagesClient, pageId: string, key: string, value: unknown): Promise { + const open = await pages.open(pageId, { viewport: null }); + try { + await open.send({ edits: [{ edit: 'setPage', view: { [key]: value } as JsonPatch }] }); + await open.saveNow(); + } finally { + await open.close(); + } +} diff --git a/app/src/features/accounts/onenote/commands.ts b/app/src/features/accounts/onenote/commands.ts new file mode 100644 index 00000000..814bdf42 --- /dev/null +++ b/app/src/features/accounts/onenote/commands.ts @@ -0,0 +1,67 @@ +// Import OneNote notebooks (Microsoft account): choose one notebook or all of them, bring them in, and say what came over. +// What did not come over is on an Import report page, so nothing is lost without the person being able to see it. +import { commandContext } from '../../../commands/registry'; +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { connectors, connectorsStore } from '../../connectors'; +import { accountsHost } from '../host'; +import { attempt, isConnected, tellNotConnected } from '../run'; +import { pickOne } from '../ui/prompts'; +import { listNotebooks } from './graph'; +import type { GraphNotebook } from './graph'; +import { importNotebooks } from './import'; + +/** The account the Microsoft connector is signed in as. */ +export function microsoftAccount(): string { + const item = connectorsStore.get().items.find((one) => one.id === 'microsoft'); + return item && item.state.kind === 'connected' ? item.state.account : ''; +} + +export async function importOneNote(): Promise { + if (!(await isConnected('microsoft'))) return tellNotConnected('Microsoft'); + await attempt('Microsoft', async () => { + const client = connectors(); + announce(t('accounts.onenote.working')); + const notebooks = await listNotebooks(client); + const choice = await pickOne({ + title: t('accounts.onenote.pick.title'), + description: t('accounts.onenote.pick.description'), + confirmLabel: t('accounts.onenote.pick.confirm'), + empty: t('accounts.onenote.pick.none'), + choices: [ + ...(notebooks.length > 1 ? [{ id: 'all', label: t('accounts.onenote.pick.all'), value: 'all' as const }] : []), + ...notebooks.map((one) => ({ + id: one.id, + label: one.name || t('accounts.onenote.untitledNotebook'), + value: one, + })), + ], + }); + if (!choice) return; + const context = commandContext('palette'); + const reports = await importNotebooks( + { + client, + host: accountsHost(), + notes: context.notes, + pages: context.platform.pages, + extras: context.platform.pageExtras ?? null, + account: microsoftAccount(), + progress: (done, total) => { + if (done === total || done % 10 === 0) announce(t('accounts.onenote.progress', { done, total })); + }, + }, + choice === 'all' ? notebooks : [choice], + ); + const pages = reports.reduce((sum, one) => sum + one.pages, 0); + const left = reports.reduce((sum, one) => sum + one.failedPages.length, 0); + const message = [ + t('accounts.onenote.done', { notebooks: reports.length, pages }), + left > 0 ? t('accounts.onenote.doneLeft', { left }) : '', + ] + .join(' ') + .trim(); + showToast({ message }); + announce(message); + }); +} diff --git a/app/src/features/accounts/onenote/graph.ts b/app/src/features/accounts/onenote/graph.ts new file mode 100644 index 00000000..13a415dd --- /dev/null +++ b/app/src/features/accounts/onenote/graph.ts @@ -0,0 +1,133 @@ +// OneNote through Microsoft Graph: the notebooks, their sections (inside section groups too), the pages of a section, +// and a page's HTML. Lists are followed across pages. Pictures are fetched one at a time into the transfer folder. +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; + +const GRAPH = 'https://graph.microsoft.com/v1.0'; +const ACCESS = ['onenoteRead'] as const; +const MAX_PAGES = 50; + +export interface GraphNotebook { + id: string; + name: string; +} + +export interface GraphSection { + id: string; + /** The section's name, with the names of the section groups it sits in before it. */ + name: string; +} + +export interface GraphPage { + id: string; + title: string; + modified: string; + /** 0 for a page, 1 and 2 for subpages. */ + level: number; +} + +interface List { + value?: T[]; + '@odata.nextLink'?: string; +} + +async function listAll( + client: ConnectorsClient, + first: string, + query?: Record, +): Promise { + const found: T[] = []; + let url: string | null = first; + let params = query; + for (let page = 0; url && page < MAX_PAGES; page += 1) { + const reply: { data?: List } = await requestConnector>( + 'microsoft', + [...ACCESS], + { url, ...(params ? { query: params } : {}) }, + client, + ); + found.push(...(reply.data?.value ?? [])); + url = reply.data?.['@odata.nextLink'] ?? null; + params = undefined; + } + return found; +} + +export async function listNotebooks(client: ConnectorsClient): Promise { + const found = await listAll<{ id: string; displayName?: string }>(client, `${GRAPH}/me/onenote/notebooks`); + return found.map((one) => ({ id: one.id, name: one.displayName ?? '' })).sort((a, b) => a.name.localeCompare(b.name)); +} + +/** The sections of a notebook, and of the section groups inside it, to any depth. */ +export async function listSections(client: ConnectorsClient, notebook: string): Promise { + const sections: GraphSection[] = []; + async function walk(base: string, prefix: string, depth: number): Promise { + const own = await listAll<{ id: string; displayName?: string }>(client, `${base}/sections`); + for (const one of own) sections.push({ id: one.id, name: `${prefix}${one.displayName ?? ''}` }); + if (depth >= 5) return; + const groups = await listAll<{ id: string; displayName?: string }>(client, `${base}/sectionGroups`); + for (const group of groups) { + await walk( + `${GRAPH}/me/onenote/sectionGroups/${encodeURIComponent(group.id)}`, + `${prefix}${group.displayName ?? ''} - `, + depth + 1, + ); + } + } + await walk(`${GRAPH}/me/onenote/notebooks/${encodeURIComponent(notebook)}`, '', 0); + return sections; +} + +export async function listPages(client: ConnectorsClient, section: string): Promise { + const found = await listAll<{ id: string; title?: string; lastModifiedDateTime?: string; level?: number }>( + client, + `${GRAPH}/me/onenote/sections/${encodeURIComponent(section)}/pages`, + { $select: 'id,title,lastModifiedDateTime,level,order', $orderby: 'order', $top: 100 }, + ); + return found.map((one) => ({ + id: one.id, + title: one.title ?? '', + modified: one.lastModifiedDateTime ?? '', + level: Math.min(2, Math.max(0, one.level ?? 0)), + })); +} + +/** The page's HTML, with the full-size address of each picture. */ +export async function pageHtml(client: ConnectorsClient, page: string): Promise { + const reply = await requestConnector( + 'microsoft', + [...ACCESS], + { + url: `${GRAPH}/me/onenote/pages/${encodeURIComponent(page)}/content`, + query: { includeIDs: 'false', includeinkML: 'true' }, + }, + client, + ); + return reply.text; +} + +/** Only a picture on Graph itself is fetched: a page can name any address, and none of them is followed. */ +export function isGraphResource(address: string): boolean { + try { + const url = new URL(address); + return url.protocol === 'https:' && url.hostname === 'graph.microsoft.com' && !url.port && !url.username; + } catch { + return false; + } +} + +/** Fetches a picture or an attached file into the transfer folder. Answers its bytes and type. */ +export async function fetchPicture( + host: AccountsHost, + address: string, + file: string, + fallbackMime = 'image/png', +): Promise<{ bytes: Uint8Array; mime: string }> { + if (!isGraphResource(address)) throw new ConnectorHttpError(400, ''); + const got = await host.download('microsoft', [...ACCESS], { method: 'GET', url: address }, file); + if (got.status < 200 || got.status >= 300) throw new ConnectorHttpError(got.status, got.body); + const bytes = await host.tempRead(file); + await host.tempRemove(file).catch(() => undefined); + return { bytes, mime: (got.contentType ?? fallbackMime).split(';')[0]?.trim() || fallbackMime }; +} diff --git a/app/src/features/accounts/onenote/html.ts b/app/src/features/accounts/onenote/html.ts new file mode 100644 index 00000000..70f13412 --- /dev/null +++ b/app/src/features/accounts/onenote/html.ts @@ -0,0 +1,321 @@ +// OneNote's page HTML into blocks. Microsoft Graph gives a page as HTML with absolute positions, spans with styles, +// to-do tags, images with a full-size address, tables, and attachments as objects. This reads that HTML (without a +// browser, so it runs anywhere) and answers the page's blocks: text blocks as Markdown and pictures to fetch, and a +// count of what could not come over, for the import report. + +import { isInk, parseInk } from './inkml'; +import type { InkSpec } from './inkml'; + +export type PageBlock = + | { kind: 'text'; markdown: string } + | { kind: 'image'; src: string; alt: string; width: number | null } + | { kind: 'attachment'; src: string; name: string; mime: string } + | { kind: 'ink'; strokes: InkSpec[] }; + +/** What was simplified or left out of one page. */ +export interface PageLosses { + tables: number; + /** Attachments with no address to fetch them from. */ + attachments: string[]; + /** Ink the page had, in strokes, which the importer draws. */ + ink: number; + /** Pictures that could not be fetched are counted by the importer. */ + other: string[]; +} + +export interface ConvertedPage { + title: string; + blocks: PageBlock[]; + losses: PageLosses; +} + +interface El { + tag: string; + attrs: Record; + children: (El | string)[]; +} + +const VOID = new Set(['img', 'br', 'hr', 'meta', 'link', 'input', 'col', 'area', 'base', 'source', 'wbr']); +const SKIP = new Set(['script', 'style', 'head', 'title']); +const CONTAINERS = new Set(['#root', 'html', 'body']); +const ENTITIES: Record = { + amp: '&', + lt: '<', + gt: '>', + quot: '"', + apos: "'", + nbsp: ' ', + ndash: '–', + mdash: '—', + hellip: '…', + lsquo: '‘', + rsquo: '’', + ldquo: '“', + rdquo: '”', +}; + +function decode(text: string): string { + return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, name: string) => { + if (name.startsWith('#')) { + const code = name[1] === 'x' || name[1] === 'X' ? parseInt(name.slice(2), 16) : Number(name.slice(1)); + return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : whole; + } + return ENTITIES[name.toLowerCase()] ?? whole; + }); +} + +function attributes(source: string): Record { + const attrs: Record = {}; + const pattern = /([^\s=/>"']+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+)))?/g; + for (let match = pattern.exec(source); match; match = pattern.exec(source)) { + attrs[(match[1] ?? '').toLowerCase()] = decode(match[2] ?? match[3] ?? match[4] ?? ''); + } + return attrs; +} + +/** A small, forgiving HTML reader: enough for what OneNote writes. Unclosed and stray tags are tolerated. */ +function parse(html: string): El { + const root: El = { tag: '#root', attrs: {}, children: [] }; + const stack: El[] = [root]; + const pattern = /|<(\/?)([a-zA-Z][\w:-]*)((?:"[^"]*"|'[^']*'|[^>"'])*)>|([^<]+|<)/g; + for (let match = pattern.exec(html); match; match = pattern.exec(html)) { + const top = stack[stack.length - 1] as El; + if (match[4] !== undefined) { + top.children.push(decode(match[4])); + } else if (match[2] !== undefined) { + const tag = match[2].toLowerCase(); + if (match[1]) { + const at = stack.map((one) => one.tag).lastIndexOf(tag); + if (at > 0) stack.length = at; + } else { + const el: El = { tag, attrs: attributes(match[3] ?? ''), children: [] }; + top.children.push(el); + if (!VOID.has(tag) && !(match[3] ?? '').trimEnd().endsWith('/')) stack.push(el); + } + } + } + return root; +} + +const style = (el: El, name: string): string => { + const found = new RegExp(`(?:^|;)\\s*${name}\\s*:\\s*([^;]+)`, 'i').exec(el.attrs.style ?? ''); + return (found?.[1] ?? '').trim().toLowerCase(); +}; + +/** Wraps text in a Markdown mark, keeping the spaces outside it. */ +function mark(text: string, around: string): string { + const inner = text.trim(); + if (inner === '') return text; + const lead = text.slice(0, text.length - text.trimStart().length); + const trail = text.slice(text.trimEnd().length); + return `${lead}${around}${inner}${around}${trail}`; +} + +const escapeText = (text: string): string => text.replace(/([\\`*_[\]])/g, '\\$1'); + +interface Context { + losses: PageLosses; + blocks: PageBlock[]; + /** Blocks found inside a line of text, which go after the text block that line becomes. */ + pending: PageBlock[]; +} + +/** Adds a block, then the attachments and ink that were inside it. */ +function emit(cx: Context, block: PageBlock): void { + cx.blocks.push(block); + cx.blocks.push(...cx.pending); + cx.pending = []; +} + +function takeInk(node: El, cx: Context): void { + const strokes = parseInk(node); + if (strokes.length === 0) return; + cx.losses.ink += strokes.length; + cx.pending.push({ kind: 'ink', strokes }); +} + +function inline(node: El | string, cx: Context): string { + if (typeof node === 'string') return escapeText(node.replace(/\s+/g, ' ')); + if (SKIP.has(node.tag)) return ''; + if (node.tag === 'br') return '\n'; + if (isInk(node.tag)) { + takeInk(node, cx); + return ''; + } + if (node.tag === 'object' && node.attrs['data-attachment']) { + const name = node.attrs['data-attachment']; + const src = node.attrs.data ?? ''; + if (src) cx.pending.push({ kind: 'attachment', src, name, mime: node.attrs.type ?? '' }); + else cx.losses.attachments.push(name); + return `[${escapeText(name)}] `; + } + let text = node.children.map((child) => inline(child, cx)).join(''); + if (node.tag === 'a' && node.attrs.href) { + text = text.trim() === '' ? '' : `[${text.trim()}](${node.attrs.href.replace(/\)/g, '%29')})`; + } + if (node.tag === 'strong' || node.tag === 'b' || /^(bold|[6-9]00)$/.test(style(node, 'font-weight'))) { + text = mark(text, '**'); + } + if (node.tag === 'em' || node.tag === 'i' || style(node, 'font-style') === 'italic') text = mark(text, '_'); + if ( + node.tag === 'del' || + node.tag === 's' || + node.tag === 'strike' || + style(node, 'text-decoration').includes('line-through') + ) { + text = mark(text, '~~'); + } + if (node.tag === 'code' || /consolas|courier|monospace/.test(style(node, 'font-family'))) text = mark(text, '`'); + return text; +} + +const clean = (text: string): string => + text + .replace(/[ \t]*\n[ \t]*/g, ' \n') + .replace(/ {2,}(?!\n)/g, ' ') + .trim(); + +function todoMark(el: El): string | null { + const tag = el.attrs['data-tag'] ?? ''; + if (tag.includes('to-do:completed')) return '- [x] '; + if (tag.startsWith('to-do')) return '- [ ] '; + return null; +} + +function listLines(list: El, depth: number, cx: Context): string[] { + const lines: string[] = []; + let number = 1; + for (const child of list.children) { + if (typeof child === 'string' || child.tag !== 'li') continue; + const own = child.children.filter((one) => typeof one === 'string' || (one.tag !== 'ul' && one.tag !== 'ol')); + const text = clean(own.map((one) => inline(one, cx)).join('')); + const bullet = todoMark(child) ?? (list.tag === 'ol' ? `${number}. ` : '- '); + lines.push(`${' '.repeat(depth)}${bullet}${text}`); + number += 1; + for (const nested of child.children) { + if (typeof nested !== 'string' && (nested.tag === 'ul' || nested.tag === 'ol')) { + lines.push(...listLines(nested, depth + 1, cx)); + } + } + } + return lines; +} + +function tableMarkdown(table: El, cx: Context): string { + const rows: string[][] = []; + const visit = (el: El) => { + for (const child of el.children) { + if (typeof child === 'string') continue; + if (child.tag === 'tr') { + rows.push( + child.children + .filter((cell): cell is El => typeof cell !== 'string' && (cell.tag === 'td' || cell.tag === 'th')) + .map((cell) => + clean(cell.children.map((one) => inline(one, cx)).join('')) + .replace(/\|/g, '\\|') + .replace(/\n/g, ' '), + ), + ); + } else visit(child); + } + }; + visit(table); + const width = Math.max(1, ...rows.map((row) => row.length)); + const pad = (row: string[]) => `| ${Array.from({ length: width }, (_, at) => row[at] ?? '').join(' | ')} |`; + const [head = [], ...body] = rows; + return [pad(head), `| ${Array.from({ length: width }, () => '---').join(' | ')} |`, ...body.map(pad)].join('\n'); +} + +function pixels(value: string | undefined): number | null { + const found = /^(\d+(?:\.\d+)?)/.exec(value ?? ''); + return found ? Math.round(Number(found[1])) : null; +} + +function flow(node: El | string, cx: Context): void { + if (typeof node === 'string') { + const text = clean(escapeText(node.replace(/\s+/g, ' '))); + if (text !== '') emit(cx, { kind: 'text', markdown: text }); + return; + } + if (SKIP.has(node.tag)) return; + if (isInk(node.tag)) { + takeInk(node, cx); + emitPending(cx); + return; + } + if (node.tag === 'img') { + const src = node.attrs['data-fullres-src'] || node.attrs.src; + if (src) emit(cx, { kind: 'image', src, alt: node.attrs.alt ?? '', width: pixels(node.attrs.width) }); + return; + } + if (node.tag === 'hr') return; + if (/^h[1-6]$/.test(node.tag)) { + const text = clean(node.children.map((child) => inline(child, cx)).join('')).replace(/\n/g, ' '); + if (text !== '') emit(cx, { kind: 'text', markdown: `${'#'.repeat(Number(node.tag[1]))} ${text}` }); + return; + } + if (node.tag === 'ul' || node.tag === 'ol') { + const lines = listLines(node, 0, cx); + if (lines.length > 0) emit(cx, { kind: 'text', markdown: lines.join('\n') }); + return; + } + if (node.tag === 'table') { + cx.losses.tables += 1; + emit(cx, { kind: 'text', markdown: tableMarkdown(node, cx) }); + return; + } + const hasBlockChild = node.children.some( + (child) => + typeof child !== 'string' && + ['div', 'p', 'img', 'ul', 'ol', 'table', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(child.tag), + ); + const todo = todoMark(node); + if ((!hasBlockChild && !CONTAINERS.has(node.tag)) || todo) { + const text = clean(node.children.map((child) => inline(child, cx)).join('')); + if (text !== '' || todo) emit(cx, { kind: 'text', markdown: `${todo ?? ''}${text}`.trimEnd() }); + return; + } + // A container: a line of text between its pictures and lists is a block of its own. + let run: (El | string)[] = []; + const flush = () => { + const text = clean(run.map((child) => inline(child, cx)).join('')); + if (text !== '') emit(cx, { kind: 'text', markdown: text }); + run = []; + }; + for (const child of node.children) { + if (typeof child === 'string' || ['span', 'a', 'b', 'i', 'u', 'em', 'strong', 'cite', 'br'].includes(child.tag)) { + run.push(child); + } else { + flush(); + flow(child, cx); + } + } + flush(); +} + +function emitPending(cx: Context): void { + cx.blocks.push(...cx.pending); + cx.pending = []; +} + +function titleOf(root: El): string { + const find = (el: El): string | null => { + for (const child of el.children) { + if (typeof child === 'string') continue; + if (child.tag === 'title') return child.children.filter((one): one is string => typeof one === 'string').join(''); + const inside = find(child); + if (inside !== null) return inside; + } + return null; + }; + return (find(root) ?? '').replace(/\s+/g, ' ').trim(); +} + +/** The page's blocks. The title is the page's ``, which is what OneNote shows above the page. */ +export function convertPage(html: string): ConvertedPage { + const root = parse(html); + const cx: Context = { losses: { tables: 0, attachments: [], ink: 0, other: [] }, blocks: [], pending: [] }; + flow(root, cx); + emitPending(cx); + return { title: titleOf(root), blocks: cx.blocks, losses: cx.losses }; +} diff --git a/app/src/features/accounts/onenote/import.ts b/app/src/features/accounts/onenote/import.ts new file mode 100644 index 00000000..3e94ffff --- /dev/null +++ b/app/src/features/accounts/onenote/import.ts @@ -0,0 +1,403 @@ +// Bringing OneNote notebooks in through Microsoft Graph. Each notebook becomes a notebook, each section (and each section +// group's sections) a section, and each page a page with its text, headings, lists, tables, and pictures. What can't +// come over is listed in an Import report page: ink, attachments, and anything simplified. Doing it again adds the pages +// that are new and leaves the ones already here alone, because they may hold the person's own writing. State is kept for +// each Microsoft account, so switching accounts never mixes one person's notebooks with another's. + +import { newId } from '../../../editor/ids'; +import { encodeRecords } from '../../../core/ink/codec'; +import { recordFromStroke } from '../../ink'; +import type { InkStroke } from '../../ink'; +import type { PageExtrasClient, PagesClient } from '../../../platform/types'; +import type { Edit } from '../../../services/pages/types'; +import type { NodeId, NotesService } from '../../../services/notes/types'; +import type { PageLevel } from '../../../services/notes/types'; +import { t } from '../../../strings/t'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import { cleanTitle, createPage, replaceMarkdown } from '../notebook'; +import { loadState, saveState } from '../state'; +import { convertPage } from './html'; +import type { PageBlock } from './html'; +import type { InkSpec } from './inkml'; +import { fetchPicture, listPages, listSections, pageHtml } from './graph'; +import type { GraphNotebook } from './graph'; + +interface PageState { + node: string; + modified: string; +} + +interface NotebookState { + node: string; + sections: Record<string, { node: string; pages: Record<string, PageState> }>; + report?: { page: string; blocks: string[] }; +} + +interface AccountState { + notebooks: Record<string, NotebookState>; +} + +export interface ImportDeps { + client: ConnectorsClient; + host: AccountsHost; + notes: NotesService; + pages: PagesClient; + /** Attaches files to a page. Without it, attachments are listed in the report instead. */ + extras?: Pick<PageExtrasClient, 'attachBytes'> | null; + /** The account the notebooks are read from, which keeps the state apart. */ + account: string; + progress?(done: number, total: number): void; +} + +export interface NotebookReport { + notebook: string; + node: NodeId; + sections: number; + pages: number; + /** Pages that were already here and were left as they are. */ + kept: number; + /** Pages that changed in OneNote after they were brought in. */ + changedInOneNote: string[]; + tables: string[]; + attachments: { page: string; name: string }[]; + /** Attached files that came over as files on their pages. */ + attached: number; + /** Strokes of ink that came over. */ + ink: number; + pictures: { fetched: number; failed: { page: string; why: string }[] }; + failedPages: { page: string; why: string }[]; +} + +/** A state file name for an account: lowercase letters, digits, and dashes. */ +export function stateName(account: string): string { + const slug = account + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 48); + return `onenote.${slug || 'account'}`; +} + +const WIDEST = 640; + +interface InkLayer { + id: string | null; +} +type OpenPage = Awaited<ReturnType<PagesClient['open']>>; + +const STROKES_PER_BATCH = 400; +const MAX_ATTACHMENT = 200 * 1024 * 1024; + +/** Draws strokes on the page's ink layer, made with the first. Answers the block to put the next one after. */ +async function writeInk( + open: OpenPage, + layer: InkLayer, + specs: readonly InkSpec[], + after: string | undefined, + report: NotebookReport, +): Promise<string | undefined> { + const now = Date.now(); + for (let from = 0; from < specs.length; from += STROKES_PER_BATCH) { + const edits: Edit[] = []; + if (!layer.id) { + layer.id = newId(); + edits.push({ + edit: 'insertBlock', + block: { id: layer.id, type: 'ink', frame: { x: 0, y: 0 }, data: { role: 'layer' } }, + }); + } + const block = layer.id; + const strokes: InkStroke[] = specs.slice(from, from + STROKES_PER_BATCH).map((spec) => ({ + id: newId(), + tool: spec.highlighter ? 'highlighter' : 'pen', + width: spec.width, + startTime: now, + startUnknown: true, + points: spec.points, + block, + slot: 0, + color: spec.color, + })); + await open.send({ + edits, + strokes: encodeRecords(strokes.map((stroke) => ({ kind: 'stroke' as const, stroke: recordFromStroke(stroke) }))), + }); + report.ink += strokes.length; + } + return after; +} + +/** Fetches an attached file and puts it on the page as a file block. A file that can't come over is listed. */ +async function writeAttachment( + deps: ImportDeps, + open: OpenPage, + pageId: string, + title: string, + block: Extract<PageBlock, { kind: 'attachment' }>, + after: string | undefined, + report: NotebookReport, +): Promise<string | undefined> { + const lost = () => { + report.attachments.push({ page: title, name: block.name }); + return after; + }; + if (!deps.extras) return lost(); + try { + const file = await fetchPicture( + deps.host, + block.src, + `onenote-${newId()}.bin`, + block.mime || 'application/octet-stream', + ); + if (file.bytes.length > MAX_ATTACHMENT) return lost(); + const asset = await deps.extras.attachBytes( + pageId, + file.bytes.slice().buffer, + block.name || t('accounts.onenote.untitledFile'), + block.mime || file.mime, + ); + const id = newId(); + await open.send({ + edits: [ + { edit: 'addAsset', asset: asset.id }, + { + edit: 'insertBlock', + block: { id, type: 'file', data: { asset: asset.id } }, + ...(after ? { after } : {}), + }, + ], + }); + report.attached += 1; + return id; + } catch { + return lost(); + } +} + +/** Writes a converted page onto a page: text blocks, then each picture fetched, in order. */ +async function writeBlocks( + deps: ImportDeps, + pageId: string, + title: string, + blocks: readonly PageBlock[], + report: NotebookReport, + // checks-disable-next-line modifiability: nested callbacks that walk one structure; flatten them when this changes +): Promise<void> { + const open = await deps.pages.open(pageId, { viewport: null }); + const layer: InkLayer = { id: null }; + try { + let after: string | undefined; + for (const block of blocks) { + const id = newId(); + if (block.kind === 'text') { + await open.send({ + edits: [ + { + edit: 'insertBlock', + block: { id, type: 'text', data: { markdown: block.markdown } }, + ...(after ? { after } : {}), + }, + ], + }); + after = id; + continue; + } + if (block.kind === 'ink') { + after = await writeInk(open, layer, block.strokes, after, report); + continue; + } + if (block.kind === 'attachment') { + after = await writeAttachment(deps, open, pageId, title, block, after, report); + continue; + } + try { + const picture = await fetchPicture(deps.host, block.src, `onenote-${id}.img`); + const asset = await open.importImage({ + kind: 'bytes', + bytes: picture.bytes.slice().buffer, + name: `${title || 'picture'}.${picture.mime.split('/')[1] ?? 'png'}`, + mime: picture.mime, + }); + await open.send({ + edits: [ + { edit: 'addAsset', asset: asset.id }, + { + edit: 'insertBlock', + block: { + id, + type: 'image', + frame: { w: Math.min(block.width ?? WIDEST, WIDEST) }, + data: { asset: asset.id, ...(block.alt ? { alt: block.alt } : {}) }, + }, + ...(after ? { after } : {}), + }, + ], + }); + after = id; + report.pictures.fetched += 1; + } catch { + report.pictures.failed.push({ page: title, why: t('accounts.onenote.report.pictureWhy') }); + } + } + await open.saveNow(); + } finally { + await open.close(); + } +} + +/** The text of the Import report page. */ +export function reportMarkdown(reports: readonly NotebookReport[]): string[] { + const out: string[] = [t('accounts.onenote.report.intro', { date: new Date().toLocaleDateString() })]; + for (const one of reports) { + out.push(`## ${one.notebook}`); + out.push(t('accounts.onenote.report.counts', { sections: one.sections, pages: one.pages, kept: one.kept })); + if (one.failedPages.length > 0) { + out.push( + t('accounts.onenote.report.failedPages'), + one.failedPages.map((page) => `- ${page.page}: ${page.why}`).join('\n'), + ); + } + if (one.changedInOneNote.length > 0) { + out.push(t('accounts.onenote.report.changed'), one.changedInOneNote.map((page) => `- ${page}`).join('\n')); + } + if (one.tables.length > 0) { + out.push(t('accounts.onenote.report.tables'), one.tables.map((page) => `- ${page}`).join('\n')); + } + if (one.attached > 0) out.push(t('accounts.onenote.report.attached', { count: one.attached })); + if (one.ink > 0) out.push(t('accounts.onenote.report.inked', { count: one.ink })); + if (one.attachments.length > 0) { + out.push( + t('accounts.onenote.report.attachments'), + one.attachments.map((file) => `- ${file.name} (${file.page})`).join('\n'), + ); + } + if (one.pictures.failed.length > 0) { + out.push( + t('accounts.onenote.report.pictures', { count: one.pictures.failed.length }), + one.pictures.failed.map((file) => `- ${file.page}`).join('\n'), + ); + } + } + return out; +} + +/** Brings the notebooks in. Answers one report for each. */ +export async function importNotebooks(deps: ImportDeps, chosen: readonly GraphNotebook[]): Promise<NotebookReport[]> { + const name = stateName(deps.account); + const state = await loadState<AccountState>(name, { notebooks: {} }, deps.host); + const reports: NotebookReport[] = []; + let done = 0; + let total = 0; + for (const notebook of chosen) { + const sections = await listSections(deps.client, notebook.id); + const listed = await Promise.all( + sections.map(async (section) => ({ section, pages: await listPages(deps.client, section.id) })), + ); + total += listed.reduce((sum, one) => sum + one.pages.length, 0); + let known: NotebookState | undefined = state.notebooks[notebook.id]; + if (known && !(await deps.notes.get(known.node as NodeId))) known = undefined; + if (!known) { + const taken = (await deps.notes.listNotebooks()).some((one) => one.title === cleanTitle(notebook.name, '')); + const title = taken ? `${notebook.name} (${deps.account})` : notebook.name; + const made = await deps.notes.create({ + kind: 'notebook', + placement: { parentId: null, beforeId: null }, + title: cleanTitle(title, t('accounts.onenote.untitledNotebook')), + }); + known = { node: made.id, sections: {} }; + } + state.notebooks[notebook.id] = known; + const report: NotebookReport = { + notebook: notebook.name, + node: known.node as NodeId, + sections: 0, + pages: 0, + kept: 0, + changedInOneNote: [], + tables: [], + attachments: [], + attached: 0, + ink: 0, + pictures: { fetched: 0, failed: [] }, + failedPages: [], + }; + for (const { section, pages } of listed) { + let sectionState: NotebookState['sections'][string] | undefined = known.sections[section.id]; + if (sectionState && !(await deps.notes.get(sectionState.node as NodeId))) sectionState = undefined; + if (!sectionState) { + const made = await deps.notes.create({ + kind: 'section', + placement: { parentId: known.node as NodeId, beforeId: null }, + title: cleanTitle(section.name, t('accounts.onenote.untitledSection')), + }); + sectionState = { node: made.id, pages: {} }; + known.sections[section.id] = sectionState; + } + report.sections += 1; + for (const page of pages) { + done += 1; + deps.progress?.(done, total); + const earlier = sectionState.pages[page.id]; + if (earlier && (await deps.notes.get(earlier.node as NodeId))) { + report.kept += 1; + if (page.modified && earlier.modified && page.modified > earlier.modified) { + report.changedInOneNote.push(page.title || t('accounts.onenote.untitledPage')); + } + continue; + } + try { + const converted = convertPage(await pageHtml(deps.client, page.id)); + const title = page.title || converted.title || t('accounts.onenote.untitledPage'); + const node = await createPage(deps.notes, deps.pages, sectionState.node as NodeId, title, []); + if (page.level > 0) await deps.notes.setPageLevel([node.id], page.level as PageLevel); + await writeBlocks(deps, node.id, title, converted.blocks, report); + sectionState.pages[page.id] = { node: node.id, modified: page.modified }; + report.pages += 1; + if (converted.losses.tables > 0) report.tables.push(title); + for (const file of converted.losses.attachments) report.attachments.push({ page: title, name: file }); + } catch (error) { + report.failedPages.push({ + page: page.title || t('accounts.onenote.untitledPage'), + why: error instanceof Error && error.message ? error.message : t('accounts.onenote.report.pageWhy'), + }); + } + await saveState(name, state, deps.host); + } + } + reports.push(report); + } + await writeReport(deps, state, chosen, reports); + await saveState(name, state, deps.host); + return reports; +} + +/** Puts the Import report on a page of the first notebook, replacing the report of the last time. */ +async function writeReport( + deps: ImportDeps, + state: AccountState, + chosen: readonly GraphNotebook[], + reports: readonly NotebookReport[], +): Promise<void> { + const first = chosen[0] ? state.notebooks[chosen[0].id] : undefined; + if (!first) return; + const text = reportMarkdown(reports); + const earlier = first.report ? await deps.notes.get(first.report.page as NodeId) : null; + if (earlier && first.report) { + first.report.blocks = await replaceMarkdown(deps.pages, first.report.page, first.report.blocks, text); + return; + } + const sectionTitle = t('accounts.onenote.report.section'); + const existing = (await deps.notes.listChildren(first.node as NodeId)).find( + (node) => node.kind === 'section' && node.title === sectionTitle, + ); + const section = + existing ?? + (await deps.notes.create({ + kind: 'section', + placement: { parentId: first.node as NodeId, beforeId: null }, + title: sectionTitle, + })); + const page = await createPage(deps.notes, deps.pages, section.id, t('accounts.onenote.report.title'), []); + first.report = { page: page.id, blocks: await replaceMarkdown(deps.pages, page.id, [], text) }; +} diff --git a/app/src/features/accounts/onenote/inkml.ts b/app/src/features/accounts/onenote/inkml.ts new file mode 100644 index 00000000..01f982ac --- /dev/null +++ b/app/src/features/accounts/onenote/inkml.ts @@ -0,0 +1,198 @@ +// OneNote's ink, as Microsoft Graph gives it when a page is asked for with InkML: traces of points with a brush for +// each. This reads the traces into strokes in page pixels. InkML writes numbers three ways (whole values, and the +// first or second difference from the point before), and all three are read. Units are HIMETRIC (0.01 mm) unless a +// brush or channel says otherwise. + +/** The part of the HTML reader's element that ink needs. */ +export interface InkEl { + tag: string; + attrs: Record<string, string>; + children: (InkEl | string)[]; +} + +export interface InkSpec { + points: { x: number; y: number; pressure?: number }[]; + /** Red, green, blue, and alpha bytes. */ + color: [number, number, number, number]; + /** The nominal width in page pixels. */ + width: number; + highlighter: boolean; +} + +const PX_PER_HIMETRIC = 96 / 2540; +const UNIT_TO_PX: Record<string, number> = { + himetric: PX_PER_HIMETRIC, + mm: 96 / 25.4, + cm: 96 / 2.54, + in: 96, + pt: 96 / 72, + px: 1, +}; +const MAX_POINTS = 20_000; +const MAX_STROKES = 5_000; + +const local = (tag: string): string => tag.slice(tag.indexOf(':') + 1); + +function walk(el: InkEl, visit: (one: InkEl) => void): void { + visit(el); + for (const child of el.children) if (typeof child !== 'string') walk(child, visit); +} + +function textOf(el: InkEl): string { + return el.children.map((child) => (typeof child === 'string' ? child : textOf(child))).join(''); +} + +interface Brush { + color: [number, number, number, number]; + width: number; + highlighter: boolean; +} + +const DEFAULT_BRUSH: Brush = { color: [0, 0, 0, 255], width: 1.5, highlighter: false }; + +function parseColor(value: string): [number, number, number] | null { + const found = /^#?([0-9a-f]{6})(?:[0-9a-f]{2})?$/i.exec(value.trim()); + if (!found) return null; + const n = parseInt(found[1] ?? '0', 16); + return [(n >> 16) & 255, (n >> 8) & 255, n & 255]; +} + +function readBrush(el: InkEl): Brush { + const brush: Brush = { ...DEFAULT_BRUSH, color: [...DEFAULT_BRUSH.color] }; + let transparency = 0; + walk(el, (one) => { + if (local(one.tag) !== 'brushproperty') return; + const name = (one.attrs.name ?? '').toLowerCase(); + const value = one.attrs.value ?? ''; + if (name === 'color') { + const rgb = parseColor(value); + if (rgb) [brush.color[0], brush.color[1], brush.color[2]] = rgb; + } else if (name === 'width' || name === 'height') { + const size = Number(value); + const scale = UNIT_TO_PX[(one.attrs.units ?? 'himetric').toLowerCase()] ?? PX_PER_HIMETRIC; + if (Number.isFinite(size) && size > 0 && name === 'width') brush.width = Math.max(0.5, size * scale); + } else if (name === 'transparency') { + const amount = Number(value); + if (Number.isFinite(amount)) transparency = Math.min(255, Math.max(0, amount)); + } else if (name === 'rasterop') { + brush.highlighter = value.toLowerCase() === 'maskpen'; + } else if (name === 'tip' && value.toLowerCase() === 'rectangle') { + brush.highlighter = true; + } + }); + brush.color[3] = Math.round(255 - transparency); + if (brush.highlighter && brush.color[3] === 255) brush.color[3] = 128; + return brush; +} + +interface Layout { + x: number; + y: number; + f: number; + count: number; +} + +function layoutOf(el: InkEl | undefined): Layout { + const layout: Layout = { x: 0, y: 1, f: -1, count: 3 }; + if (!el) return layout; + const names: string[] = []; + walk(el, (one) => { + if (local(one.tag) === 'channel') names.push((one.attrs.name ?? '').toUpperCase()); + }); + if (names.length === 0) return layout; + const x = names.indexOf('X'); + const y = names.indexOf('Y'); + return { x: x < 0 ? 0 : x, y: y < 0 ? 1 : y, f: names.indexOf('F'), count: names.length }; +} + +/** The points of one trace's text: points are separated by commas, a point's values by spaces. */ +export function parseTrace(text: string, layout: Layout, scale: number): InkSpec['points'] { + const value: number[] = []; + const speed: number[] = []; + const mode: ('!' | "'" | '"')[] = []; + const points: InkSpec['points'] = []; + for (const raw of text.split(',')) { + const tokens = raw.trim().split(/\s+/).filter(Boolean); + if (tokens.length === 0) continue; + for (let at = 0; at < layout.count && at < tokens.length; at += 1) { + let token = tokens[at] ?? ''; + const first = token[0]; + if (first === '!' || first === "'" || first === '"') { + mode[at] = first; + token = token.slice(1); + } else if (first === '*' || first === '?') { + token = ''; + } + if (token === '') { + if (mode[at] === '"') { + speed[at] = speed[at] ?? 0; + value[at] = (value[at] ?? 0) + (speed[at] ?? 0); + } + continue; + } + const number = Number(token); + if (!Number.isFinite(number)) continue; + const current = mode[at] ?? '!'; + if (current === '!') { + value[at] = number; + speed[at] = 0; + } else if (current === "'") { + speed[at] = number; + value[at] = (value[at] ?? 0) + number; + } else { + speed[at] = (speed[at] ?? 0) + number; + value[at] = (value[at] ?? 0) + (speed[at] ?? 0); + } + } + const x = value[layout.x]; + const y = value[layout.y]; + if (x === undefined || y === undefined) continue; + const force = layout.f >= 0 ? value[layout.f] : undefined; + points.push({ + x: Math.round(x * scale * 100) / 100, + y: Math.round(y * scale * 100) / 100, + ...(force !== undefined && force > 0 ? { pressure: Math.min(1, force / 1023) } : {}), + }); + if (points.length >= MAX_POINTS) break; + } + return points; +} + +/** The strokes in one InkML element. */ +export function parseInk(root: InkEl): InkSpec[] { + const brushes = new Map<string, Brush>(); + let layoutEl: InkEl | undefined; + let lastBrush: Brush = DEFAULT_BRUSH; + let scale = PX_PER_HIMETRIC; + walk(root, (one) => { + const name = local(one.tag); + if (name === 'brush') { + const brush = readBrush(one); + brushes.set(one.attrs['xml:id'] ?? one.attrs.id ?? '', brush); + lastBrush = brush; + } else if (name === 'traceformat' && !layoutEl) { + layoutEl = one; + } else if (name === 'channel' && (one.attrs.name ?? '').toUpperCase() === 'X' && one.attrs.units) { + scale = UNIT_TO_PX[one.attrs.units.toLowerCase()] ?? scale; + } + }); + const layout = layoutOf(layoutEl); + const strokes: InkSpec[] = []; + walk(root, (one) => { + if (local(one.tag) !== 'trace' || strokes.length >= MAX_STROKES) return; + const points = parseTrace(textOf(one), layout, scale); + if (points.length === 0) return; + const ref = (one.attrs.brushref ?? '').replace(/^#/, ''); + const brush = brushes.get(ref) ?? lastBrush; + strokes.push({ + points, + color: [...brush.color], + width: brush.width, + highlighter: brush.highlighter, + }); + }); + return strokes; +} + +/** True for an element that holds InkML. */ +export const isInk = (tag: string): boolean => local(tag) === 'ink' && (tag === 'ink' || tag.startsWith('inkml:')); diff --git a/app/src/features/accounts/onenote/onenote.test.ts b/app/src/features/accounts/onenote/onenote.test.ts new file mode 100644 index 00000000..6c723710 --- /dev/null +++ b/app/src/features/accounts/onenote/onenote.test.ts @@ -0,0 +1,284 @@ +// checks-disable-file brand-consistency: the test colors are inputs to the code under test, not design colors + +// OneNote through a mock Microsoft Graph: OneNote's HTML as blocks, notebooks with section groups, and pages across result +// pages. It also covers pictures fetched onto pages, the Import report, a second import that adds only what is new, and switching from +// one Microsoft account to another without mixing their notebooks. +import { describe, expect, it } from 'vitest'; +import { addOneNote } from '../../../../../tests/mock-servers/onenote'; +import type { MockOneNotebook } from '../../../../../tests/mock-servers/onenote'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { createMemoryNotesService } from '../../../services/notes/memory'; +import { createMemoryPageService } from '../../../services/pages/memory'; +import { createFakeConnectors } from '../../connectors'; +import { createFakeAccountsHost } from '../fakeHost'; +import { isGraphResource, listNotebooks, listSections } from './graph'; +import { convertPage } from './html'; +import { importNotebooks, stateName } from './import'; + +const PAGE = `<html><head><title>Cell notes + +
    +

    Mitosis

    +

    The nucleus divides & the cell splits. See the lecture.

    +

    Read chapter 4

    +

    Draw the phases

    +
    • Prophase
      • Chromosomes condense
    • Metaphase
    +
    1. First
    2. Second
    +
    PhaseTime
    Prophase30 | 60 min
    +A cell +

    +1000 2000 500, '100 '50 '10, 100 -50 00 0 0, 2646 0 0 +

    Old idea x = 1

    +
    `; + +describe('OneNote HTML', () => { + it('becomes headings, marks, links, to-dos, lists, a table, a picture, and an attachment note', () => { + const page = convertPage(PAGE); + expect(page.title).toBe('Cell notes'); + const texts = page.blocks + .filter((one) => one.kind === 'text') + .map((one) => (one.kind === 'text' ? one.markdown : '')); + expect(texts[0]).toBe('# Mitosis'); + expect(texts[1]).toBe( + 'The **nucleus** divides & _the cell_ splits. See [the lecture](https://example.com/a(b%29).', + ); + expect(texts[2]).toBe('- [ ] Read chapter 4'); + expect(texts[3]).toBe('- [x] Draw the phases'); + expect(texts[4]).toBe('- Prophase\n - Chromosomes condense\n- Metaphase'); + expect(texts[5]).toBe('1. First\n2. Second'); + expect(texts[6]).toBe('| Phase | Time |\n| --- | --- |\n| Prophase | 30 \\| 60 min |'); + expect(texts.at(-1)).toBe('~~Old idea~~ `x = 1`'); + const image = page.blocks.find((one) => one.kind === 'image'); + expect(image).toEqual({ + kind: 'image', + src: 'https://graph.microsoft.com/v1.0/me/onenote/resources/big/$value', + alt: 'A cell', + width: 300, + }); + expect(page.losses.tables).toBe(1); + expect(page.losses.attachments).toEqual([]); + expect(page.blocks.find((one) => one.kind === 'attachment')).toEqual({ + kind: 'attachment', + src: 'https://graph.microsoft.com/v1.0/me/onenote/resources/pdf/$value', + name: 'slides.pdf', + mime: 'application/pdf', + }); + expect(page.losses.ink).toBe(2); + }); + + it('reads InkML traces in whole values and differences, in pixels, with the brush', () => { + const ink = convertPage(PAGE).blocks.find((one) => one.kind === 'ink'); + if (ink?.kind !== 'ink') throw new Error('no ink'); + const [first, second] = ink.strokes; + expect(first?.color).toEqual([255, 0, 0, 255]); + expect(first?.width).toBeCloseTo(5.67, 1); + expect(first?.points.map((p) => [Math.round(p.x), Math.round(p.y)])).toEqual([ + [38, 76], + [42, 77], + [45, 76], + ]); + expect(second?.points[1]?.x).toBeCloseTo(100, 0); + }); + + it('survives stray and unclosed tags, and takes only addresses on Graph for pictures', () => { + const page = convertPage('

    One

    Two

    '); + expect(page.blocks).toEqual([ + { kind: 'text', markdown: 'One' }, + { kind: 'text', markdown: 'Two' }, + ]); + expect(isGraphResource('https://graph.microsoft.com/v1.0/me/onenote/resources/a/$value')).toBe(true); + expect(isGraphResource('https://evil.example/x.png')).toBe(false); + expect(isGraphResource('http://graph.microsoft.com/x')).toBe(false); + expect(isGraphResource('https://graph.microsoft.com.evil.example/x')).toBe(false); + }); +}); + +const page = (id: string, title: string, modified = '2026-10-01T10:00:00Z', level = 0, html = PAGE) => ({ + id, + title, + modified, + level, + html, +}); + +function notebooks(): MockOneNotebook[] { + return [ + { + id: 'nb1', + name: 'Biology', + sections: [ + { + id: 's1', + name: 'Cells', + pages: [page('p1', 'Mitosis'), page('p2', 'Meiosis', undefined, 1), page('p3', 'Osmosis')], + }, + ], + groups: [ + { + id: 'g1', + name: 'Labs', + sections: [ + { + id: 's2', + name: 'Week 1', + pages: [page('p4', 'Microscope', undefined, 0, '

    Focus first.

    ')], + }, + ], + }, + ], + }, + ]; +} + +function storage() { + const notes = createMemoryNotesService({ seed: 'empty' }); + const pages = createMemoryPageService([], { + missing: (id) => ({ id, title: '', created: '', modified: '', tags: [], view: {}, blocks: [], assets: {} }), + }); + return { notes, pages }; +} + +function rig(account: string, server: MockServer) { + const connectors = createFakeConnectors({ connected: { microsoft: account }, respond: server.respond }); + return { client: connectors.client, host: createFakeAccountsHost(() => connectors.client) }; +} + +const pictures = { + big: { type: 'image/png', body: 'PICTURE-BYTES' }, + pdf: { type: 'application/pdf', body: 'PDF-BYTES' }, +}; + +describe('Graph lists', () => { + it('follows result pages and finds the sections inside section groups', async () => { + const server = new MockServer(); + addOneNote(server, notebooks(), pictures, { pageSize: 1 }); + const { client } = rig('alice@example.com', server); + expect((await listNotebooks(client)).map((one) => one.name)).toEqual(['Biology']); + expect((await listSections(client, 'nb1')).map((one) => one.name)).toEqual(['Cells', 'Labs - Week 1']); + }); +}); + +// checks-disable-next-line modifiability: one test group that shares its fixtures; split it when it grows again +describe('importing notebooks', () => { + it('brings in sections, pages, levels, pictures, and an Import report', async () => { + const server = new MockServer(); + addOneNote(server, notebooks(), pictures); + const { client, host } = rig('alice@example.com', server); + const { notes, pages } = storage(); + const [report] = await importNotebooks({ client, host, notes, pages, account: 'alice@example.com' }, [ + { id: 'nb1', name: 'Biology' }, + ]); + expect(report).toMatchObject({ sections: 2, pages: 4, kept: 0, tables: ['Mitosis', 'Meiosis', 'Osmosis'] }); + expect(report?.attachments).toHaveLength(3); + expect(report?.attached).toBe(0); + expect(report?.pictures.fetched).toBe(3); + + const books = await notes.listNotebooks(); + expect(books.map((one) => one.title)).toEqual(['Biology']); + const sections = await notes.listChildren(books[0]?.id as never); + expect(sections.map((one) => one.title).sort()).toEqual(['Cells', 'Import report', 'Labs - Week 1']); + const cells = sections.find((one) => one.title === 'Cells'); + const children = await notes.listChildren(cells?.id as never); + expect(children.map((one) => [one.title, one.pageLevel])).toEqual([ + ['Mitosis', 0], + ['Meiosis', 1], + ['Osmosis', 0], + ]); + const open = await pages.open(children[0]?.id as string, { viewport: null }); + const types = open.initial.blocks.map((block) => block.type); + expect(types).toContain('image'); + expect(open.initial.blocks[0]?.data).toEqual({ markdown: '# Mitosis' }); + await open.close(); + const reportSection = sections.find((one) => one.title === 'Import report'); + const reportPage = (await notes.listChildren(reportSection?.id as never))[0]; + const reportOpen = await pages.open(reportPage?.id as string, { viewport: null }); + const reportText = JSON.stringify(reportOpen.initial.blocks); + expect(reportText).toContain('slides.pdf'); + expect(reportText).toContain('slides.pdf'); + await reportOpen.close(); + }); + + it('brings ink over as strokes and attached files over as files', async () => { + const server = new MockServer(); + addOneNote(server, notebooks(), pictures); + const { client, host } = rig('alice@example.com', server); + const { notes, pages } = storage(); + const extras = { + attachBytes: async (pageId: string, bytes: ArrayBuffer, name: string, mime: string) => { + const open = await pages.open(pageId, { viewport: null }); + try { + return await open.importImage({ kind: 'bytes', bytes, name, mime }); + } finally { + await open.close(); + } + }, + }; + const [report] = await importNotebooks({ client, host, notes, pages, extras, account: 'alice@example.com' }, [ + { id: 'nb1', name: 'Biology' }, + ]); + expect(report).toMatchObject({ attached: 3, ink: 6, attachments: [] }); + const cells = (await notes.listChildren((await notes.listNotebooks())[0]?.id as never)).find( + (one) => one.title === 'Cells', + ); + const first = (await notes.listChildren(cells?.id as never))[0]; + const open = await pages.open(first?.id as string, { viewport: null }); + const types = open.initial.blocks.map((block) => block.type); + expect(types).toContain('file'); + expect(types).toContain('ink'); + expect((await open.ink?.readAll())?.length ?? 0).toBeGreaterThan(0); + await open.close(); + }); + + it('adds only new pages the second time and leaves changed pages alone', async () => { + const server = new MockServer(); + const book = notebooks(); + addOneNote(server, book, pictures); + const { client, host } = rig('alice@example.com', server); + const { notes, pages } = storage(); + const deps = { client, host, notes, pages, account: 'alice@example.com' }; + await importNotebooks(deps, [{ id: 'nb1', name: 'Biology' }]); + + // OneNote gets a new page, and an old one changes. + const section = book[0]?.sections[0]; + section?.pages.push(page('p5', 'Cytokinesis')); + const changed = section?.pages[0]; + if (changed) changed.modified = '2026-10-05T10:00:00Z'; + const [report] = await importNotebooks(deps, [{ id: 'nb1', name: 'Biology' }]); + expect(report).toMatchObject({ pages: 1, kept: 4, changedInOneNote: ['Mitosis'] }); + expect((await notes.listNotebooks()).map((one) => one.title)).toEqual(['Biology']); + }); + + it('keeps one account’s notebooks apart from another’s when the account is switched', async () => { + const { notes, pages } = storage(); + const server = new MockServer(); + addOneNote(server, notebooks(), pictures); + const alice = rig('alice@example.com', server); + await importNotebooks({ ...alice, notes, pages, account: 'alice@example.com' }, [{ id: 'nb1', name: 'Biology' }]); + + // Bob signs in. His notebook has the same name and a page of his own. + const bobBook: MockOneNotebook[] = [ + { + id: 'nbB', + name: 'Biology', + sections: [ + { id: 'sB', name: 'Notes', pages: [page('pB', 'Bob’s page', undefined, 0, '

    Mine.

    ')] }, + ], + }, + ]; + const bobServer = new MockServer(); + addOneNote(bobServer, bobBook, pictures); + const bob = rig('bob@example.com', bobServer); + await importNotebooks({ ...bob, notes, pages, account: 'bob@example.com' }, [{ id: 'nbB', name: 'Biology' }]); + + const books = await notes.listNotebooks(); + expect(books.map((one) => one.title).sort()).toEqual(['Biology', 'Biology (bob@example.com)']); + // Alice's notebook gained nothing from Bob. + const aliceBook = books.find((one) => one.title === 'Biology'); + const aliceSections = await notes.listChildren(aliceBook?.id as never); + expect(aliceSections.map((one) => one.title)).not.toContain('Notes'); + expect(stateName('Alice@Example.com')).toBe('onenote.alice-example-com'); + expect(stateName('Alice@Example.com')).not.toBe(stateName('bob@example.com')); + expect(alice.host.state.has(stateName('alice@example.com'))).toBe(true); + expect(alice.host.state.has(stateName('bob@example.com'))).toBe(false); + }); +}); diff --git a/app/src/features/accounts/penSync/auto.ts b/app/src/features/accounts/penSync/auto.ts new file mode 100644 index 00000000..39944819 --- /dev/null +++ b/app/src/features/accounts/penSync/auto.ts @@ -0,0 +1,27 @@ +// The quiet pen settings sync. Once pen settings have been synced by hand, OpenNote keeps them in step while the app is +// open: a minute after start and then every half hour, with no message. A failure is ignored and tried again later. +import { isEnabled } from '../../../app/flags'; +import { runPenSync, syncedService } from './commands'; + +const EVERY_MS = 30 * 60_000; +const FIRST_MS = 90_000; +let started = false; + +async function once(): Promise { + if (!isEnabled('accounts.penSync')) return; + try { + const service = await syncedService(); + if (service) await runPenSync(service); + } catch { + // Not now. The next run tries again. + } +} + +export function startAutoSync(): void { + if (started || typeof window === 'undefined') return; + started = true; + window.setTimeout(() => { + void once(); + window.setInterval(() => void once(), EVERY_MS); + }, FIRST_MS); +} diff --git a/app/src/features/accounts/penSync/channel.ts b/app/src/features/accounts/penSync/channel.ts new file mode 100644 index 00000000..9780ffb3 --- /dev/null +++ b/app/src/features/accounts/penSync/channel.ts @@ -0,0 +1,102 @@ +// The settings channel: one small JSON file in the app's own hidden folder of the linked cloud account. On Google that is +// the app data folder of Drive (only OpenNote can see it, and it does not show in the person's Drive). On Microsoft it is +// the app folder of OneDrive (`Apps/OpenNote`). The file holds the items and nothing else; no token is in it. +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { RemoteDoc } from './merge'; + +export type PenService = 'google' | 'microsoft'; + +export interface Channel { + service: PenService; + /** The file's content, or null when there is none yet or it is not ours to read. */ + read(): Promise; + write(doc: RemoteDoc): Promise; +} + +const NAME = 'opennote-settings.json'; + +function parse(text: string): RemoteDoc | null { + try { + const value = JSON.parse(text) as Partial | null; + if (!value || value.v !== 1 || typeof value.items !== 'object' || value.items === null) return null; + const items: RemoteDoc['items'] = {}; + for (const [key, item] of Object.entries(value.items)) { + if (item && typeof item.updated === 'string') items[key] = item; + } + return { v: 1, items }; + } catch { + return null; + } +} + +const DRIVE = 'https://www.googleapis.com/drive/v3/files'; +const DRIVE_UPLOAD = 'https://www.googleapis.com/upload/drive/v3/files'; + +function google(client: ConnectorsClient): Channel { + const access = ['driveAppData']; + async function find(): Promise { + const reply = await requestConnector<{ files?: { id: string }[] }>( + 'google', + access, + { url: DRIVE, query: { spaces: 'appDataFolder', q: `name='${NAME}'`, fields: 'files(id)', pageSize: 1 } }, + client, + ); + return reply.data?.files?.[0]?.id ?? null; + } + return { + service: 'google', + async read() { + const id = await find(); + if (!id) return null; + const reply = await requestConnector( + 'google', + access, + { url: `${DRIVE}/${encodeURIComponent(id)}`, query: { alt: 'media' } }, + client, + ); + return parse(reply.text); + }, + async write(doc) { + const id = await find(); + const boundary = `opennote${Math.random().toString(36).slice(2, 12)}`; + const metadata = id ? {} : { name: NAME, parents: ['appDataFolder'] }; + const body = + `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${JSON.stringify(metadata)}\r\n` + + `--${boundary}\r\nContent-Type: application/json\r\n\r\n${JSON.stringify(doc)}\r\n--${boundary}--`; + const answer = await client.request('google', access, { + method: id ? 'PATCH' : 'POST', + url: `${DRIVE_UPLOAD}${id ? `/${encodeURIComponent(id)}` : ''}?uploadType=multipart&fields=id`, + contentType: `multipart/related; boundary=${boundary}`, + body, + }); + if (answer.status < 200 || answer.status >= 300) throw new ConnectorHttpError(answer.status, answer.body); + }, + }; +} + +const APPROOT = `https://graph.microsoft.com/v1.0/me/drive/special/approot:/${NAME}:/content`; + +function microsoft(client: ConnectorsClient): Channel { + const access = ['onedriveFiles']; + return { + service: 'microsoft', + async read() { + const reply = await requestConnector('microsoft', access, { url: APPROOT, accept: [404] }, client); + return reply.status === 404 ? null : parse(reply.text); + }, + async write(doc) { + const answer = await client.request('microsoft', access, { + method: 'PUT', + url: APPROOT, + contentType: 'application/json', + body: JSON.stringify(doc), + }); + if (answer.status < 200 || answer.status >= 300) throw new ConnectorHttpError(answer.status, answer.body); + }, + }; +} + +export function channelFor(service: PenService, client: ConnectorsClient): Channel { + return service === 'google' ? google(client) : microsoft(client); +} diff --git a/app/src/features/accounts/penSync/commands.ts b/app/src/features/accounts/penSync/commands.ts new file mode 100644 index 00000000..bb6a3e61 --- /dev/null +++ b/app/src/features/accounts/penSync/commands.ts @@ -0,0 +1,104 @@ +// Sync pen settings: the pens, their colors, and the drawing bar's choices follow the person to their other devices, through +// a small file in the app folder of the cloud account that is linked (Google or Microsoft). The first sync asks which +// account when both are linked; after that the sync runs by itself while the app is open, with no message. +import { getSettings, updateSettings } from '../../../state/settings'; +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { connectors } from '../../connectors'; +import { inkPrefs, setInkPrefs } from '../../ink'; +import type { InkPrefs } from '../../ink'; +import { accountsHost } from '../host'; +import { attempt, isConnected, tellNotConnected } from '../run'; +import { loadState, saveState } from '../state'; +import { pickOne } from '../ui/prompts'; +import { channelFor } from './channel'; +import type { PenService } from './channel'; +import { PREF_KEYS } from './items'; +import type { Patches, PenState } from './items'; +import { EMPTY_SYNC } from './merge'; +import type { PenSyncState } from './merge'; +import { syncPenSettings } from './sync'; +import type { PenDevice } from './sync'; + +const NAMES: Record = { google: 'Google', microsoft: 'Microsoft' }; +const CHOICE = 'pensync'; +const stateName = (service: PenService): string => `pensync.${service}`; + +/** This device's pen settings, from the settings and the ink preferences. */ +export function thisDevice(): PenDevice { + return { + read(): PenState { + const settings = getSettings(); + const prefs = inkPrefs.get(); + return { + pens: settings.ink.pens, + formattingBar: settings.editing.formattingBar, + handedness: settings.ink.handedness, + prefs: Object.fromEntries(PREF_KEYS.map((key) => [key, prefs[key]])) as PenState['prefs'], + }; + }, + async apply(patches: Patches) { + const patch: Record> = {}; + if (patches.pens) patch.ink = { ...patch.ink, pens: patches.pens }; + if (patches.handedness) patch.ink = { ...patch.ink, handedness: patches.handedness }; + if (patches.formattingBar) patch.editing = { formattingBar: patches.formattingBar }; + if (Object.keys(patch).length > 0) await updateSettings(patch as Parameters[0]); + if (patches.prefs) setInkPrefs(patches.prefs as Partial); + }, + }; +} + +/** The account the pen settings go through: the one chosen before, else the one linked, else the person chooses. */ +async function serviceToUse(): Promise { + const host = accountsHost(); + const linked: PenService[] = []; + for (const service of ['google', 'microsoft'] as const) if (await isConnected(service)) linked.push(service); + const saved = (await loadState<{ service: PenService | null }>(CHOICE, { service: null }, host)).service; + if (saved && linked.includes(saved)) return saved; + if (linked.length === 0) return null; + if (linked.length === 1) return linked[0] ?? null; + const chosen = await pickOne({ + title: t('accounts.penSync.pick.title'), + description: t('accounts.penSync.pick.description'), + confirmLabel: t('accounts.penSync.pick.confirm'), + choices: linked.map((service) => ({ id: service, label: NAMES[service], value: service })), + }); + if (chosen) await saveState(CHOICE, { service: chosen }, host); + return chosen; +} + +/** One sync, saved. Throws what the service says. */ +export async function runPenSync(service: PenService) { + const host = accountsHost(); + const state = await loadState(stateName(service), EMPTY_SYNC, host); + const done = await syncPenSettings(channelFor(service, connectors()), thisDevice(), state); + await saveState(stateName(service), done.state, host); + return done.result; +} + +/** The service the quiet sync uses, or null when pen settings have not been synced on this device. */ +export async function syncedService(): Promise { + const saved = (await loadState<{ service: PenService | null }>(CHOICE, { service: null }, accountsHost())).service; + return saved && (await isConnected(saved)) ? saved : null; +} + +export async function syncNow(): Promise { + const service = await serviceToUse(); + if (!service) return tellNotConnected(`${NAMES.google} or ${NAMES.microsoft}`); + await attempt(NAMES[service], async () => { + announce(t('accounts.penSync.working', { service: NAMES[service] })); + const result = await runPenSync(service); + const message = + result.pushed + result.pulled === 0 + ? t('accounts.penSync.upToDate') + : t('accounts.penSync.done', { pushed: result.pushed, pulled: result.pulled }); + showToast({ message }); + announce(message); + }); +} + +/** Stops the syncing on this device. The file at the service stays, and other devices keep using it. */ +export async function turnOff(): Promise { + await saveState(CHOICE, { service: null }, accountsHost()); + showToast({ message: t('accounts.penSync.off') }); +} diff --git a/app/src/features/accounts/penSync/items.ts b/app/src/features/accounts/penSync/items.ts new file mode 100644 index 00000000..4e42e342 --- /dev/null +++ b/app/src/features/accounts/penSync/items.ts @@ -0,0 +1,83 @@ +// What of the pen settings is synced, as items. A pen is one item by its ID, so two devices that each add a pen keep both. +// The order of the pens is one item. The drawing bar's choices (which helpers are out, the hand, the formatting bar) are +// one item each. What is NOT synced: the settings of each physical pen (its buttons and pressure curve), because those +// belong to the pen and to the screen it is used on. +import type { PenSlot } from '../../../platform/bindings/PenSlot'; +import type { Items } from './merge'; + +export const ORDER = 'pens:order'; +const PEN = 'pen:'; +const BAR = 'bar:'; + +/** The bar choices that are kept in the ink preferences of the device. */ +export const PREF_KEYS = ['hover', 'ruler', 'protractor', 'gridSnap', 'gridMm', 'zoomBox', 'penEdit'] as const; +export type PrefKey = (typeof PREF_KEYS)[number]; + +export interface PenState { + pens: readonly PenSlot[]; + formattingBar: string; + handedness: string; + prefs: Readonly>; +} + +/** The device's pen settings as items. */ +export function collect(state: PenState): Items { + const items: Items = { [ORDER]: state.pens.map((pen) => pen.id) }; + for (const pen of state.pens) items[`${PEN}${pen.id}`] = pen; + items[`${BAR}formattingBar`] = state.formattingBar; + items[`${BAR}handedness`] = state.handedness; + for (const key of PREF_KEYS) items[`${BAR}${key}`] = state.prefs[key]; + return items; +} + +const isPen = (value: unknown): value is PenSlot => + typeof value === 'object' && + value !== null && + typeof (value as PenSlot).id === 'string' && + ['pen', 'pencil', 'highlighter'].includes((value as PenSlot).tool) && + typeof (value as PenSlot).color === 'string' && + typeof (value as PenSlot).width === 'number'; + +export interface Patches { + pens?: PenSlot[]; + formattingBar?: string; + handedness?: string; + prefs?: Partial>; +} + +/** + * What to change on the device for items taken from the service. A pen that is not valid is left out, so a damaged file + * can't break the toolbar. The pens keep the order the service gave, with any this device has that the order lacks last. + */ +export function patchesFor(state: PenState, apply: Items, remove: readonly string[]): Patches { + const patches: Patches = {}; + const byId = new Map(state.pens.map((pen) => [pen.id, pen])); + let touched = false; + for (const [key, value] of Object.entries(apply)) { + if (key.startsWith(PEN) && isPen(value)) { + const id = key.slice(PEN.length); + byId.set(id, { ...value, id }); + touched = true; + } + } + for (const key of remove) { + if (key.startsWith(PEN)) touched = byId.delete(key.slice(PEN.length)) || touched; + } + const order = Array.isArray(apply[ORDER]) + ? (apply[ORDER] as unknown[]).filter((id): id is string => typeof id === 'string') + : null; + if (touched || order) { + const wanted = order ?? state.pens.map((pen) => pen.id); + const ids = [...wanted.filter((id) => byId.has(id)), ...[...byId.keys()].filter((id) => !wanted.includes(id))]; + patches.pens = ids.flatMap((id) => byId.get(id) ?? []); + } + const bar = apply[`${BAR}formattingBar`]; + if (typeof bar === 'string') patches.formattingBar = bar; + const hand = apply[`${BAR}handedness`]; + if (typeof hand === 'string') patches.handedness = hand; + for (const key of PREF_KEYS) { + const value = apply[`${BAR}${key}`]; + if (value !== undefined) (patches.prefs ??= {})[key] = value; + } + return patches; +} diff --git a/app/src/features/accounts/penSync/merge.ts b/app/src/features/accounts/penSync/merge.ts new file mode 100644 index 00000000..6b880603 --- /dev/null +++ b/app/src/features/accounts/penSync/merge.ts @@ -0,0 +1,103 @@ +// Merging pen settings, one item at a time. An item is one pen (with its tool, color, and width), the order of the pens, +// or one choice about the drawing bar (the formatting bar, the hand, the ruler). Each device keeps what both sides held +// after the last sync (the base). An item that differs from the base has changed. A change on one side is copied to the +// other. A change on both sides keeps the newer one. An item removed on one side is removed on the other only if the +// other side did not change it. Nothing here touches a device or a service: it takes values and answers values. + +export type Items = Record; + +export interface RemoteItem { + value?: unknown; + /** When the item was last changed, as an ISO time. */ + updated: string; + deleted?: true; +} + +export interface RemoteDoc { + v: 1; + items: Record; +} + +export interface PenSyncState { + /** For each item, what both sides held after the last sync. */ + base: Record; + /** When this device first saw its own change to an item that had not been sent yet. */ + seen: Record; + lastSync: string | null; +} + +export const EMPTY_SYNC: PenSyncState = { base: {}, seen: {}, lastSync: null }; + +export interface MergeResult { + doc: RemoteDoc; + /** Values this device should take on. */ + apply: Items; + /** Items this device should drop. */ + remove: string[]; + state: PenSyncState; + pushed: number; + pulled: number; + /** Items changed on both sides, where the newer change won. */ + conflicts: number; + /** Whether the file at the service needs writing. */ + dirty: boolean; +} + +const hashOf = (value: unknown): string | null => (value === undefined ? null : JSON.stringify(value)); + +export function mergeItems( + local: Items, + remote: RemoteDoc | null, + previous: PenSyncState, + now: number = Date.now(), +): MergeResult { + const stamp = new Date(now).toISOString(); + const doc: RemoteDoc = { v: 1, items: { ...(remote?.items ?? {}) } }; + const state: PenSyncState = structuredClone(previous); + const apply: Items = {}; + const remove: string[] = []; + const firstWithFile = previous.lastSync === null && remote !== null; + const out = { pushed: 0, pulled: 0, conflicts: 0, dirty: remote === null }; + + const keys = new Set([...Object.keys(local), ...Object.keys(doc.items), ...Object.keys(state.base)]); + for (const key of keys) { + const mine = local[key]; + const theirs = doc.items[key]; + const hashMine = hashOf(mine); + const hashTheirs = theirs && !theirs.deleted ? hashOf(theirs.value) : null; + const base = state.base[key] ?? null; + if (hashMine === hashTheirs) { + if (hashMine === null) delete state.base[key]; + else state.base[key] = hashMine; + delete state.seen[key]; + continue; + } + const changedHere = hashMine !== base; + const changedThere = hashTheirs !== base; + let push = changedHere; + if (changedHere && changedThere) { + state.seen[key] ??= now; + // A device that has never synced has only its defaults, so on a first sync the account's settings win. + push = !firstWithFile && (state.seen[key] ?? now) >= Date.parse(theirs?.updated ?? '1970-01-01T00:00:00Z'); + out.conflicts += 1; + } + if (push) { + doc.items[key] = mine === undefined ? { deleted: true, updated: stamp } : { value: mine, updated: stamp }; + if (hashMine === null) delete state.base[key]; + else state.base[key] = hashMine; + out.pushed += 1; + out.dirty = true; + } else if (hashTheirs === null) { + remove.push(key); + delete state.base[key]; + out.pulled += 1; + } else { + apply[key] = theirs?.value; + state.base[key] = hashTheirs; + out.pulled += 1; + } + delete state.seen[key]; + } + state.lastSync = stamp; + return { doc, apply, remove, state, ...out }; +} diff --git a/app/src/features/accounts/penSync/penSync.test.ts b/app/src/features/accounts/penSync/penSync.test.ts new file mode 100644 index 00000000..dfeaefe4 --- /dev/null +++ b/app/src/features/accounts/penSync/penSync.test.ts @@ -0,0 +1,185 @@ +// checks-disable-file brand-consistency: the test colors are inputs to the code under test, not design colors + +// Pen settings sync: the per-item merge (pure), the items a device offers and takes. It ends with a full sync of two devices +// through mocks of Google's app data folder and OneDrive's app folder. +import { describe, expect, it } from 'vitest'; +import type { PenSlot } from '../../../platform/bindings/PenSlot'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { addAppFolders } from '../../../../../tests/mock-servers/storage'; +import { createFakeConnectors } from '../../connectors'; +import { channelFor } from './channel'; +import { collect, patchesFor, PREF_KEYS } from './items'; +import type { Patches, PenState } from './items'; +import { EMPTY_SYNC, mergeItems } from './merge'; +import { syncPenSettings } from './sync'; +import type { PenDevice } from './sync'; + +const pen = (id: string, color = 'Ink', width = 0.5): PenSlot => ({ id, tool: 'pen', color, width }); +const prefs = { + hover: true, + ruler: false, + protractor: false, + gridSnap: false, + gridMm: 5, + zoomBox: false, + penEdit: { strike: true }, +}; + +const stateOf = (pens: PenSlot[], extra: Partial = {}): PenState => ({ + pens, + formattingBar: 'touchAndPen', + handedness: 'right', + prefs, + ...extra, +}); + +/** A device in memory that applies the patches the way the app does. */ +function device(initial: PenState): PenDevice & { state: PenState; applied: Patches[] } { + const self = { + state: initial, + applied: [] as Patches[], + read: () => self.state, + apply(patches: Patches) { + self.applied.push(patches); + self.state = { + ...self.state, + ...(patches.pens ? { pens: patches.pens } : {}), + ...(patches.formattingBar ? { formattingBar: patches.formattingBar } : {}), + ...(patches.handedness ? { handedness: patches.handedness } : {}), + prefs: { ...self.state.prefs, ...(patches.prefs ?? {}) } as PenState['prefs'], + }; + }, + }; + return self; +} + +describe('the items', () => { + it('offers each pen by its ID, the order, and the bar choices, and not the settings of a physical pen', () => { + const items = collect(stateOf([pen('a'), pen('b')])); + expect(Object.keys(items).sort()).toEqual( + [ + 'pens:order', + 'pen:a', + 'pen:b', + 'bar:formattingBar', + 'bar:handedness', + ...PREF_KEYS.map((key) => `bar:${key}`), + ].sort(), + ); + expect(items['pens:order']).toEqual(['a', 'b']); + }); + + it('takes pens in the order given, keeps pens the order lacks, and ignores a damaged pen', () => { + const state = stateOf([pen('a'), pen('local')]); + const patches = patchesFor( + state, + { 'pens:order': ['c', 'a', 'broken'], 'pen:c': pen('x', 'Sky'), 'pen:broken': { id: 'broken', tool: 'laser' } }, + [], + ); + expect(patches.pens?.map((one) => one.id)).toEqual(['c', 'a', 'local']); + expect(patches.pens?.[0]).toMatchObject({ id: 'c', color: 'Sky' }); + }); + + it('removes a pen that was removed elsewhere', () => { + expect(patchesFor(stateOf([pen('a'), pen('b')]), {}, ['pen:a']).pens?.map((one) => one.id)).toEqual(['b']); + }); +}); + +describe('the merge', () => { + const at = Date.parse('2026-10-05T12:00:00Z'); + + it('sends everything when there is no file, and sends nothing the second time', () => { + const items = collect(stateOf([pen('a')])); + const first = mergeItems(items, null, EMPTY_SYNC, at); + expect(first.dirty).toBe(true); + expect(first.pushed).toBe(Object.keys(items).length); + const second = mergeItems(items, first.doc, first.state, at + 1000); + expect(second).toMatchObject({ dirty: false, pushed: 0, pulled: 0, conflicts: 0 }); + }); + + it('merges a pen added on each device without losing either', () => { + const a = mergeItems(collect(stateOf([pen('a'), pen('p1')])), null, EMPTY_SYNC, at); + // Device B has synced once, then adds its own pen. + const bFirst = mergeItems(collect(stateOf([pen('a'), pen('p1')])), a.doc, a.state, at + 1); + const bItems = collect(stateOf([pen('a'), pen('p1'), pen('fromB')])); + const b = mergeItems(bItems, a.doc, bFirst.state, at + 2); + // Device A adds a different pen before it hears from B. + const aItems = collect(stateOf([pen('a'), pen('p1'), pen('fromA')])); + const aAgain = mergeItems(aItems, b.doc, a.state, at + 3); + expect(Object.keys(aAgain.apply)).toContain('pen:fromB'); + expect(aAgain.doc.items['pen:fromA']?.value).toMatchObject({ id: 'fromA' }); + expect(aAgain.doc.items['pen:fromB']?.value).toMatchObject({ id: 'fromB' }); + }); + + it('keeps the newer change when both devices changed one pen, and the account’s on a first sync', () => { + const start = mergeItems(collect(stateOf([pen('a', 'Ink')])), null, EMPTY_SYNC, at); + const remote = structuredClone(start.doc); + remote.items['pen:a'] = { value: pen('a', 'Sky'), updated: '2026-10-05T13:00:00Z' }; + // Changed here at 12:30, there at 13:00: there is newer. + const state = structuredClone(start.state); + state.seen['pen:a'] = Date.parse('2026-10-05T12:30:00Z'); + const lost = mergeItems(collect(stateOf([pen('a', 'Honey')])), remote, state, Date.parse('2026-10-05T14:00:00Z')); + expect(lost.conflicts).toBe(1); + expect(lost.apply['pen:a']).toMatchObject({ color: 'Sky' }); + // Changed here at 13:30: here is newer. + state.seen['pen:a'] = Date.parse('2026-10-05T13:30:00Z'); + const won = mergeItems(collect(stateOf([pen('a', 'Honey')])), remote, state, Date.parse('2026-10-05T14:00:00Z')); + expect(won.doc.items['pen:a']?.value).toMatchObject({ color: 'Honey' }); + // A device that has never synced takes the account's version. + const first = mergeItems( + collect(stateOf([pen('a', 'Honey')])), + remote, + EMPTY_SYNC, + Date.parse('2026-10-05T14:00:00Z'), + ); + expect(first.apply['pen:a']).toMatchObject({ color: 'Sky' }); + }); + + it('removes a pen on both devices when one removed it and the other did not change it', () => { + const both = collect(stateOf([pen('a'), pen('b')])); + const first = mergeItems(both, null, EMPTY_SYNC, at); + const removedHere = mergeItems(collect(stateOf([pen('a')])), first.doc, first.state, at + 1); + expect(removedHere.doc.items['pen:b']).toMatchObject({ deleted: true }); + const other = mergeItems(both, removedHere.doc, first.state, at + 2); + expect(other.remove).toContain('pen:b'); + }); +}); + +describe('two devices through the cloud', () => { + it('moves a pen and the toolbar from one device to the other through Google’s app data folder', async () => { + const server = new MockServer(); + const folders = addAppFolders(server); + const { client } = createFakeConnectors({ connected: { google: 'sam@example.com' }, respond: server.respond }); + const channel = channelFor('google', client); + const laptop = device( + stateOf([pen('a'), pen('custom', '#aa2200', 1.2)], { formattingBar: 'always', handedness: 'left' }), + ); + const first = await syncPenSettings(channel, laptop, EMPTY_SYNC); + expect(first.result.pushed).toBeGreaterThan(0); + expect(folders.drive.parents).toEqual(['appDataFolder']); + expect(JSON.parse(folders.drive.content).v).toBe(1); + + const tablet = device(stateOf([pen('a')])); + const second = await syncPenSettings(channel, tablet, EMPTY_SYNC); + expect(second.result.pulled).toBeGreaterThan(0); + expect(tablet.state.pens.map((one) => one.id)).toEqual(['a', 'custom']); + expect(tablet.state.formattingBar).toBe('always'); + expect(tablet.state.handedness).toBe('left'); + // A third sync with nothing new writes nothing. + const versions = folders.drive.versions; + await syncPenSettings(channel, tablet, second.state); + expect(folders.drive.versions).toBe(versions); + }); + + it('works through OneDrive’s app folder too, and reads a damaged file as none', async () => { + const server = new MockServer(); + const folders = addAppFolders(server); + const { client } = createFakeConnectors({ connected: { microsoft: 'sam@example.com' }, respond: server.respond }); + const channel = channelFor('microsoft', client); + const desk = device(stateOf([pen('a'), pen('b')])); + await syncPenSettings(channel, desk, EMPTY_SYNC); + expect(JSON.parse(folders.onedrive.content ?? '{}').items['pen:b'].value.id).toBe('b'); + folders.onedrive.content = '{ not json'; + expect(await channel.read()).toBeNull(); + }); +}); diff --git a/app/src/features/accounts/penSync/sync.ts b/app/src/features/accounts/penSync/sync.ts new file mode 100644 index 00000000..9e6589a0 --- /dev/null +++ b/app/src/features/accounts/penSync/sync.ts @@ -0,0 +1,38 @@ +// One sync of the pen settings: read the file at the service, merge it with the device's items, write the file if it +// changed, and apply what came from the service to the device. The device and the service are passed in, so the sync is +// tested without a screen or an account. +import type { Channel } from './channel'; +import { collect, patchesFor } from './items'; +import type { Patches, PenState } from './items'; +import { mergeItems } from './merge'; +import type { PenSyncState } from './merge'; + +export interface PenDevice { + read(): PenState; + apply(patches: Patches): Promise | void; +} + +export interface PenSyncResult { + pushed: number; + pulled: number; + conflicts: number; +} + +export async function syncPenSettings( + channel: Channel, + device: PenDevice, + state: PenSyncState, + now?: number, +): Promise<{ state: PenSyncState; result: PenSyncResult }> { + const remote = await channel.read(); + const before = device.read(); + const merged = mergeItems(collect(before), remote, state, now); + if (merged.dirty) await channel.write(merged.doc); + if (Object.keys(merged.apply).length > 0 || merged.remove.length > 0) { + await device.apply(patchesFor(before, merged.apply, merged.remove)); + } + return { + state: merged.state, + result: { pushed: merged.pushed, pulled: merged.pulled, conflicts: merged.conflicts }, + }; +} diff --git a/app/src/features/accounts/readwise/api.ts b/app/src/features/accounts/readwise/api.ts new file mode 100644 index 00000000..102a85dd --- /dev/null +++ b/app/src/features/accounts/readwise/api.ts @@ -0,0 +1,58 @@ +// Readwise's export API (v2): every book with the highlights that changed after a time, a page at a time. The token is +// the host's; this code only asks. `includeDeleted` lets a highlight deleted at Readwise leave the notebook too. + +import type { ConnectorReply, ConnectorsClient } from '../../connectors'; +import { requestConnector } from '../../connectors'; + +export interface ExportHighlight { + id: number; + text: string; + note?: string | null; + location?: number | null; + location_type?: string | null; + is_deleted?: boolean; + tags?: { name: string }[]; + updated?: string; +} + +export interface ExportBook { + user_book_id: number; + title: string; + author?: string | null; + category?: string | null; + source?: string | null; + readwise_url?: string | null; + summary?: string | null; + highlights: ExportHighlight[]; +} + +interface ExportPage { + nextPageCursor?: string | null; + results?: ExportBook[]; +} + +/** The most pages one sync reads, so a loop at the service can't run forever. */ +export const MAX_PAGES = 500; + +/** Reads the export page by page. `onPage` runs for each page before the next is asked for. */ +export async function readExport( + client: ConnectorsClient, + updatedAfter: string | null, + onPage: (books: ExportBook[]) => Promise, +): Promise { + let cursor: string | null = null; + for (let page = 0; page < MAX_PAGES; page += 1) { + const reply: ConnectorReply = await requestConnector( + 'readwise', + ['readwiseRead'], + { + url: 'https://readwise.io/api/v2/export/', + query: { updatedAfter, pageCursor: cursor, includeDeleted: 'true' }, + }, + client, + ); + await onPage(reply.data?.results ?? []); + cursor = reply.data?.nextPageCursor ?? null; + if (!cursor) return; + } +} diff --git a/app/src/features/accounts/readwise/commands.ts b/app/src/features/accounts/readwise/commands.ts new file mode 100644 index 00000000..8b783d17 --- /dev/null +++ b/app/src/features/accounts/readwise/commands.ts @@ -0,0 +1,41 @@ +// The Sync Readwise command: runs a sync with the app's own notes, pages, and connectors, and says how it went. + +import { commandContext } from '../../../commands/registry'; +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { ConnectorHttpError } from '../../connectors'; +import { accountsHost } from '../host'; +import { services } from '../notebook'; +import { attempt, isConnected, tellNotConnected } from '../run'; +import { syncReadwise } from './sync'; + +const WORKING = 'readwise-sync'; + +export async function sync(): Promise { + if (!(await isConnected('readwise'))) return tellNotConnected('Readwise'); + const { notes, pages } = services(); + const working = showToast({ id: WORKING, message: t('accounts.readwise.working') }); + const finished = await attempt('Readwise', async () => { + try { + const result = await syncReadwise({ + notes, + pages, + host: accountsHost(), + client: commandContext('palette').platform.connectors, + }); + const none = result.added + result.updated + result.removed === 0; + const message = none + ? t('accounts.readwise.upToDate') + : t('accounts.readwise.done', { added: result.added, updated: result.updated, removed: result.removed }); + showToast({ id: WORKING, message }); + announce(message); + } catch (error) { + if (error instanceof ConnectorHttpError && error.status === 429) { + showToast({ id: WORKING, message: t('accounts.readwise.rateLimited'), tone: 'danger' }); + return; + } + throw error; + } + }); + if (!finished) working.dismiss(); +} diff --git a/app/src/features/accounts/readwise/format.ts b/app/src/features/accounts/readwise/format.ts new file mode 100644 index 00000000..a97bb665 --- /dev/null +++ b/app/src/features/accounts/readwise/format.ts @@ -0,0 +1,62 @@ +// How a Readwise book and its highlights are written as Markdown blocks. Pure functions of the data and the strings, so +// a sync that runs twice writes the same text. + +import { t } from '../../../strings/t'; +import type { ExportBook, ExportHighlight } from './api'; + +/** The section a book goes in, by its Readwise category. */ +export function categoryTitle(category: string | null | undefined): string { + switch ((category ?? '').toLowerCase()) { + case 'books': + return t('accounts.readwise.categories.books'); + case 'articles': + return t('accounts.readwise.categories.articles'); + case 'tweets': + return t('accounts.readwise.categories.tweets'); + case 'podcasts': + return t('accounts.readwise.categories.podcasts'); + case 'supplementals': + return t('accounts.readwise.categories.supplementals'); + default: + return t('accounts.readwise.categories.other'); + } +} + +/** The block at the top of a book's page: who wrote it, where it came from, and a link back. */ +export function headerMarkdown(book: ExportBook): string { + const lines: string[] = []; + if (book.author) lines.push(t('accounts.readwise.header.by', { author: book.author })); + if (book.source) lines.push(t('accounts.readwise.header.source', { source: book.source })); + if (book.summary) lines.push(t('accounts.readwise.header.summary', { summary: book.summary })); + if (book.readwise_url) lines.push(t('accounts.readwise.header.open', { link: book.readwise_url })); + return lines.join('\n\n'); +} + +function locationLine(highlight: ExportHighlight): string | null { + if (highlight.location === null || highlight.location === undefined) return null; + const location = String(highlight.location); + switch (highlight.location_type) { + case 'page': + return t('accounts.readwise.highlight.page', { location }); + case 'time_offset': + return t('accounts.readwise.highlight.time', { location }); + default: + return t('accounts.readwise.highlight.location', { location }); + } +} + +/** One highlight as one block: the quote, then the note, the place, and the tags when it has them. */ +export function highlightMarkdown(highlight: ExportHighlight): string { + const quote = highlight.text + .trim() + .split(/\r?\n+/) + .map((line) => `> ${line}`) + .join('\n'); + const extras: string[] = []; + if (highlight.note?.trim()) extras.push(t('accounts.readwise.highlight.note', { note: highlight.note.trim() })); + const place = locationLine(highlight); + if (place) extras.push(place); + const tags = (highlight.tags ?? []).map((tag) => tag.name).filter(Boolean); + if (tags.length > 0) extras.push(t('accounts.readwise.highlight.tags', { tags: tags.join(', ') })); + return [quote, ...extras].join('\n\n'); +} diff --git a/app/src/features/accounts/readwise/sync.test.ts b/app/src/features/accounts/readwise/sync.test.ts new file mode 100644 index 00000000..465cc3f9 --- /dev/null +++ b/app/src/features/accounts/readwise/sync.test.ts @@ -0,0 +1,116 @@ +// Sync Readwise against a mock Readwise: the first run writes a notebook, sections, and a page for each book; the next +// run reads only what changed and updates, adds, and removes in place; the cursor and the block map survive a restart. +import { describe, expect, it } from 'vitest'; +import { readwiseMock } from '../../../../../tests/mock-servers/readwise'; +import type { MockBook } from '../../../../../tests/mock-servers/readwise'; +import { createMemoryPageService } from '../../../services/pages/memory'; +import { createMemoryNotesService } from '../../../services/notes/memory'; +import { createFakeConnectors } from '../../connectors'; +import { createFakeAccountsHost } from '../fakeHost'; +import { STATE_NAME, syncReadwise } from './sync'; +import type { ReadwiseState } from './sync'; + +const book = (id: number, title: string, category: string, highlights: MockBook['highlights']): MockBook => ({ + user_book_id: id, + title, + author: 'Ada Writer', + category, + readwise_url: `https://readwise.io/bookreview/${id}`, + highlights, +}); + +function rig(books: MockBook[]) { + const mock = readwiseMock(books); + const connectors = createFakeConnectors({ connected: { readwise: '' }, respond: mock.server.respond }); + const host = createFakeAccountsHost(() => connectors.client); + const notes = createMemoryNotesService({ seed: 'empty' }); + const pages = createMemoryPageService([], { + missing: (id) => ({ id, title: '', created: '', modified: '', tags: [], view: {}, blocks: [], assets: {} }), + }); + const deps = { notes, pages, host, client: connectors.client, now: () => new Date('2026-10-07T12:00:00Z') }; + return { mock, deps, host, connectors, notes, pages }; +} + +const textOf = (page: ReturnType, id: string) => + (page.held(id)?.blocks ?? []).map((block) => String(block.data.markdown)); + +describe('syncReadwise', () => { + it('writes a notebook with a section for each category and a page for each book', async () => { + const { deps, notes, pages } = rig([ + book(1, 'The Overstory', 'books', [ + { id: 11, text: 'Trees talk.', note: 'Remember this', location: 40, updated: '2026-10-01T00:00:00Z' }, + { id: 12, text: 'Roots\nand shoots.', updated: '2026-10-01T00:00:00Z' }, + ]), + book(2, 'A long read', 'articles', [{ id: 21, text: 'First point.', updated: '2026-10-02T00:00:00Z' }]), + ]); + const result = await syncReadwise(deps); + expect(result).toMatchObject({ books: 2, added: 3, updated: 0, removed: 0 }); + const [notebook] = await notes.listNotebooks(); + expect(notebook?.title).toBe('Readwise'); + const sections = await notes.listChildren(notebook?.id ?? ''); + expect(sections.map((one) => one.title)).toEqual(['Books', 'Articles']); + const [overstory] = await notes.listChildren(sections[0]?.id ?? ''); + expect(overstory?.title).toBe('The Overstory'); + const blocks = textOf(pages, overstory?.id ?? ''); + expect(blocks[0]).toContain('By Ada Writer'); + expect(blocks[1]).toBe('> Trees talk.\n\nNote: Remember this\n\nLocation 40'); + expect(blocks[2]).toBe('> Roots\n> and shoots.'); + }); + + it('reads only what changed next time, and updates, adds, and removes in place', async () => { + const { deps, mock, host, notes, pages } = rig([ + book(1, 'The Overstory', 'books', [ + { id: 11, text: 'Trees talk.', updated: '2026-10-01T00:00:00Z' }, + { id: 12, text: 'Roots.', updated: '2026-10-01T00:00:00Z' }, + ]), + ]); + await syncReadwise(deps); + const saved = (await host.stateGet(STATE_NAME)) as ReadwiseState; + expect(saved.updatedAfter).toBe('2026-10-07T11:59:00.000Z'); + // The person adds a line of their own to the page, which a sync must leave alone. + const pageId = saved.books['1']?.page ?? ''; + const open = await pages.open(pageId, { viewport: null }); + await open.send({ + edits: [{ edit: 'insertBlock', block: { id: 'mine', type: 'text', data: { markdown: 'My own thought.' } } }], + }); + await open.close(); + + const book1 = mock.state.books[0]; + if (!book1) throw new Error('no book'); + book1.highlights = [ + { id: 11, text: 'Trees talk, slowly.', updated: '2026-10-08T00:00:00Z' }, + { id: 12, text: 'Roots.', updated: '2026-10-08T00:00:00Z', is_deleted: true }, + { id: 13, text: 'Seeds.', updated: '2026-10-08T00:00:00Z' }, + ]; + mock.server.requests.length = 0; + const result = await syncReadwise(deps); + expect(result).toMatchObject({ books: 1, added: 1, updated: 1, removed: 1 }); + expect(mock.server.requests[0]?.query.updatedAfter).toBe('2026-10-07T11:59:00.000Z'); + expect(textOf(pages, pageId).slice(1)).toEqual(['> Trees talk, slowly.', 'My own thought.', '> Seeds.']); + expect((await notes.listNotebooks()).length).toBe(1); + }); + + it('keeps going from where it stopped when the service stops answering', async () => { + const { deps, mock, host, pages } = rig([ + book(1, 'One', 'books', [{ id: 1, text: 'A', updated: '2026-10-01T00:00:00Z' }]), + book(2, 'Two', 'books', [{ id: 2, text: 'B', updated: '2026-10-01T00:00:00Z' }]), + ]); + // The service answers the first page and then throttles: the first book is kept, the cursor is not moved. + const original = mock.server.respond; + let calls = 0; + const connectors = createFakeConnectors({ + connected: { readwise: '' }, + respond: (id, request) => (++calls > 1 ? { status: 429, contentType: null, body: '{}' } : original(id, request)), + }); + const stopped = { ...deps, client: connectors.client }; + await expect(syncReadwise(stopped)).rejects.toThrow(); + const saved = (await host.stateGet(STATE_NAME)) as ReadwiseState; + expect(saved.updatedAfter).toBeNull(); + expect(Object.keys(saved.books)).toEqual(['1']); + expect(textOf(pages, saved.books['1']?.page ?? '')).toHaveLength(2); + // The next run reads everything again and finishes without duplicating the first book's highlight. + const result = await syncReadwise(deps); + expect(result).toMatchObject({ books: 2, added: 1 }); + expect(textOf(pages, saved.books['1']?.page ?? '')).toHaveLength(2); + }); +}); diff --git a/app/src/features/accounts/readwise/sync.ts b/app/src/features/accounts/readwise/sync.ts new file mode 100644 index 00000000..a48b0755 --- /dev/null +++ b/app/src/features/accounts/readwise/sync.ts @@ -0,0 +1,174 @@ +// Sync Readwise. The first run reads every highlight, later runs read only what changed after the last successful run +// (the cursor is saved when a run finishes, and each book is saved as it is written, so an interrupted run goes on from +// where it stopped). Each book is a page in a section of the Readwise notebook, and each highlight is one block that the +// sync remembers by its Readwise ID. A highlight edited at Readwise is updated in place, one deleted there is removed, +// and every other block on the page, such as a note the person wrote, is left alone. + +import { newId } from '../../../editor/ids'; +import type { PagesClient } from '../../../platform/types'; +import type { NodeId, NodeSummary, NotesService } from '../../../services/notes/types'; +import type { NewBlock, OpenPage } from '../../../services/pages/types'; +import { t } from '../../../strings/t'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import { cleanTitle, findOrCreate } from '../notebook'; +import { loadState, saveState } from '../state'; +import { readExport } from './api'; +import type { ExportBook } from './api'; +import { categoryTitle, headerMarkdown, highlightMarkdown } from './format'; + +export const STATE_NAME = 'readwise.sync'; + +interface BookState { + page: string; + /** The block at the top of the page, which the sync writes. */ + header: string; + /** The block of each highlight, by Readwise highlight ID. */ + highlights: Record; +} + +export interface ReadwiseState { + /** Only what changed after this time is read next. Null before the first run. */ + updatedAfter: string | null; + notebook: string | null; + books: Record; +} + +const EMPTY: ReadwiseState = { updatedAfter: null, notebook: null, books: {} }; + +export interface SyncDeps { + notes: NotesService; + pages: PagesClient; + host: AccountsHost; + client: ConnectorsClient; + now?: () => Date; +} + +export interface SyncResult { + books: number; + added: number; + updated: number; + removed: number; + notebook: NodeId; +} + +/** The slack before the start of a run, so a highlight that changed while the run began is read again, not lost. */ +const OVERLAP_MS = 60_000; + +async function notebookFor(deps: SyncDeps, state: ReadwiseState): Promise { + const known = state.notebook ? await deps.notes.get(state.notebook as NodeId) : null; + return known ?? findOrCreate(deps.notes, null, 'notebook', t('accounts.readwise.notebook')); +} + +async function pageFor( + deps: SyncDeps, + section: NodeId, + book: ExportBook, + known: BookState | undefined, +): Promise<{ id: string; made: boolean }> { + const existing = known ? await deps.notes.get(known.page as NodeId) : null; + if (existing) return { id: existing.id, made: false }; + const page = await deps.notes.create({ + kind: 'page', + placement: { parentId: section, beforeId: null }, + title: cleanTitle(book.title, t('accounts.readwise.notebook')), + }); + return { id: page.id, made: true }; +} + +interface BookChange { + state: BookState; + added: number; + updated: number; + removed: number; +} + +const markdownOf = (open: OpenPage, block: string): string | null => { + const found = open.initial.blocks.find((one) => one.id === block); + const markdown = found?.data.markdown; + return found ? (typeof markdown === 'string' ? markdown : '') : null; +}; + +async function insertAfter(open: OpenPage, markdown: string, after: string | undefined): Promise { + const id = newId(); + const block: NewBlock = { id, type: 'text', data: { markdown } }; + await open.send({ edits: [{ edit: 'insertBlock', block, ...(after ? { after } : {}) }] }); + return id; +} + +/** Writes one book's header and highlights into its page. */ +async function writeBook(pages: PagesClient, pageId: string, book: ExportBook, known: BookState): Promise { + const open = await pages.open(pageId, { viewport: null }); + const state: BookState = { page: pageId, header: known.header, highlights: { ...known.highlights } }; + const change: BookChange = { state, added: 0, updated: 0, removed: 0 }; + try { + let last = open.initial.blocks.at(-1)?.id; + const header = headerMarkdown(book); + if (header) { + const current = state.header ? markdownOf(open, state.header) : null; + if (current === null) { + state.header = await insertAfter(open, header, undefined); + last ??= state.header; + } else if (current !== header) { + await open.send({ edits: [{ edit: 'setText', block: state.header, markdown: header }] }); + } + } + for (const highlight of book.highlights) { + const key = String(highlight.id); + const block = state.highlights[key]; + const present = block ? markdownOf(open, block) : null; + if (highlight.is_deleted) { + if (block && present !== null) await open.send({ edits: [{ edit: 'deleteBlocks', blocks: [block] }] }); + if (block) { + delete state.highlights[key]; + change.removed += 1; + } + continue; + } + const markdown = highlightMarkdown(highlight); + if (block && present !== null) { + if (present !== markdown) { + await open.send({ edits: [{ edit: 'setText', block, markdown }] }); + change.updated += 1; + } + continue; + } + const id = await insertAfter(open, markdown, last); + state.highlights[key] = id; + last = id; + change.added += 1; + } + await open.saveNow(); + return change; + } finally { + await open.close(); + } +} + +/** Runs one sync. Throws what the connector throws, after keeping what was written so far. */ +export async function syncReadwise(deps: SyncDeps): Promise { + const startedAt = new Date((deps.now ?? (() => new Date()))().getTime() - OVERLAP_MS).toISOString(); + const state = await loadState(STATE_NAME, EMPTY, deps.host); + const notebook = await notebookFor(deps, state); + state.notebook = notebook.id; + const result: SyncResult = { books: 0, added: 0, updated: 0, removed: 0, notebook: notebook.id }; + await readExport(deps.client, state.updatedAfter, async (books) => { + for (const book of books) { + const section = await findOrCreate(deps.notes, notebook.id, 'section', categoryTitle(book.category)); + const key = String(book.user_book_id); + const known = state.books[key]; + const page = await pageFor(deps, section.id, book, known); + const base = known && !page.made ? known : { page: page.id, header: '', highlights: {} }; + const change = await writeBook(deps.pages, page.id, book, base); + state.books[key] = change.state; + result.books += 1; + result.added += change.added; + result.updated += change.updated; + result.removed += change.removed; + await saveState(STATE_NAME, state, deps.host); + } + }); + state.updatedAfter = startedAt; + await saveState(STATE_NAME, state, deps.host); + return result; +} diff --git a/app/src/features/accounts/register.ts b/app/src/features/accounts/register.ts new file mode 100644 index 00000000..ea911b6a --- /dev/null +++ b/app/src/features/accounts/register.ts @@ -0,0 +1,259 @@ +// Registers the commands of the account features. This file loads at start-up, so it holds definitions only: each +// command's work, and the dialogs it opens, load on first use. A command is hidden while its flag is off, and it is +// always there when the flag is on: with the account not connected it says so and offers Settings. +import { isEnabled } from '../../app/flags'; +import type { FlagId } from '../../app/flags'; +import { commandContext, defineCommand } from '../../commands/registry'; +import type { CommandCategory, CommandDef } from '../../commands/types'; +import { commands } from '../../registries'; +import type { NodeId } from '../../services/notes/types'; +import type { MessageKey } from '../../strings/t'; +import { shownMounted } from '../page'; + +interface Spec { + id: string; + title: MessageKey; + keywords: MessageKey; + flag: FlagId; + category?: CommandCategory; + /** The command needs a page on screen. */ + needsPage?: boolean; + run(): Promise; +} + +const pageShown = () => shownMounted.get() !== null; + +const SPECS: readonly Spec[] = [ + { + id: 'meetings.new', + title: 'accounts.meetings.commands.new', + keywords: 'accounts.meetings.commands.keywords', + flag: 'accounts.meetings', + run: () => import('./meetings/commands').then((module) => module.newMeetingNote()), + }, + { + id: 'share.slack', + title: 'accounts.share.commands.slack', + keywords: 'accounts.share.commands.keywords', + flag: 'accounts.share', + needsPage: true, + run: () => import('./share/commands').then((module) => module.shareSlack()), + }, + { + id: 'share.teams', + title: 'accounts.share.commands.teams', + keywords: 'accounts.share.commands.keywords', + flag: 'accounts.share', + needsPage: true, + run: () => import('./share/commands').then((module) => module.shareTeams()), + }, + { + id: 'lms.bring', + title: 'accounts.lms.commands.bring', + keywords: 'accounts.lms.commands.keywords', + flag: 'accounts.lms', + run: () => import('./lms/commands').then((module) => module.bringAssignments()), + }, + { + id: 'lms.submit', + title: 'accounts.lms.commands.submit', + keywords: 'accounts.lms.commands.keywords', + flag: 'accounts.lms', + needsPage: true, + run: () => import('./lms/commands').then((module) => module.submitPage()), + }, + { + id: 'sendto.drive', + title: 'accounts.sendTo.commands.drive', + keywords: 'accounts.sendTo.commands.keywords', + flag: 'accounts.sendTo', + needsPage: true, + run: () => import('./sendto/commands').then((module) => module.sendTo('drive')), + }, + { + id: 'sendto.onedrive', + title: 'accounts.sendTo.commands.onedrive', + keywords: 'accounts.sendTo.commands.keywords', + flag: 'accounts.sendTo', + needsPage: true, + run: () => import('./sendto/commands').then((module) => module.sendTo('onedrive')), + }, + { + id: 'sendto.dropbox', + title: 'accounts.sendTo.commands.dropbox', + keywords: 'accounts.sendTo.commands.keywords', + flag: 'accounts.cloudSync', + needsPage: true, + run: () => import('./sendto/commands').then((module) => module.sendTo('dropbox')), + }, + { + id: 'sendto.box', + title: 'accounts.sendTo.commands.box', + keywords: 'accounts.sendTo.commands.keywords', + flag: 'accounts.cloudSync', + needsPage: true, + run: () => import('./sendto/commands').then((module) => module.sendTo('box')), + }, + { + id: 'sendto.webdav', + title: 'accounts.sendTo.commands.webdav', + keywords: 'accounts.sendTo.commands.keywords', + flag: 'accounts.cloudSync', + needsPage: true, + run: () => import('./sendto/commands').then((module) => module.sendTo('webdav')), + }, + { + id: 'sendto.update', + title: 'accounts.sendTo.commands.update', + keywords: 'accounts.sendTo.commands.keywords', + flag: 'accounts.sendTo', + needsPage: true, + run: () => import('./sendto/commands').then((module) => module.updateHere()), + }, + { + id: 'sendto.sync', + title: 'accounts.sendTo.commands.sync', + keywords: 'accounts.sendTo.commands.keywords', + flag: 'accounts.cloudSync', + run: () => import('./sendto/commands').then((module) => void module.syncCopies()), + }, + { + id: 'sendto.autoOn', + title: 'accounts.sendTo.commands.autoOn', + keywords: 'accounts.sendTo.commands.keywords', + flag: 'accounts.cloudSync', + run: () => import('./sendto/auto').then((module) => module.setAutoSync(true)), + }, + { + id: 'sendto.autoOff', + title: 'accounts.sendTo.commands.autoOff', + keywords: 'accounts.sendTo.commands.keywords', + flag: 'accounts.cloudSync', + run: () => import('./sendto/auto').then((module) => module.setAutoSync(false)), + }, + { + id: 'googleOffice.import', + title: 'accounts.googleOffice.commands.import', + keywords: 'accounts.googleOffice.commands.keywords', + flag: 'accounts.googleOffice', + category: 'notebooks', + run: () => import('./googleOffice/commands').then((module) => module.importFromGoogle()), + }, + { + id: 'googleOffice.docs', + title: 'accounts.googleOffice.commands.docs', + keywords: 'accounts.googleOffice.commands.keywords', + flag: 'accounts.googleOffice', + needsPage: true, + run: () => import('./googleOffice/commands').then((module) => module.exportToGoogle('gdoc')), + }, + { + id: 'googleOffice.sheets', + title: 'accounts.googleOffice.commands.sheets', + keywords: 'accounts.googleOffice.commands.keywords', + flag: 'accounts.googleOffice', + needsPage: true, + run: () => import('./googleOffice/commands').then((module) => module.exportToGoogle('gsheet')), + }, + { + id: 'googleOffice.slides', + title: 'accounts.googleOffice.commands.slides', + keywords: 'accounts.googleOffice.commands.keywords', + flag: 'accounts.googleOffice', + needsPage: true, + run: () => import('./googleOffice/commands').then((module) => module.exportToGoogle('gslides')), + }, + { + id: 'onenote.import', + title: 'accounts.onenote.commands.import', + keywords: 'accounts.onenote.commands.keywords', + flag: 'accounts.onenoteGraph', + category: 'notebooks', + run: () => import('./onenote/commands').then((module) => module.importOneNote()), + }, + { + id: 'tasks.microsoft', + title: 'accounts.tasks.commands.microsoft', + keywords: 'accounts.tasks.commands.keywords', + flag: 'accounts.tasksSync', + category: 'general', + run: () => import('./tasks/commands').then((module) => module.syncNow('microsoft')), + }, + { + id: 'tasks.google', + title: 'accounts.tasks.commands.google', + keywords: 'accounts.tasks.commands.keywords', + flag: 'accounts.tasksSync', + category: 'general', + run: () => import('./tasks/commands').then((module) => module.syncNow('google')), + }, + { + id: 'tasks.log', + title: 'accounts.tasks.commands.log', + keywords: 'accounts.tasks.commands.keywords', + flag: 'accounts.tasksSync', + category: 'general', + run: () => import('./tasks/commands').then((module) => module.showConflicts()), + }, + { + id: 'penSync.sync', + title: 'accounts.penSync.commands.sync', + keywords: 'accounts.penSync.commands.keywords', + flag: 'accounts.penSync', + category: 'general', + run: () => import('./penSync/commands').then((module) => module.syncNow()), + }, + { + id: 'penSync.off', + title: 'accounts.penSync.commands.off', + keywords: 'accounts.penSync.commands.keywords', + flag: 'accounts.penSync', + category: 'general', + run: () => import('./penSync/commands').then((module) => module.turnOff()), + }, + { + id: 'readwise.sync', + title: 'accounts.readwise.commands.sync', + keywords: 'accounts.readwise.commands.keywords', + flag: 'accounts.readwise', + category: 'general', + run: () => import('./readwise/commands').then((module) => module.sync()), + }, +]; + +for (const spec of SPECS) { + const def: CommandDef = { + id: spec.id as CommandDef['id'], + title: spec.title, + category: spec.category ?? 'insert', + keywords: spec.keywords, + flag: spec.flag, + ...(spec.needsPage ? { when: pageShown } : {}), + run: spec.run, + }; + commands.register(defineCommand(def)); +} + +// A page that was sent and has changed since offers to update its copy when it is opened. The check reads one small +// file and costs nothing until a page is shown. +shownMounted.subscribe(() => { + const mounted = shownMounted.get(); + if (!mounted || !isEnabled('accounts.sendTo')) return; + const page = mounted.page.id; + void import('./sendto/commands').then(async (module) => { + const node = await commandContext('palette').notes.get(page as NodeId); + if (node) await module.offerUpdate(page, node.modified); + }); +}); + +// After a task list has been synced by hand, it is kept in step quietly while the app is open. The timer starts a minute +// after start-up and only when the flag is on. +if (typeof window !== 'undefined' && isEnabled('accounts.tasksSync')) { + void import('./tasks/auto').then((module) => module.startAutoSync()); +} +if (typeof window !== 'undefined' && isEnabled('accounts.cloudSync')) { + void import('./sendto/auto').then((module) => module.startCopySync()); +} +if (typeof window !== 'undefined' && isEnabled('accounts.penSync')) { + void import('./penSync/auto').then((module) => module.startAutoSync()); +} diff --git a/app/src/features/accounts/run.ts b/app/src/features/accounts/run.ts new file mode 100644 index 00000000..b2021df7 --- /dev/null +++ b/app/src/features/accounts/run.ts @@ -0,0 +1,85 @@ +// How an account command starts and how it ends. It starts by checking that the account is connected, and says so, +// with a way to Settings, when it isn't. It ends by saying what went wrong in plain words: the connector's own +// sentences for a refusal (offline, expired, missing access) and the service's words for a failed request. It never +// shows a stack, an address, or a token. + +import { navigate } from '../../app/location'; +import { t } from '../../strings/t'; +import { announce, showToast } from '../../ui'; +import { ConnectorError, ConnectorHttpError, errorText, refreshConnectors } from '../connectors'; +import { connectorsStore } from '../connectors'; + +/** The words a failure is shown with. */ +export function failureText(error: unknown, service: string): string { + if (error instanceof ConnectorError) { + if (error.code === 'notConnected') return t('accounts.common.notConnected', { service }); + if (error.code === 'missingAccess') return t('accounts.common.needsAccess', { service }); + if (error.code === 'offline') return t('accounts.common.offline', { service }); + return errorText(error.code, service); + } + if (error instanceof ConnectorHttpError) + return t('accounts.common.failed', { service, message: serviceWords(error) }); + return t('accounts.common.failed', { service, message: error instanceof Error ? error.message : '' }).trim(); +} + +/** The short reason in a service's error body, or the status when the body has none. */ +export function serviceWords(error: ConnectorHttpError): string { + try { + const body = JSON.parse(error.body) as Record; + const nested = body.error; + const message = + typeof nested === 'string' + ? nested + : typeof (nested as { message?: unknown } | undefined)?.message === 'string' + ? (nested as { message: string }).message + : typeof body.message === 'string' + ? body.message + : ''; + if (message) return message.slice(0, 200); + } catch { + // Not JSON: the status says enough. + } + return t('accounts.common.status', { status: error.status }); +} + +/** Whether the connector is connected, reading the list first when it has not been read. */ +export async function isConnected(connector: string): Promise { + if (!connectorsStore.get().loaded) await refreshConnectors(); + return connectorsStore.get().items.some((item) => item.id === connector && item.state.kind === 'connected'); +} + +/** Says an account isn't connected and offers the way to Settings. */ +export function tellNotConnected(service: string): void { + showToast({ + message: t('accounts.common.notConnected', { service }), + action: { + label: t('accounts.common.openSettings'), + run: () => navigate({ view: 'settings', section: 'connectors' }), + }, + }); +} + +/** + * Runs the work for a command that needs the account. Resolves true when the work finished. Anything it throws is + * said in words and resolves false. + */ +export async function withAccount(connector: string, service: string, work: () => Promise): Promise { + if (!(await isConnected(connector))) { + tellNotConnected(service); + return false; + } + return attempt(service, work); +} + +/** Runs the work and says what went wrong, without checking the connection first. */ +export async function attempt(service: string, work: () => Promise): Promise { + try { + await work(); + return true; + } catch (error) { + const message = failureText(error, service); + showToast({ message, tone: 'danger' }); + announce(message); + return false; + } +} diff --git a/app/src/features/accounts/sendto/auto.test.ts b/app/src/features/accounts/sendto/auto.test.ts new file mode 100644 index 00000000..b647fe99 --- /dev/null +++ b/app/src/features/accounts/sendto/auto.test.ts @@ -0,0 +1,33 @@ +// Keeping Dropbox, Box, and WebDAV copies in step: which copies are sent again, and the on and off setting. +import { describe, expect, it } from 'vitest'; +import { createMemoryNotesService } from '../../../services/notes/memory'; +import { copiesToKeep, latestChange } from './auto'; +import type { SentCopy } from './types'; + +const base = { remote: 'r', name: 'n', link: '', folder: '', format: 'pdf' } as const; + +describe('keeping copies in step', () => { + it('sends a page copy again after the page changed, and a section copy when any page in it changed', async () => { + const notes = createMemoryNotesService({ seed: 'empty' }); + const book = await notes.create({ kind: 'notebook', placement: { parentId: null, beforeId: null }, title: 'B' }); + const section = await notes.create({ + kind: 'section', + placement: { parentId: book.id, beforeId: null }, + title: 'S', + }); + const page = await notes.create({ kind: 'page', placement: { parentId: section.id, beforeId: null }, title: 'P' }); + expect(await latestChange(notes, section.id)).not.toBeNull(); + const old = '2000-01-01T00:00:00Z'; + const future = '2999-01-01T00:00:00Z'; + const copies: SentCopy[] = [ + { ...base, service: 'dropbox', node: page.id, scope: 'page', sentAt: old }, + { ...base, service: 'box', node: page.id, scope: 'page', sentAt: future }, + { ...base, service: 'webdav', node: section.id, scope: 'section', sentAt: old }, + { ...base, service: 'webdav', node: book.id, scope: 'notebook', sentAt: future }, + { ...base, service: 'drive', node: page.id, scope: 'page', sentAt: old }, + { ...base, service: 'dropbox', node: page.id, scope: 'page', sentAt: old, format: 'png' }, + ]; + const due = await copiesToKeep(notes, copies); + expect(due.map((one) => `${one.service}:${one.scope}`)).toEqual(['dropbox:page', 'webdav:section']); + }); +}); diff --git a/app/src/features/accounts/sendto/auto.ts b/app/src/features/accounts/sendto/auto.ts new file mode 100644 index 00000000..1ddb7650 --- /dev/null +++ b/app/src/features/accounts/sendto/auto.ts @@ -0,0 +1,99 @@ +// Keeping Dropbox, Box, and WebDAV copies in step on its own. Off until the person turns it on (Ctrl+K, "Keep Dropbox, +// Box, and WebDAV copies up to date"). Then, while the app is open, a few minutes after start and then every quarter +// hour, each copy whose page changed since it was sent is sent again over the copy, with no message. A section's or +// notebook's copy is sent again when any page inside it changed. A failure is ignored and tried again later. +import { isEnabled } from '../../../app/flags'; +import { commandContext } from '../../../commands/registry'; +import type { NodeId, NotesService } from '../../../services/notes/types'; +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { loadState, saveState } from '../state'; +import { changedSince, loadCopies } from './copies'; +import type { SendService, SentCopy } from './types'; + +const NAME = 'sendto-auto'; +const EVERY_MS = 15 * 60_000; +const FIRST_MS = 3 * 60_000; +const MAX_NODES = 2000; + +export const KEPT: readonly SendService[] = ['dropbox', 'box', 'webdav']; + +export async function autoSyncOn(): Promise { + return (await loadState<{ on: boolean }>(NAME, { on: false })).on === true; +} + +export async function setAutoSync(on: boolean): Promise { + await saveState(NAME, { on }); + const message = t(on ? 'accounts.sendTo.auto.on' : 'accounts.sendTo.auto.off'); + showToast({ message }); + announce(message); +} + +/** The newest change to a page under the node (or the page itself). Null when it can't be told. */ +export async function latestChange(notes: NotesService, node: NodeId): Promise { + let newest: string | null = null; + let seen = 0; + const queue: NodeId[] = [node]; + while (queue.length > 0 && seen < MAX_NODES) { + const id = queue.shift() as NodeId; + const summary = await notes.get(id); + if (!summary) continue; + seen += 1; + if (summary.kind === 'page' && (newest === null || summary.modified > newest)) newest = summary.modified; + if (summary.kind !== 'page') { + for (const child of await notes.listChildren(id)) queue.push(child.id); + } + } + return newest; +} + +/** The copies to send again now. */ +export async function copiesToKeep(notes: NotesService, copies: readonly SentCopy[]): Promise { + const due: SentCopy[] = []; + for (const copy of copies) { + if (!KEPT.includes(copy.service) || copy.format === 'png') continue; + const modified = copy.scope === 'page' ? ((await notes.get(copy.node as NodeId))?.modified ?? null) : null; + const changed = copy.scope === 'page' ? modified : await latestChange(notes, copy.node as NodeId); + if (changed !== null && changedSince(copy, changed)) due.push(copy); + } + return due; +} + +let running = false; + +/** One quiet pass. Answers how many copies were sent again. */ +export async function keepInStep(): Promise { + if (running) return 0; + running = true; + try { + if (!(await autoSyncOn())) return 0; + const due = await copiesToKeep(commandContext('palette').notes, await loadCopies()); + if (due.length === 0) return 0; + const { updateCopy } = await import('./commands'); + let sent = 0; + for (const copy of due) { + try { + await updateCopy(copy, true); + sent += 1; + } catch { + // Not now. The next pass tries again. + } + } + return sent; + } catch { + return 0; + } finally { + running = false; + } +} + +let started = false; + +export function startCopySync(): void { + if (started || typeof window === 'undefined' || !isEnabled('accounts.cloudSync')) return; + started = true; + window.setTimeout(() => { + void keepInStep(); + window.setInterval(() => void keepInStep(), EVERY_MS); + }, FIRST_MS); +} diff --git a/app/src/features/accounts/sendto/box.ts b/app/src/features/accounts/sendto/box.ts new file mode 100644 index 00000000..6ca6a659 --- /dev/null +++ b/app/src/features/accounts/sendto/box.ts @@ -0,0 +1,96 @@ +// Box: the folders to choose from, and a file sent as one multipart request (the file's name and folder, then its bytes), +// new, or over the file sent before. Box names a folder or file by an ID, and the top folder is "0". A name that is +// taken in a folder is refused by Box with a conflict, and the file is then sent again under a new name. +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import type { Destination, LocalFile, RemoteFolder, Sent } from './types'; + +const API = 'https://api.box.com/2.0'; +const UPLOAD = 'https://upload.box.com/api/2.0/files'; +const MAX_PAGES = 30; +const LIMIT = 100; + +interface Items { + entries?: { id: string; name: string; type: string }[]; + total_count?: number; +} + +interface Uploaded { + entries?: { id: string; name: string }[]; +} + +export function boxDestination(deps: { client: ConnectorsClient; host: AccountsHost }): Destination { + const { client, host } = deps; + + async function multipart(url: string, attributes: Record, local: LocalFile) { + const boundary = `opennote${Math.random().toString(36).slice(2, 12)}`; + return host.upload( + 'box', + ['boxFiles'], + { method: 'POST', url }, + [ + { + kind: 'text', + text: + `--${boundary}\r\nContent-Disposition: form-data; name="attributes"\r\n\r\n${JSON.stringify(attributes)}\r\n` + + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${local.name.replace(/"/g, '')}"\r\n` + + `Content-Type: ${local.mime}\r\n\r\n`, + }, + { kind: 'file', file: local.file }, + { kind: 'text', text: `\r\n--${boundary}--` }, + ], + `multipart/form-data; boundary=${boundary}`, + ); + } + + async function finish(answer: { status: number; body: string }, local: LocalFile): Promise { + if (answer.status < 200 || answer.status >= 300) throw new ConnectorHttpError(answer.status, answer.body); + const entry = (JSON.parse(answer.body || '{}') as Uploaded).entries?.[0]; + if (!entry) throw new ConnectorHttpError(502, answer.body); + return { id: entry.id, name: entry.name || local.name, link: `https://app.box.com/file/${entry.id}` }; + } + + return { + service: 'box', + async folders(parent) { + const found: RemoteFolder[] = []; + for (let page = 0; page < MAX_PAGES; page += 1) { + const reply = await requestConnector( + 'box', + ['boxFiles'], + { + url: `${API}/folders/${encodeURIComponent(parent ?? '0')}/items`, + query: { fields: 'id,name,type', limit: LIMIT, offset: page * LIMIT }, + }, + client, + ); + const entries = reply.data?.entries ?? []; + for (const entry of entries) if (entry.type === 'folder') found.push({ id: entry.id, name: entry.name }); + if (entries.length < LIMIT) break; + } + return found.sort((a, b) => a.name.localeCompare(b.name)); + }, + async send(local, folder) { + const parent = { id: folder ?? '0' }; + const first = await multipart(`${UPLOAD}/content`, { name: local.name, parent }, local); + if (first.status !== 409) return finish(first, local); + // The name is taken: Box says so, and the file goes again with a number after its name. + const dot = local.name.lastIndexOf('.'); + const stem = dot > 0 ? local.name.slice(0, dot) : local.name; + const extension = dot > 0 ? local.name.slice(dot) : ''; + for (let copy = 1; copy <= 9; copy += 1) { + const name = `${stem} (${copy})${extension}`; + const again = await multipart(`${UPLOAD}/content`, { name, parent }, { ...local, name }); + if (again.status !== 409) return finish(again, { ...local, name }); + } + throw new ConnectorHttpError(409, JSON.stringify({ message: 'The name is taken.' })); + }, + async replace(local, remote) { + return finish( + await multipart(`${UPLOAD}/${encodeURIComponent(remote)}/content`, { name: local.name }, local), + local, + ); + }, + }; +} diff --git a/app/src/features/accounts/sendto/commands.ts b/app/src/features/accounts/sendto/commands.ts new file mode 100644 index 00000000..c950719e --- /dev/null +++ b/app/src/features/accounts/sendto/commands.ts @@ -0,0 +1,327 @@ +// Send to Google Drive and Send to OneDrive, and Update the copy. The person chooses what to send (the lasso's selection, +// the page, its section, or its notebook), the file type, and the folder, and nothing is sent until the last button. A +// page, section, or notebook that was sent stays linked to its file, so after the notes change OpenNote can offer to +// update the copy. It offers once for each page in a session and never again until the page is opened next time. +import { getLocation } from '../../../app/location'; +import { commandContext } from '../../../commands/registry'; +import type { NodeId } from '../../../services/notes/types'; +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { connectors } from '../../connectors'; +import { pageSelection, shownMounted } from '../../page'; +import { accountsHost } from '../host'; +import type { AccountsHost } from '../host'; +import { attempt, isConnected, tellNotConnected } from '../run'; +import { pickOne } from '../ui/prompts'; +import type { Choice } from '../ui/prompts'; +import { changedSince, copiesDue, copiesOf, copyOf, loadCopies, rememberCopy } from './copies'; +import { boxDestination } from './box'; +import { driveDestination } from './drive'; +import { dropboxDestination } from './dropbox'; +import { onedriveDestination } from './onedrive'; +import { webdavDestination } from './webdav'; +import { prepare } from './render'; +import { OFFICE, officeOf } from './types'; +import type { Destination, LocalFile, RemoteFolder, SendFormat, SendScope, SendService, SentCopy } from './types'; + +const NAMES = { + drive: 'Google Drive', + onedrive: 'OneDrive', + dropbox: 'Dropbox', + box: 'Box', + webdav: 'WebDAV', +} as const; +const CONNECTOR = { drive: 'google', onedrive: 'microsoft', dropbox: 'dropbox', box: 'box', webdav: 'webdav' } as const; + +export function destinationFor(service: SendService, host: AccountsHost = accountsHost()): Destination { + const deps = { client: connectors(), host }; + switch (service) { + case 'drive': + return driveDestination(deps); + case 'onedrive': + return onedriveDestination(deps); + case 'dropbox': + return dropboxDestination(deps); + case 'box': + return boxDestination(deps); + case 'webdav': + return webdavDestination(deps); + } +} + +/** The types of file a scope can be sent as, for a service. */ +export function formatsFor(service: SendService, scope: SendScope): SendFormat[] { + const converted: SendFormat[] = service === 'drive' ? ['gdoc', 'gsheet', 'gslides'] : []; + // A selection is a picture, so it can stay a picture or go in a Word file; the rest can be any Office file. + if (scope === 'selection') return ['png', 'docx', ...converted.slice(0, 1)]; + const office: SendFormat[] = ['docx', 'xlsx', 'pptx', ...converted]; + return scope === 'page' ? ['pdf', ...office] : office; +} + +/** Sends a prepared file, new or over an earlier copy, and links the node to it. Resolves the record. */ +export async function sendFile( + destination: Destination, + local: LocalFile, + options: { + node: string; + scope: SendScope; + format: SendFormat; + folder: string | null; + folderName: string; + replace?: string; + host?: AccountsHost; + now?: Date; + }, +): Promise { + const office = officeOf(options.format); + const convertTo = office?.convert ? OFFICE[office.type].convertTo : null; + const sent = options.replace + ? await destination.replace(local, options.replace) + : await destination.send(local, options.folder, convertTo); + const copy = copyOf( + { + service: destination.service, + node: options.node, + scope: options.scope === 'selection' ? 'page' : options.scope, + format: options.format, + folder: options.folderName, + }, + sent, + options.now, + ); + // A selection can't be made again later, so it isn't linked. + if (options.scope !== 'selection') await rememberCopy(copy, options.host); + return copy; +} + +/** Walks the folders of the service: use this one, open one inside it, or go back up. */ +async function chooseFolder(destination: Destination): Promise<{ id: string | null; name: string } | null> { + type Step = { kind: 'use' } | { kind: 'open'; folder: RemoteFolder } | { kind: 'up' }; + const trail: RemoteFolder[] = []; + const top = t('accounts.sendTo.folder.top'); + for (;;) { + const here = trail[trail.length - 1] ?? null; + const inside = await destination.folders(here?.id ?? null); + const choices: Choice[] = [ + { id: 'use', label: t('accounts.sendTo.folder.use', { name: here?.name ?? top }), value: { kind: 'use' } }, + ...inside.map((folder) => ({ + id: folder.id, + label: folder.name, + detail: t('accounts.sendTo.folder.open'), + value: { kind: 'open', folder } as Step, + })), + ...(here ? [{ id: 'up', label: t('accounts.sendTo.folder.up'), value: { kind: 'up' } as Step }] : []), + ]; + const step = await pickOne({ + title: t('accounts.sendTo.folder.title'), + description: t('accounts.sendTo.folder.description', { name: here?.name ?? top }), + confirmLabel: t('accounts.sendTo.folder.confirm'), + choices, + }); + if (!step) return null; + if (step.kind === 'use') return { id: here?.id ?? null, name: here?.name ?? top }; + if (step.kind === 'open') trail.push(step.folder); + else trail.pop(); + } +} + +interface Where { + page: NodeId | null; + section: NodeId | null; + notebook: NodeId | null; +} + +function whereNow(): Where { + const here = getLocation(); + return here.view === 'workspace' + ? { page: here.pageId, section: here.sectionId, notebook: here.notebookId } + : { page: null, section: null, notebook: null }; +} + +async function titleOf(node: NodeId | null, fallback: string): Promise { + const found = node ? await commandContext('palette').notes.get(node) : null; + return found?.title.trim() || fallback; +} + +function chosenOnPage(): boolean { + const { blocks, strokes } = pageSelection.get(); + return blocks.length + strokes.length > 0; +} + +export async function sendTo(service: SendService, preset?: { format: SendFormat }): Promise { + const name = NAMES[service]; + if (!shownMounted.get()) return void showToast({ message: t('accounts.sendTo.nothing') }); + if (!(await isConnected(CONNECTOR[service]))) return tellNotConnected(name); + const where = whereNow(); + const scopes: Choice[] = []; + if (chosenOnPage()) { + scopes.push({ id: 'selection', label: t('accounts.sendTo.scope.selection'), value: 'selection' }); + } + if (where.page) scopes.push({ id: 'page', label: t('accounts.sendTo.scope.page'), value: 'page' }); + if (where.section) scopes.push({ id: 'section', label: t('accounts.sendTo.scope.section'), value: 'section' }); + if (where.notebook) scopes.push({ id: 'notebook', label: t('accounts.sendTo.scope.notebook'), value: 'notebook' }); + const scope = await pickOne({ + title: t('accounts.sendTo.scope.title', { service: name }), + description: t('accounts.sendTo.scope.description'), + confirmLabel: t('accounts.sendTo.scope.confirm'), + choices: scopes, + empty: t('accounts.sendTo.nothing'), + }); + if (!scope) return; + const labels: Record = { + pdf: t('accounts.sendTo.format.pdf'), + docx: t('accounts.sendTo.format.docx'), + xlsx: t('accounts.sendTo.format.xlsx'), + pptx: t('accounts.sendTo.format.pptx'), + gdoc: t('accounts.sendTo.format.gdoc'), + gsheet: t('accounts.sendTo.format.gsheet'), + gslides: t('accounts.sendTo.format.gslides'), + png: t('accounts.sendTo.format.png'), + }; + const offered = formatsFor(service, scope); + const format = + preset && offered.includes(preset.format) + ? preset.format + : await pickOne({ + title: t('accounts.sendTo.format.title'), + confirmLabel: t('accounts.sendTo.format.confirm'), + choices: offered.map((one) => ({ id: one, label: labels[one], value: one })), + }); + if (!format) return; + await attempt(name, async () => { + const destination = destinationFor(service); + const folder = await chooseFolder(destination); + if (!folder) return; + const node = scope === 'selection' ? where.page : where[scope]; + const title = await titleOf(node, t('accounts.sendTo.untitled')); + const host = accountsHost(); + announce(t('accounts.sendTo.working', { service: name })); + const local = await prepare(host, scope, format, node, title); + if (!local || !node) return void showToast({ message: t('accounts.sendTo.empty'), tone: 'danger' }); + try { + const copy = await sendFile(destination, local, { + node, + scope, + format, + folder: folder.id, + folderName: folder.name, + }); + const message = t('accounts.sendTo.done', { name: copy.name, service: name, folder: folder.name }); + showToast({ message }); + announce(message); + } finally { + await host.tempRemove(local.file).catch(() => undefined); + } + }); +} + +/** The copies of the page that is open, or of the section and notebook it sits in. */ +async function copiesHere(): Promise { + const where = whereNow(); + const nodes = [where.page, where.section, where.notebook].filter((one): one is NodeId => one !== null); + return copiesOf(await loadCopies(), nodes).filter((copy) => copy.format !== 'png'); +} + +/** Sends the node again over its copy. */ +export async function updateCopy(copy: SentCopy, quiet = false): Promise { + const name = NAMES[copy.service]; + if (!(await isConnected(CONNECTOR[copy.service]))) return quiet ? undefined : tellNotConnected(name); + const work = async () => { + const host = accountsHost(); + const title = await titleOf(copy.node as NodeId, copy.name); + if (!quiet) announce(t('accounts.sendTo.updating', { name: copy.name })); + const local = await prepare(host, copy.scope, copy.format, copy.node as NodeId, title); + if (!local) return void (quiet || showToast({ message: t('accounts.sendTo.empty'), tone: 'danger' })); + try { + const updated = await sendFile(destinationFor(copy.service, host), local, { + node: copy.node, + scope: copy.scope, + format: copy.format, + folder: null, + folderName: copy.folder, + replace: copy.remote, + }); + if (!quiet) { + const message = t('accounts.sendTo.updated', { name: updated.name, service: name }); + showToast({ message }); + announce(message); + } + } finally { + await host.tempRemove(local.file).catch(() => undefined); + } + }; + if (quiet) await work(); + else await attempt(name, work); +} + +/** Update the copy: the page's copy, or a choice when it, its section, and its notebook have several. */ +export async function updateHere(): Promise { + const copies = await copiesHere(); + if (copies.length === 0) return void showToast({ message: t('accounts.sendTo.noCopy') }); + const copy = + copies.length === 1 + ? copies[0] + : await pickOne({ + title: t('accounts.sendTo.update.title'), + confirmLabel: t('accounts.sendTo.update.confirm'), + choices: copies.map((one) => ({ + id: `${one.service}:${one.remote}`, + label: one.name, + detail: t('accounts.sendTo.update.detail', { service: NAMES[one.service], folder: one.folder }), + value: one, + })), + }); + if (copy) await updateCopy(copy); +} + +const offered = new Set(); + +/** + * After a page that was sent is opened and has changed since, offers to update the copy. Quiet: a toast with one button, + * once for a page until the app restarts. Nothing is sent unless the person presses it. + */ +export async function offerUpdate(page: string, modified: string): Promise { + if (offered.has(page)) return; + const copy = (await loadCopies()).find( + (one) => one.node === page && one.scope === 'page' && one.format !== 'png' && changedSince(one, modified), + ); + if (!copy) return; + offered.add(page); + showToast({ + message: t('accounts.sendTo.offer', { service: NAMES[copy.service] }), + action: { + label: t('accounts.sendTo.offerAction', { service: NAMES[copy.service] }), + run: () => void updateCopy(copy), + }, + }); +} + +export const resetOffers = (): void => offered.clear(); + +const SYNCED: readonly SendService[] = ['dropbox', 'box', 'webdav']; + +/** + * Brings every Dropbox, Box, and WebDAV copy up to date in one go: a page's copy when the page changed since it was + * sent; a section's or notebook's copy always (their own times don't follow their pages). Each is sent over its copy, + * so nothing new appears in the folder. Quiet when nothing needs it. + */ +export async function syncCopies(): Promise { + const notes = commandContext('palette').notes; + const copies = await loadCopies(); + const times = new Map(); + for (const copy of copies) { + if (copy.scope !== 'page' || times.has(copy.node)) continue; + const node = await notes.get(copy.node as NodeId); + if (node) times.set(copy.node, node.modified); + } + const due = copiesDue(copies, SYNCED, (node) => times.get(node) ?? null); + if (due.length === 0) { + showToast({ message: t('accounts.sendTo.sync.nothing') }); + return 0; + } + for (const copy of due) await updateCopy(copy); + const message = t('accounts.sendTo.sync.done', { count: due.length }); + showToast({ message }); + announce(message); + return due.length; +} diff --git a/app/src/features/accounts/sendto/copies.ts b/app/src/features/accounts/sendto/copies.ts new file mode 100644 index 00000000..6f7dc9e2 --- /dev/null +++ b/app/src/features/accounts/sendto/copies.ts @@ -0,0 +1,68 @@ +// The copies a page, section, or notebook was sent as, kept by the host in this device's folder so a page stays linked to +// its Drive or OneDrive file after OpenNote restarts. A copy holds where the file is and when it was sent, never a token. +import type { AccountsHost } from '../host'; +import { loadState, saveState } from '../state'; +import type { SendService, Sent, SentCopy } from './types'; + +const NAME = 'sendto-copies'; + +interface Saved { + copies: SentCopy[]; +} + +export async function loadCopies(host?: AccountsHost): Promise { + const saved = await loadState(NAME, { copies: [] }, host); + return Array.isArray(saved.copies) ? saved.copies.filter((one) => typeof one?.remote === 'string') : []; +} + +/** Links the node to the file. A node sent again to the same service in the same way keeps one link, the newest. */ +export async function rememberCopy(copy: SentCopy, host?: AccountsHost): Promise { + const kept = (await loadCopies(host)).filter( + (one) => !(one.service === copy.service && one.node === copy.node && one.scope === copy.scope), + ); + const all = [...kept, copy]; + await saveState(NAME, { copies: all } satisfies Saved, host); + return all; +} + +export async function forgetCopy(service: SendService, remote: string, host?: AccountsHost): Promise { + const all = (await loadCopies(host)).filter((one) => !(one.service === service && one.remote === remote)); + await saveState(NAME, { copies: all } satisfies Saved, host); +} + +/** The copies of any of these nodes (a page and the section and notebook it sits in). */ +export const copiesOf = (copies: readonly SentCopy[], nodes: readonly string[]): SentCopy[] => + copies.filter((one) => nodes.includes(one.node)); + +/** Whether the node changed after the copy was sent. */ +export function changedSince(copy: Pick, modified: string): boolean { + const sent = Date.parse(copy.sentAt); + const changed = Date.parse(modified); + return Number.isFinite(sent) && Number.isFinite(changed) && changed > sent; +} + +/** The record for a file that was just sent. */ +export function copyOf( + base: Pick, + sent: Sent, + now = new Date(), +): SentCopy { + return { ...base, remote: sent.id, name: sent.name, link: sent.link, sentAt: now.toISOString() }; +} + +/** + * The copies a sync must send again: a page's copy when the page changed after it was sent; a section's or notebook's + * always (their own times don't follow their pages). A picture of a selection can't be made again, so it is left out. + */ +export function copiesDue( + copies: readonly SentCopy[], + services: readonly SendService[], + modifiedOf: (node: string) => string | null, +): SentCopy[] { + return copies.filter((copy) => { + if (!services.includes(copy.service) || copy.format === 'png') return false; + if (copy.scope !== 'page') return true; + const modified = modifiedOf(copy.node); + return modified !== null && changedSince(copy, modified); + }); +} diff --git a/app/src/features/accounts/sendto/drive.ts b/app/src/features/accounts/sendto/drive.ts new file mode 100644 index 00000000..19332f9d --- /dev/null +++ b/app/src/features/accounts/sendto/drive.ts @@ -0,0 +1,108 @@ +// Google Drive: the folders to choose from (read with the Drive read access), and a file sent in one multipart request +// (the metadata and the bytes together), new or over a copy sent before. A Word file can be turned into a Google Doc +// as it arrives. The app only changes files it made itself (the `drive.file` access). +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import type { Destination, LocalFile, RemoteFolder, Sent } from './types'; + +const FILES = 'https://www.googleapis.com/drive/v3/files'; +const UPLOAD = 'https://www.googleapis.com/upload/drive/v3/files'; +const FOLDER = 'application/vnd.google-apps.folder'; +const MAX_PAGES = 20; + +interface FileList { + files?: { id: string; name: string }[]; + nextPageToken?: string; +} + +interface DriveFile { + id?: string; + name?: string; + webViewLink?: string; +} + +/** An ID for Drive's search language: the quote and the backslash are the only characters that need care. */ +const quoted = (id: string): string => `'${id.replace(/[\\']/g, '')}'`; + +export function driveDestination(deps: { client: ConnectorsClient; host: AccountsHost }): Destination { + const { client, host } = deps; + + async function multipart( + method: 'POST' | 'PATCH', + url: string, + metadata: Record, + local: LocalFile, + ): Promise { + const boundary = `opennote${Math.random().toString(36).slice(2, 12)}`; + const answer = await host.upload( + 'google', + ['driveFiles'], + { method, url }, + [ + { + kind: 'text', + text: + `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${JSON.stringify(metadata)}\r\n` + + `--${boundary}\r\nContent-Type: ${local.mime}\r\n\r\n`, + }, + { kind: 'file', file: local.file }, + { kind: 'text', text: `\r\n--${boundary}--` }, + ], + `multipart/related; boundary=${boundary}`, + ); + if (answer.status < 200 || answer.status >= 300) throw new ConnectorHttpError(answer.status, answer.body); + const file = JSON.parse(answer.body || '{}') as DriveFile; + if (!file.id) throw new ConnectorHttpError(502, answer.body); + return { id: file.id, name: file.name ?? local.name, link: file.webViewLink ?? '' }; + } + + return { + service: 'drive', + async folders(parent) { + const found: RemoteFolder[] = []; + let token: string | undefined; + for (let page = 0; page < MAX_PAGES; page += 1) { + const reply = await requestConnector( + 'google', + ['driveRead'], + { + url: FILES, + query: { + q: `${quoted(parent ?? 'root')} in parents and mimeType='${FOLDER}' and trashed=false`, + fields: 'files(id,name),nextPageToken', + pageSize: 100, + orderBy: 'name', + pageToken: token, + }, + }, + client, + ); + for (const one of reply.data?.files ?? []) found.push({ id: one.id, name: one.name }); + token = reply.data?.nextPageToken; + if (!token) break; + } + return found; + }, + send(local, folder, convertTo) { + return multipart( + 'POST', + `${UPLOAD}?uploadType=multipart&fields=id,name,webViewLink`, + { + name: convertTo ? local.name.replace(/\.(docx|xlsx|pptx)$/i, '') : local.name, + ...(folder ? { parents: [folder] } : {}), + ...(convertTo ? { mimeType: convertTo } : {}), + }, + local, + ); + }, + replace(local, remote) { + return multipart( + 'PATCH', + `${UPLOAD}/${encodeURIComponent(remote)}?uploadType=multipart&fields=id,name,webViewLink`, + {}, + local, + ); + }, + }; +} diff --git a/app/src/features/accounts/sendto/dropbox.ts b/app/src/features/accounts/sendto/dropbox.ts new file mode 100644 index 00000000..f4e55ba5 --- /dev/null +++ b/app/src/features/accounts/sendto/dropbox.ts @@ -0,0 +1,85 @@ +// Dropbox: the folders to choose from, and a file sent with one upload request (up to 150 MB), new or over the file sent +// before. Dropbox names files by path, so the path is the copy's ID. A name that is taken is kept and the new file is +// renamed, so nothing is overwritten unless the person asked to update the copy. +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import type { Destination, LocalFile, RemoteFolder, Sent } from './types'; + +const API = 'https://api.dropboxapi.com/2/files'; +const UPLOAD = 'https://content.dropboxapi.com/2/files/upload'; +const MAX_PAGES = 30; + +interface Entry { + '.tag': string; + name: string; + path_lower?: string; + path_display?: string; +} + +interface Listing { + entries?: Entry[]; + cursor?: string; + has_more?: boolean; +} + +interface Uploaded { + name?: string; + path_display?: string; + path_lower?: string; +} + +export function dropboxDestination(deps: { client: ConnectorsClient; host: AccountsHost }): Destination { + const { client, host } = deps; + + async function upload(path: string, mode: 'add' | 'overwrite', local: LocalFile): Promise { + const answer = await host.upload( + 'dropbox', + ['dropboxWrite'], + { + method: 'POST', + url: UPLOAD, + headers: { 'Dropbox-API-Arg': JSON.stringify({ path, mode, autorename: mode === 'add', mute: true }) }, + }, + [{ kind: 'file', file: local.file }], + 'application/octet-stream', + ); + if (answer.status < 200 || answer.status >= 300) throw new ConnectorHttpError(answer.status, answer.body); + const done = JSON.parse(answer.body || '{}') as Uploaded; + const shown = done.path_display ?? path; + const folder = shown.slice(0, shown.lastIndexOf('/')); + return { + id: done.path_lower ?? path.toLowerCase(), + name: done.name ?? local.name, + link: `https://www.dropbox.com/home${encodeURI(folder)}`, + }; + } + + return { + service: 'dropbox', + async folders(parent) { + const found: RemoteFolder[] = []; + let page = await requestConnector( + 'dropbox', + ['dropboxRead'], + { url: `${API}/list_folder`, json: { path: parent ?? '', recursive: false, limit: 500 } }, + client, + ); + for (let more = 0; more < MAX_PAGES; more += 1) { + for (const entry of page.data?.entries ?? []) { + if (entry['.tag'] === 'folder') found.push({ id: entry.path_display ?? entry.name, name: entry.name }); + } + if (!page.data?.has_more || !page.data.cursor) break; + page = await requestConnector( + 'dropbox', + ['dropboxRead'], + { url: `${API}/list_folder/continue`, json: { cursor: page.data.cursor } }, + client, + ); + } + return found.sort((a, b) => a.name.localeCompare(b.name)); + }, + send: (local, folder) => upload(`${folder ?? ''}/${local.name}`, 'add', local), + replace: (local, remote) => upload(remote, 'overwrite', local), + }; +} diff --git a/app/src/features/accounts/sendto/onedrive.ts b/app/src/features/accounts/sendto/onedrive.ts new file mode 100644 index 00000000..3272eaea --- /dev/null +++ b/app/src/features/accounts/sendto/onedrive.ts @@ -0,0 +1,70 @@ +// OneDrive through Microsoft Graph: the folders to choose from, and a file sent with one PUT (Graph takes up to 250 MB +// that way). A name that is taken is kept and the new file is renamed, so nothing is overwritten. A copy sent before is +// replaced by its ID. +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import type { Destination, LocalFile, RemoteFolder, Sent } from './types'; + +const GRAPH = 'https://graph.microsoft.com/v1.0'; +const MAX_PAGES = 20; + +interface Children { + value?: { id: string; name: string; folder?: object }[]; + '@odata.nextLink'?: string; +} + +interface Item { + id?: string; + name?: string; + webUrl?: string; +} + +export function onedriveDestination(deps: { client: ConnectorsClient; host: AccountsHost }): Destination { + const { client, host } = deps; + + async function put(url: string, local: LocalFile): Promise { + const answer = await host.upload( + 'microsoft', + ['onedriveFiles'], + { method: 'PUT', url }, + [{ kind: 'file', file: local.file }], + local.mime, + ); + if (answer.status < 200 || answer.status >= 300) throw new ConnectorHttpError(answer.status, answer.body); + const item = JSON.parse(answer.body || '{}') as Item; + if (!item.id) throw new ConnectorHttpError(502, answer.body); + return { id: item.id, name: item.name ?? local.name, link: item.webUrl ?? '' }; + } + + return { + service: 'onedrive', + async folders(parent) { + const found: RemoteFolder[] = []; + let url: string | null = parent + ? `${GRAPH}/me/drive/items/${encodeURIComponent(parent)}/children` + : `${GRAPH}/me/drive/root/children`; + let query: Record | undefined = { $select: 'id,name,folder', $top: 200 }; + for (let page = 0; url && page < MAX_PAGES; page += 1) { + const reply: { data?: Children } = await requestConnector( + 'microsoft', + ['onedriveFiles'], + { url, ...(query ? { query } : {}) }, + client, + ); + for (const one of reply.data?.value ?? []) if (one.folder) found.push({ id: one.id, name: one.name }); + url = reply.data?.['@odata.nextLink'] ?? null; + query = undefined; + } + return found.sort((a, b) => a.name.localeCompare(b.name)); + }, + send(local, folder) { + const name = encodeURIComponent(local.name); + const where = folder ? `items/${encodeURIComponent(folder)}` : 'root'; + return put(`${GRAPH}/me/drive/${where}:/${name}:/content?@microsoft.graph.conflictBehavior=rename`, local); + }, + replace(local, remote) { + return put(`${GRAPH}/me/drive/items/${encodeURIComponent(remote)}/content`, local); + }, + }; +} diff --git a/app/src/features/accounts/sendto/render.ts b/app/src/features/accounts/sendto/render.ts new file mode 100644 index 00000000..2fbc62e1 --- /dev/null +++ b/app/src/features/accounts/sendto/render.ts @@ -0,0 +1,78 @@ +// Making the file that is sent, in the transfer folder: a PDF or a picture of the page, a Word file of the page, section, +// or notebook (the same export that Export makes), or a Word file of the lasso's selection. Nothing leaves the PC here. +import { commandContext } from '../../../commands/registry'; +import type { NodeId } from '../../../services/notes/types'; +import { t } from '../../../strings/t'; +import { collectRequest, pageCount, resolveTarget } from '../../interop'; +import { renderShownPage } from '../../pages'; +import type { AccountsHost } from '../host'; +import { bytesPart } from '../host'; +import { DOCX, OFFICE, officeOf } from './types'; +import type { LocalFile, SendFormat, SendScope } from './types'; + +/** A file name made from a title. */ +export function fileNameOf(title: string, extension: string): string { + const base = title + .replace(/[^\p{L}\p{N} _.-]+/gu, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 80) + .trim(); + return `${base || t('accounts.sendTo.untitled')}.${extension}`; +} + +/** The width and height in a PNG's header, or zeros. */ +function pngSize(bytes: Uint8Array): { width: number; height: number } { + if (bytes.length < 24) return { width: 0, height: 0 }; + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + return { width: view.getUint32(16), height: view.getUint32(20) }; +} + +async function stage(host: AccountsHost, name: string, bytes: Uint8Array, mime: string): Promise { + await host.tempWrite(name, bytesPart(bytes)); + return { file: name, name, mime }; +} + +/** + * The file for what is sent. `node` is the page, section, or notebook; null for the lasso's selection. Answers null when + * there is nothing to send (an empty section). + */ +export async function prepare( + host: AccountsHost, + scope: SendScope, + format: SendFormat, + node: NodeId | null, + title: string, +): Promise { + const context = commandContext('palette'); + const office = officeOf(format); + if (scope === 'selection') { + const picture = await renderShownPage(context, 'png', true); + if (!picture) return null; + if (format === 'png') return stage(host, picture.name, picture.bytes, picture.mime); + const word = await context.platform.exports.selectionDocx(picture.bytes, { + title, + alt: title, + ...pngSize(picture.bytes), + }); + return stage(host, fileNameOf(title, 'docx'), word, DOCX); + } + if (scope === 'page' && format === 'pdf') { + const pdf = await renderShownPage(context, 'pdf'); + return pdf ? stage(host, pdf.name, pdf.bytes, pdf.mime) : null; + } + if (!node || !office) return null; + const target = await resolveTarget(context.notes, node); + if (!target) return null; + const request = await collectRequest(context.notes, target, scope); + if (pageCount(request) === 0) return null; + const folder = await host.tempDir(); + const outcome = await context.platform.interop.exportTo(`send-${Date.now()}`, { + ...request, + format: office.type, + folder, + }); + if (outcome.status !== 'done') return null; + const path = outcome.result.reveal; + return { file: path, name: fileNameOf(title, office.type), mime: OFFICE[office.type].mime }; +} diff --git a/app/src/features/accounts/sendto/sendto.test.ts b/app/src/features/accounts/sendto/sendto.test.ts new file mode 100644 index 00000000..aeb33be8 --- /dev/null +++ b/app/src/features/accounts/sendto/sendto.test.ts @@ -0,0 +1,211 @@ +// Send to Google Drive and OneDrive against mock services: the folders (across pages), a file sent new, converted to a +// Google Doc, and sent again over its copy; the link kept between a page and its copy; and when the page counts as changed. +import { describe, expect, it } from 'vitest'; +import { addDrive, addOneDrive } from '../../../../../tests/mock-servers/storage'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { createFakeConnectors } from '../../connectors'; +import { createFakeAccountsHost } from '../fakeHost'; +import { formatsFor, sendFile } from './commands'; +import { changedSince, copiesDue, copiesOf, loadCopies } from './copies'; +import { driveDestination } from './drive'; +import { onedriveDestination } from './onedrive'; +import { DOCX } from './types'; +import type { LocalFile } from './types'; + +async function rig(connected: Record, server: MockServer) { + const connectors = createFakeConnectors({ connected, respond: server.respond }); + const host = createFakeAccountsHost(() => connectors.client); + await host.tempWrite('Cell notes.docx', { kind: 'text', text: 'WORD-FILE-1' }); + const local: LocalFile = { file: 'Cell notes.docx', name: 'Cell notes.docx', mime: DOCX }; + return { client: connectors.client, host, local }; +} + +describe('Google Drive', () => { + const folders = [ + { id: 'f2', name: 'Chemistry', parent: 'root' }, + { id: 'f1', name: 'Biology', parent: 'root' }, + { id: 'f3', name: 'Lab reports', parent: 'f1' }, + ]; + + it('lists the folders of a folder, across pages, by name', async () => { + const server = new MockServer(); + addDrive(server, folders, 1); + const { client, host } = await rig({ google: 'sam@example.com' }, server); + const drive = driveDestination({ client, host }); + expect((await drive.folders(null)).map((one) => one.name)).toEqual(['Biology', 'Chemistry']); + expect((await drive.folders('f1')).map((one) => one.name)).toEqual(['Lab reports']); + expect(server.requests).toHaveLength(3); + }); + + it('sends a file into a folder, links it, and sends over the same copy next time', async () => { + const server = new MockServer(); + const { files } = addDrive(server, folders); + const { client, host, local } = await rig({ google: 'sam@example.com' }, server); + const drive = driveDestination({ client, host }); + const copy = await sendFile(drive, local, { + node: 'page1', + scope: 'page', + format: 'docx', + folder: 'f3', + folderName: 'Lab reports', + host, + now: new Date('2026-10-01T10:00:00Z'), + }); + expect(copy).toMatchObject({ service: 'drive', node: 'page1', link: 'https://drive.example/file1' }); + expect(files.get('file1')).toMatchObject({ name: 'Cell notes.docx', parent: 'f3', content: 'WORD-FILE-1' }); + expect(await loadCopies(host)).toHaveLength(1); + + await host.tempWrite('Cell notes.docx', { kind: 'text', text: 'WORD-FILE-2' }); + await sendFile(drive, local, { + node: 'page1', + scope: 'page', + format: 'docx', + folder: null, + folderName: 'Lab reports', + replace: copy.remote, + host, + }); + expect(files.size).toBe(1); + expect(files.get('file1')).toMatchObject({ content: 'WORD-FILE-2', versions: 2 }); + // Still one link for the page, not two. + expect(await loadCopies(host)).toHaveLength(1); + }); + + it('turns a Word file into a Google Doc when asked', async () => { + const server = new MockServer(); + const { files } = addDrive(server, folders); + const { client, host, local } = await rig({ google: 'sam@example.com' }, server); + await sendFile(driveDestination({ client, host }), local, { + node: 'page1', + scope: 'page', + format: 'gdoc', + folder: null, + folderName: 'My Drive', + host, + }); + expect(files.get('file1')).toMatchObject({ name: 'Cell notes', mimeType: 'application/vnd.google-apps.document' }); + }); + + it('does not link a selection, which cannot be made again', async () => { + const server = new MockServer(); + addDrive(server, folders); + const { client, host, local } = await rig({ google: 'sam@example.com' }, server); + await sendFile(driveDestination({ client, host }), local, { + node: 'page1', + scope: 'selection', + format: 'docx', + folder: null, + folderName: 'My Drive', + host, + }); + expect(await loadCopies(host)).toEqual([]); + }); +}); + +describe('OneDrive', () => { + const items = [ + { id: 'a', name: 'Notes' }, + { id: 'b', name: 'Readme.txt', file: true }, + { id: 'c', name: 'School', parent: undefined }, + { id: 'd', name: 'Biology', parent: 'c' }, + ]; + + it('lists only folders', async () => { + const server = new MockServer(); + addOneDrive(server, items); + const { client, host } = await rig({ microsoft: 'sam@example.com' }, server); + const onedrive = onedriveDestination({ client, host }); + expect((await onedrive.folders(null)).map((one) => one.name)).toEqual(['Notes', 'School']); + expect((await onedrive.folders('c')).map((one) => one.name)).toEqual(['Biology']); + }); + + it('sends into a folder, renames rather than overwrites, and replaces its own copy', async () => { + const server = new MockServer(); + const { files } = addOneDrive(server, items); + const { client, host, local } = await rig({ microsoft: 'sam@example.com' }, server); + const onedrive = onedriveDestination({ client, host }); + const first = await sendFile(onedrive, local, { + node: 'sec1', + scope: 'section', + format: 'docx', + folder: 'd', + folderName: 'Biology', + host, + }); + const second = await sendFile(onedrive, local, { + node: 'sec2', + scope: 'section', + format: 'docx', + folder: 'd', + folderName: 'Biology', + host, + }); + expect(files.get(first.remote)?.name).toBe('Cell notes.docx'); + expect(files.get(second.remote)?.name).toBe('Cell notes.docx 1'); + await host.tempWrite('Cell notes.docx', { kind: 'text', text: 'WORD-FILE-2' }); + await sendFile(onedrive, local, { + node: 'sec1', + scope: 'section', + format: 'docx', + folder: null, + folderName: 'Biology', + replace: first.remote, + host, + }); + expect(files.get(first.remote)).toMatchObject({ content: 'WORD-FILE-2', versions: 2 }); + }); +}); + +describe('the link between a page and its copy', () => { + it('finds the copies of a page and the section and notebook it sits in', () => { + const base = { + service: 'drive', + remote: 'r', + name: 'n', + link: '', + folder: '', + format: 'docx', + sentAt: '', + } as const; + const copies = [ + { ...base, node: 'page1', scope: 'page' }, + { ...base, node: 'sec1', scope: 'section' }, + { ...base, node: 'other', scope: 'page' }, + ] as const; + expect(copiesOf(copies, ['page1', 'sec1', 'nb1']).map((one) => one.node)).toEqual(['page1', 'sec1']); + }); + + it('counts a page as changed only when it changed after the copy was sent', () => { + expect(changedSince({ sentAt: '2026-10-01T10:00:00Z' }, '2026-10-01T10:00:01Z')).toBe(true); + expect(changedSince({ sentAt: '2026-10-01T10:00:00Z' }, '2026-10-01T09:00:00Z')).toBe(false); + expect(changedSince({ sentAt: 'nonsense' }, '2026-10-01T09:00:00Z')).toBe(false); + }); + + it('picks the Dropbox, Box, and WebDAV copies a sync sends again', () => { + const base = { + remote: 'r', + name: 'n', + link: '', + folder: '', + format: 'pdf', + sentAt: '2026-10-01T10:00:00Z', + } as const; + const copies = [ + { ...base, service: 'dropbox', node: 'changed', scope: 'page' }, + { ...base, service: 'box', node: 'quiet', scope: 'page' }, + { ...base, service: 'webdav', node: 'sec', scope: 'section' }, + { ...base, service: 'drive', node: 'changed', scope: 'page' }, + { ...base, service: 'dropbox', node: 'changed', scope: 'page', format: 'png' }, + ] as const; + const modified = (node: string) => (node === 'quiet' ? '2026-10-01T09:00:00Z' : '2026-10-02T09:00:00Z'); + const due = copiesDue(copies, ['dropbox', 'box', 'webdav'], modified); + expect(due.map((one) => [one.service, one.node].join(':'))).toEqual(['dropbox:changed', 'webdav:sec']); + }); + + it('offers the types each service can take', () => { + expect(formatsFor('drive', 'page')).toEqual(['pdf', 'docx', 'xlsx', 'pptx', 'gdoc', 'gsheet', 'gslides']); + expect(formatsFor('onedrive', 'page')).toEqual(['pdf', 'docx', 'xlsx', 'pptx']); + expect(formatsFor('onedrive', 'notebook')).toEqual(['docx', 'xlsx', 'pptx']); + expect(formatsFor('drive', 'selection')).toEqual(['png', 'docx', 'gdoc']); + }); +}); diff --git a/app/src/features/accounts/sendto/storage.test.ts b/app/src/features/accounts/sendto/storage.test.ts new file mode 100644 index 00000000..f3164ace --- /dev/null +++ b/app/src/features/accounts/sendto/storage.test.ts @@ -0,0 +1,124 @@ +// Send to Dropbox, Box, and a WebDAV server against mocks: the folders, a file sent new, a taken name never overwritten, +// and a copy updated in place. +import { describe, expect, it } from 'vitest'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { addBox, addDropbox, addWebdav } from '../../../../../tests/mock-servers/storage'; +import { createFakeConnectors } from '../../connectors'; +import { createFakeAccountsHost } from '../fakeHost'; +import { boxDestination } from './box'; +import { sendFile } from './commands'; +import { loadCopies } from './copies'; +import { dropboxDestination } from './dropbox'; +import { DOCX } from './types'; +import type { LocalFile } from './types'; +import { foldersIn, webdavDestination } from './webdav'; + +async function rig(connector: string, server: MockServer, connect?: { token: string; baseUrl: string }) { + const connectors = createFakeConnectors({ + respond: server.respond, + ...(connect ? {} : { connected: { [connector]: 'sam@example.com' } }), + }); + if (connect) await connectors.client.connect(connector, connect); + const host = createFakeAccountsHost(() => connectors.client); + await host.tempWrite('Notes.docx', { kind: 'text', text: 'FILE-1' }); + const local: LocalFile = { file: 'Notes.docx', name: 'Notes.docx', mime: DOCX }; + return { client: connectors.client, host, local }; +} + +const options = (host: Awaited>['host'], extra: Record = {}) => ({ + node: 'page1', + scope: 'page' as const, + format: 'docx' as const, + folder: null, + folderName: 'top', + host, + ...extra, +}); + +describe('Dropbox', () => { + it('lists folders, sends a file, renames a taken name, and updates the copy by its path', async () => { + const server = new MockServer(); + const { files } = addDropbox(server, ['/School', '/School/Biology', '/Work']); + const { client, host, local } = await rig('dropbox', server); + const dropbox = dropboxDestination({ client, host }); + expect((await dropbox.folders(null)).map((one) => one.name)).toEqual(['School', 'Work']); + expect((await dropbox.folders('/School')).map((one) => one.name)).toEqual(['Biology']); + + const first = await sendFile(dropbox, local, options(host, { folder: '/School/Biology' })); + const second = await sendFile(dropbox, local, options(host, { folder: '/School/Biology', node: 'page2' })); + expect(first.remote).toBe('/school/biology/notes.docx'); + expect(second.name).toBe('Notes (1).docx'); + expect(files.size).toBe(2); + + await host.tempWrite('Notes.docx', { kind: 'text', text: 'FILE-2' }); + await sendFile(dropbox, local, options(host, { replace: first.remote })); + expect(files.get('/school/biology/notes.docx')).toMatchObject({ content: 'FILE-2', versions: 2 }); + expect(files.size).toBe(2); + expect(await loadCopies(host)).toHaveLength(2); + }); +}); + +describe('Box', () => { + it('lists only folders, sends a file, and sends again under a number when the name is taken', async () => { + const server = new MockServer(); + const { files } = addBox(server, [ + { id: '11', name: 'Biology', parent: '0' }, + { id: '12', name: 'Chemistry', parent: '0' }, + ]); + const { client, host, local } = await rig('box', server); + const box = boxDestination({ client, host }); + expect((await box.folders(null)).map((one) => one.name)).toEqual(['Biology', 'Chemistry']); + const first = await sendFile(box, local, options(host, { folder: '11' })); + const second = await sendFile(box, local, options(host, { folder: '11', node: 'page2' })); + expect(files.get(first.remote)).toMatchObject({ name: 'Notes.docx', parent: '11', content: 'FILE-1' }); + expect(second.name).toBe('Notes (1).docx'); + + await host.tempWrite('Notes.docx', { kind: 'text', text: 'FILE-2' }); + await sendFile(box, local, options(host, { replace: first.remote })); + expect(files.get(first.remote)).toMatchObject({ content: 'FILE-2', versions: 2 }); + }); +}); + +describe('WebDAV', () => { + const connect = { token: 'sam:app-password', baseUrl: 'https://files.example.org/dav/' }; + + it('reads the folders of a PROPFIND answer without the folder itself or the files', () => { + const xml = + // checks-disable-next-line length: one line of WebDAV XML, as the server sends it + '/dav/' + + // checks-disable-next-line length: one line of WebDAV XML, as the server sends it + '/dav/My%20Notes/' + + '/dav/a.txt'; + expect(foldersIn(xml, '/dav/')).toEqual([{ id: '/dav/My Notes', name: 'My Notes' }]); + }); + + it('lists folders from the server address, sends without overwriting, and updates the copy', async () => { + const server = new MockServer(); + const { files } = addWebdav(server, ['/dav/School', '/dav/School/Biology']); + const { client, host, local } = await rig('webdav', server, connect); + const webdav = webdavDestination({ client, host }); + expect((await webdav.folders(null)).map((one) => one.name)).toEqual(['School']); + expect((await webdav.folders('/dav/School')).map((one) => one.name)).toEqual(['Biology']); + + const first = await sendFile(webdav, local, options(host, { folder: '/dav/School/Biology' })); + const second = await sendFile(webdav, local, options(host, { folder: '/dav/School/Biology', node: 'page2' })); + expect(first.remote).toBe('/dav/School/Biology/Notes.docx'); + expect(second.remote).toBe('/dav/School/Biology/Notes (1).docx'); + expect(files.size).toBe(2); + expect(server.requests.filter((one) => one.method === 'PUT')[0]?.headers['if-none-match']).toBe('*'); + + await host.tempWrite('Notes.docx', { kind: 'text', text: 'FILE-2' }); + await sendFile(webdav, local, options(host, { replace: first.remote })); + expect(files.get(first.remote)).toMatchObject({ content: 'FILE-2', versions: 2 }); + // Updating does not ask the server to refuse. + expect(server.requests.filter((one) => one.method === 'PUT').at(-1)?.headers['if-none-match']).toBeUndefined(); + }); + + it('sends to the top of the server when no folder is chosen', async () => { + const server = new MockServer(); + const { files } = addWebdav(server, []); + const { client, host, local } = await rig('webdav', server, connect); + await sendFile(webdavDestination({ client, host }), local, options(host)); + expect([...files.keys()]).toEqual(['/dav/Notes.docx']); + }); +}); diff --git a/app/src/features/accounts/sendto/types.ts b/app/src/features/accounts/sendto/types.ts new file mode 100644 index 00000000..b51d4cd9 --- /dev/null +++ b/app/src/features/accounts/sendto/types.ts @@ -0,0 +1,79 @@ +// Send to Google Drive and OneDrive: the words the parts share. + +export type SendService = 'drive' | 'onedrive' | 'dropbox' | 'box' | 'webdav'; +export type SendScope = 'selection' | 'page' | 'section' | 'notebook'; +/** `gdoc`, `gsheet`, and `gslides` are Office files that Drive turns into Google's own as it takes them. */ +export type SendFormat = 'pdf' | 'docx' | 'xlsx' | 'pptx' | 'gdoc' | 'gsheet' | 'gslides' | 'png'; + +export interface RemoteFolder { + id: string; + name: string; +} + +/** A file made on this PC, in the transfer folder, ready to send. */ +export interface LocalFile { + /** A name in the transfer folder, or a path inside it. */ + file: string; + name: string; + mime: string; +} + +export interface Sent { + id: string; + name: string; + link: string; +} + +/** A service as the Send to flow uses it. */ +export interface Destination { + service: SendService; + /** The folders inside a folder; null is the top. */ + folders(parent: string | null): Promise; + /** Sends a new file. `convertTo` is the type of Google file an Office file becomes, or null to keep it as it is. */ + send(local: LocalFile, folder: string | null, convertTo: string | null): Promise; + /** Replaces the content of a file sent before. */ + replace(local: LocalFile, remote: string): Promise; +} + +/** A page, section, or notebook that is linked to the file it was sent as. */ +export interface SentCopy { + service: SendService; + remote: string; + name: string; + link: string; + /** The page, section, or notebook that was sent. */ + node: string; + scope: Exclude; + format: SendFormat; + folder: string; + sentAt: string; +} + +export const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +export const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +export const PPTX = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'; + +/** What each Office type is as a file, and the Google file Drive makes of it. */ +export const OFFICE: Record<'docx' | 'xlsx' | 'pptx', { mime: string; convertTo: string }> = { + docx: { mime: DOCX, convertTo: 'application/vnd.google-apps.document' }, + xlsx: { mime: XLSX, convertTo: 'application/vnd.google-apps.spreadsheet' }, + pptx: { mime: PPTX, convertTo: 'application/vnd.google-apps.presentation' }, +}; + +/** The Office type a format is made from, and whether Drive converts it. Null for a PDF or a picture. */ +export function officeOf(format: SendFormat): { type: 'docx' | 'xlsx' | 'pptx'; convert: boolean } | null { + switch (format) { + case 'docx': + case 'xlsx': + case 'pptx': + return { type: format, convert: false }; + case 'gdoc': + return { type: 'docx', convert: true }; + case 'gsheet': + return { type: 'xlsx', convert: true }; + case 'gslides': + return { type: 'pptx', convert: true }; + default: + return null; + } +} diff --git a/app/src/features/accounts/sendto/webdav.ts b/app/src/features/accounts/sendto/webdav.ts new file mode 100644 index 00000000..e0dd6150 --- /dev/null +++ b/app/src/features/accounts/sendto/webdav.ts @@ -0,0 +1,100 @@ +// WebDAV: a server the person gave an address, a user name, and a password for (Nextcloud, ownCloud, a NAS, and the +// like). Folders are listed with PROPFIND, and a file is sent with PUT. A name that is taken is not overwritten: the +// request says "only if nothing is there" and the file goes again under a new name when the server says it is taken. +// Updating the copy overwrites on purpose. The address is the server's own, and the user and password are the host's. +import { ConnectorHttpError } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import { schoolAddress } from '../lms/canvas'; +import type { Destination, LocalFile, RemoteFolder, Sent } from './types'; + +const PROPFIND = ``; + +/** The folders named in a PROPFIND answer for a folder, without the folder itself. */ +export function foldersIn(xml: string, here: string): RemoteFolder[] { + const found: RemoteFolder[] = []; + const base = decodeURIComponent(here).replace(/\/+$/, ''); + for (const part of xml.split(/<(?:\w+:)?response[\s>]/i).slice(1)) { + const href = /<(?:\w+:)?href>([^<]*)<\/(?:\w+:)?href>/i.exec(part)?.[1]; + if (!href || !/<(?:\w+:)?collection\b/i.test(part)) continue; + let path = href.trim(); + try { + path = decodeURIComponent(new URL(path, 'https://example.invalid').pathname); + } catch { + continue; + } + path = path.replace(/\/+$/, ''); + if (path === base || path === '') continue; + const name = path.slice(path.lastIndexOf('/') + 1); + if (name) found.push({ id: path, name }); + } + return found.sort((a, b) => a.name.localeCompare(b.name)); +} + +export function webdavDestination(deps: { client: ConnectorsClient; host: AccountsHost }): Destination { + const { client, host } = deps; + + /** The part of the server's address before the folders: its path, which a folder's path includes. */ + async function root(): Promise<{ origin: string; path: string }> { + const url = new URL(await schoolAddress(client, 'webdav')); + return { origin: url.origin, path: url.pathname.replace(/\/+$/, '') }; + } + + const encoded = (path: string): string => path.split('/').map(encodeURIComponent).join('/'); + + async function put(path: string, local: LocalFile, replace: boolean): Promise { + const { origin } = await root(); + return host.upload( + 'webdav', + ['webdavFiles'], + { method: 'PUT', url: `${origin}${encoded(path)}`, ...(replace ? {} : { headers: { 'If-None-Match': '*' } }) }, + [{ kind: 'file', file: local.file }], + local.mime, + ); + } + + type Reply = Awaited>; + + async function sent(path: string): Promise { + const { origin } = await root(); + return { id: path, name: path.slice(path.lastIndexOf('/') + 1), link: `${origin}${encoded(path)}` }; + } + + return { + service: 'webdav', + async folders(parent) { + const { origin, path } = await root(); + const here = parent ?? `${path}/`; + const answer = await client.request('webdav', ['webdavFiles'], { + method: 'PROPFIND', + url: `${origin}${encoded(here.endsWith('/') ? here : `${here}/`)}`, + headers: { Depth: '1' }, + contentType: 'application/xml', + body: PROPFIND, + }); + if (answer.status < 200 || answer.status >= 300) throw new ConnectorHttpError(answer.status, answer.body); + return foldersIn(answer.body, here); + }, + async send(local, folder) { + const { path } = await root(); + const dir = (folder ?? path).replace(/\/+$/, ''); + const dot = local.name.lastIndexOf('.'); + const stem = dot > 0 ? local.name.slice(0, dot) : local.name; + const extension = dot > 0 ? local.name.slice(dot) : ''; + for (let copy = 0; copy <= 9; copy += 1) { + const name = copy === 0 ? local.name : `${stem} (${copy})${extension}`; + const answer = await put(`${dir}/${name}`, local, false); + // 412 is "something is there already", and some servers say 409 or 405 for the same thing. + if ([412, 409, 405].includes(answer.status)) continue; + if (answer.status < 200 || answer.status >= 300) throw new ConnectorHttpError(answer.status, answer.body); + return sent(`${dir}/${name}`); + } + throw new ConnectorHttpError(409, JSON.stringify({ message: 'The name is taken.' })); + }, + async replace(local, remote) { + const answer = await put(remote, local, true); + if (answer.status < 200 || answer.status >= 300) throw new ConnectorHttpError(answer.status, answer.body); + return sent(remote); + }, + }; +} diff --git a/app/src/features/accounts/share/commands.ts b/app/src/features/accounts/share/commands.ts new file mode 100644 index 00000000..a717c15f --- /dev/null +++ b/app/src/features/accounts/share/commands.ts @@ -0,0 +1,119 @@ +// Share page to Slack and Share page to Teams: choose the channel, choose what to post (a PDF, a picture, or a link), and +// post it as the person. Nothing is posted until the last button, and the page itself is never changed. + +import { commandContext } from '../../../commands/registry'; +import type { NodeId } from '../../../services/notes/types'; +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { shownMounted } from '../../page'; +import { renderShownPage } from '../../pages'; +import { formatLink } from '../../search'; +import { accountsHost } from '../host'; +import { attempt, isConnected, tellNotConnected } from '../run'; +import { pickOne } from '../ui/prompts'; +import type { Choice } from '../ui/prompts'; +import { shareToSlack, listSlackChannels } from './slack'; +import { listTeamChannels, listTeams, shareToTeams } from './teams'; +import type { Channel, ShareKind, Shared } from './types'; + +const linkText = (title: string, link: string): string => t('accounts.share.message.link', { title, link }); + +function chooseChannel(channels: readonly Channel[], title: string, service: string): Promise { + const choices: Choice[] = channels.map((channel) => ({ + id: channel.id, + label: channel.name, + value: channel, + })); + return pickOne({ + title, + description: t('accounts.share.channel.description'), + choices, + confirmLabel: t('accounts.share.channel.confirm'), + empty: t('accounts.share.channel.none', { service }), + }); +} + +function chooseKind(): Promise { + return pickOne({ + title: t('accounts.share.kind.title'), + description: t('accounts.share.kind.description'), + confirmLabel: t('accounts.share.kind.confirm'), + choices: [ + { id: 'pdf', label: t('accounts.share.kind.pdf'), detail: t('accounts.share.kind.pdfDetail'), value: 'pdf' }, + { id: 'png', label: t('accounts.share.kind.png'), detail: t('accounts.share.kind.pngDetail'), value: 'png' }, + { id: 'link', label: t('accounts.share.kind.link'), detail: t('accounts.share.kind.linkDetail'), value: 'link' }, + ], + }); +} + +/** The shown page as what is posted, or null when there is nothing to post. */ +async function prepare(kind: ShareKind): Promise { + const mounted = shownMounted.get(); + if (!mounted) { + showToast({ message: t('accounts.share.nothing') }); + return null; + } + const context = commandContext('palette'); + if (kind === 'link') { + const title = (await context.notes.get(mounted.page.id as NodeId))?.title ?? ''; + return { kind: 'link', title: title || t('pageViews.print.untitled'), link: formatLink(mounted.page.id) }; + } + const rendered = await renderShownPage(context, kind, true); + if (!rendered) { + showToast({ message: t('accounts.share.failedRender'), tone: 'danger' }); + return null; + } + return { kind: 'file', title: rendered.title, name: rendered.name, mime: rendered.mime, bytes: rendered.bytes }; +} + +async function finish(channel: Channel, shared: Shared, post: () => Promise): Promise { + const label = `#${channel.name}`; + announce(t('accounts.share.working', { channel: label })); + await post(); + const message = t('accounts.share.done', { title: shared.title, channel: label }); + showToast({ message }); + announce(message); +} + +export async function shareSlack(): Promise { + if (!shownMounted.get()) return void showToast({ message: t('accounts.share.nothing') }); + if (!(await isConnected('slack'))) return tellNotConnected('Slack'); + const client = commandContext('palette').platform.connectors; + await attempt('Slack', async () => { + const channel = await chooseChannel(await listSlackChannels(client), t('accounts.share.channel.title'), 'Slack'); + if (!channel) return; + const kind = await chooseKind(); + if (!kind) return; + const shared = await prepare(kind); + if (!shared) return; + await finish(channel, shared, () => shareToSlack(client, accountsHost(), channel, shared, linkText)); + }); +} + +export async function shareTeams(): Promise { + if (!shownMounted.get()) return void showToast({ message: t('accounts.share.nothing') }); + if (!(await isConnected('microsoft'))) return tellNotConnected('Microsoft'); + const client = commandContext('palette').platform.connectors; + await attempt('Teams', async () => { + const teams = await listTeams(client); + const team = await pickOne({ + title: t('accounts.share.channel.teamsTitle'), + description: t('accounts.share.channel.description'), + choices: teams.map((one) => ({ id: one.id, label: one.name, value: one })), + confirmLabel: t('accounts.share.channel.confirm'), + empty: t('accounts.share.channel.none', { service: 'Teams' }), + }); + if (!team) return; + const channel = await chooseChannel( + await listTeamChannels(client, team), + t('accounts.share.channel.teamsChannelTitle', { team: team.name }), + 'Teams', + ); + if (!channel) return; + const kind = await chooseKind(); + if (!kind) return; + const shared = await prepare(kind); + if (!shared) return; + await finish(channel, shared, () => shareToTeams(client, accountsHost(), channel, shared, linkText)); + }); +} diff --git a/app/src/features/accounts/share/share.test.ts b/app/src/features/accounts/share/share.test.ts new file mode 100644 index 00000000..9b4f7ed1 --- /dev/null +++ b/app/src/features/accounts/share/share.test.ts @@ -0,0 +1,136 @@ +// Share to Slack and Teams against mock services: the channel lists, a link message, a file in three steps, a picture in +// a Teams message, a PDF in the channel's Files folder, and Slack's refusals said as failures. +import { describe, expect, it } from 'vitest'; +import { addSlack, addTeams } from '../../../../../tests/mock-servers/chat'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { ConnectorHttpError, createFakeConnectors } from '../../connectors'; +import { createFakeAccountsHost } from '../fakeHost'; +import { serviceWords } from '../run'; +import { listSlackChannels, shareToSlack } from './slack'; +import { listTeamChannels, listTeams, shareToTeams } from './teams'; +import type { Shared } from './types'; + +const text = (title: string, link: string) => `${title}: ${link}`; +const bytes = new TextEncoder().encode('%PDF-1.7 sample'); +const pdf: Shared = { kind: 'file', title: 'Cell notes', name: 'Cell notes.pdf', mime: 'application/pdf', bytes }; +const png: Shared = { kind: 'file', title: 'Cell notes', name: 'Cell notes.png', mime: 'image/png', bytes }; +const link: Shared = { kind: 'link', title: 'Cell notes', link: 'opennote://page/abc123' }; + +function rig(connected: Record, server: MockServer) { + const connectors = createFakeConnectors({ connected, respond: server.respond }); + return { client: connectors.client, host: createFakeAccountsHost(() => connectors.client) }; +} + +describe('Slack', () => { + const channels = [ + { id: 'C3', name: 'random' }, + { id: 'C1', name: 'biology' }, + { id: 'C2', name: 'old', is_archived: true }, + { id: 'C4', name: 'not-joined', is_member: false }, + { id: 'C5', name: 'chem' }, + ]; + + it('lists the channels the person is in, across pages, by name', async () => { + const server = new MockServer(); + addSlack(server, channels); + const { client } = rig({ slack: 'Lab' }, server); + expect((await listSlackChannels(client)).map((one) => one.name)).toEqual(['biology', 'chem', 'random']); + expect(server.requests).toHaveLength(3); + }); + + it('posts a link as a message', async () => { + const server = new MockServer(); + const slack = addSlack(server, channels); + const { client, host } = rig({ slack: 'Lab' }, server); + await shareToSlack(client, host, { id: 'C1', name: 'biology' }, link, text); + expect(slack.posted).toEqual([{ channel: 'C1', text: 'Cell notes: opennote://page/abc123' }]); + }); + + it('uploads a file in three steps and shares it in the channel', async () => { + const server = new MockServer(); + const slack = addSlack(server, channels); + const { client, host } = rig({ slack: 'Lab' }, server); + await shareToSlack(client, host, { id: 'C1', name: 'biology' }, pdf, text); + expect(server.requests.map((one) => one.path)).toEqual([ + '/api/files.getUploadURLExternal', + '/upload/v1/F1', + '/api/files.completeUploadExternal', + ]); + expect(server.requests[0]?.form).toEqual({ filename: 'Cell notes.pdf', length: String(bytes.length) }); + expect(slack.posted).toEqual([ + { channel: 'C1', file: { id: 'F1', filename: 'Cell notes.pdf', bytes: bytes.length, title: 'Cell notes' } }, + ]); + }); + + it('says a missing permission as a failure that names the scope problem, and never posts', async () => { + const server = new MockServer(); + const slack = addSlack(server, channels, { scopes: ['channels:read'] }); + const { client, host } = rig({ slack: 'Lab' }, server); + const error = await shareToSlack(client, host, { id: 'C1', name: 'biology' }, link, text).catch((one) => one); + expect(error).toBeInstanceOf(ConnectorHttpError); + expect((error as ConnectorHttpError).status).toBe(403); + expect(serviceWords(error as ConnectorHttpError)).toBe('missing_scope'); + expect(slack.posted).toEqual([]); + }); +}); + +describe('Teams', () => { + const teams = [ + { + id: 't2', + displayName: 'Study group', + channels: [ + { id: 'c9', displayName: 'General' }, + { id: 'c10', displayName: 'Exams' }, + ], + }, + { id: 't1', displayName: 'Biology 101', channels: [{ id: 'c1', displayName: 'General' }] }, + ]; + + it('lists the teams and their channels', async () => { + const server = new MockServer(); + addTeams(server, teams); + const { client } = rig({ microsoft: 'sam@example.com' }, server); + const listed = await listTeams(client); + expect(listed.map((one) => one.name)).toEqual(['Biology 101', 'Study group']); + const channels = await listTeamChannels(client, listed[1] ?? { id: '', name: '' }); + expect(channels).toEqual([ + { id: 'c9', name: 'General', team: 't2' }, + { id: 'c10', name: 'Exams', team: 't2' }, + ]); + }); + + it('posts a link, a hosted picture, and a PDF through the channel Files folder', async () => { + const server = new MockServer(); + const teamsMock = addTeams(server, teams); + const { client, host } = rig({ microsoft: 'sam@example.com' }, server); + const channel = { id: 'c9', name: 'General', team: 't2' }; + await shareToTeams(client, host, channel, link, text); + await shareToTeams(client, host, channel, png, text); + await shareToTeams(client, host, channel, pdf, text); + expect(teamsMock.posted.map((one) => one.hostedContents)).toEqual([0, 1, 0]); + expect(teamsMock.posted[0]?.html).toBe('

    Cell notes: opennote://page/abc123

    '); + expect(teamsMock.posted[1]?.html).toContain('hostedContents/1/$value'); + expect(teamsMock.files).toEqual([{ drive: 'drive-t2', folder: '', name: 'Cell notes.pdf', bytes: bytes.length }]); + expect(teamsMock.posted[2]?.html).toBe( + '

    Cell notes.pdf

    ', + ); + const scopes = server.requests.map((one) => one.connector); + expect(new Set(scopes)).toEqual(new Set(['microsoft'])); + }); + + it('escapes a page title that holds markup', async () => { + const server = new MockServer(); + const teamsMock = addTeams(server, teams); + const { client, host } = rig({ microsoft: 'sam@example.com' }, server); + await shareToTeams( + client, + host, + { id: 'c9', name: 'General', team: 't2' }, + { ...png, title: 'x & "y"' }, + text, + ); + expect(teamsMock.posted[0]?.html).toContain('<b>x</b> & "y"'); + expect(teamsMock.posted[0]?.html).not.toContain(''); + }); +}); diff --git a/app/src/features/accounts/share/slack.ts b/app/src/features/accounts/share/slack.ts new file mode 100644 index 00000000..f6e27110 --- /dev/null +++ b/app/src/features/accounts/share/slack.ts @@ -0,0 +1,132 @@ +// Slack: the channels the person can post in, a message, and a file. Slack answers 200 even when it refuses, with +// `ok: false` and an error word, so every answer is checked. The token is the host's and posts as the person. + +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import { bytesPart } from '../host'; +import type { Channel, Shared } from './types'; + +const BASE = 'https://slack.com/api'; +const MAX_PAGES = 20; + +interface SlackAnswer { + ok?: boolean; + error?: string; +} + +/** Slack's refusal as the failure the commands already say in words. */ +function check(answer: T | undefined): T { + if (!answer || answer.ok !== true) { + const error = answer?.error ?? 'unknown_error'; + const status = error === 'missing_scope' || error === 'invalid_auth' ? 403 : 400; + throw new ConnectorHttpError(status, JSON.stringify({ error })); + } + return answer; +} + +interface ChannelList extends SlackAnswer { + channels?: { id: string; name: string; is_archived?: boolean; is_member?: boolean }[]; + response_metadata?: { next_cursor?: string }; +} + +/** The public and private channels the person is in, by name. */ +export async function listSlackChannels(client: ConnectorsClient): Promise { + const found: Channel[] = []; + let cursor = ''; + for (let page = 0; page < MAX_PAGES; page += 1) { + const reply = await requestConnector( + 'slack', + ['slackChannels'], + { + url: `${BASE}/conversations.list`, + query: { + types: 'public_channel,private_channel', + exclude_archived: 'true', + limit: 200, + cursor: cursor || null, + }, + }, + client, + ); + const answer = check(reply.data); + for (const channel of answer.channels ?? []) { + if (channel.is_archived || channel.is_member === false) continue; + found.push({ id: channel.id, name: channel.name }); + } + cursor = answer.response_metadata?.next_cursor ?? ''; + if (!cursor) break; + } + return found.sort((a, b) => a.name.localeCompare(b.name)); +} + +async function post(client: ConnectorsClient, channel: string, text: string): Promise { + const reply = await requestConnector( + 'slack', + ['slackPost'], + { url: `${BASE}/chat.postMessage`, json: { channel, text, unfurl_links: false } }, + client, + ); + check(reply.data); +} + +interface UploadSlot extends SlackAnswer { + upload_url?: string; + file_id?: string; +} + +/** Uploads the file in Slack's three steps (a slot, the bytes, then the post) and shares it in the channel. */ +async function postFile( + client: ConnectorsClient, + host: AccountsHost, + channel: string, + shared: Extract, +): Promise { + const slot = check( + ( + await requestConnector( + 'slack', + ['slackFiles'], + { + url: `${BASE}/files.getUploadURLExternal`, + form: { filename: shared.name, length: String(shared.bytes.length) }, + }, + client, + ) + ).data, + ); + if (!slot.upload_url || !slot.file_id) throw new ConnectorHttpError(502, ''); + const sent = await host.upload( + 'slack', + ['slackFiles'], + { method: 'POST', url: slot.upload_url }, + [bytesPart(shared.bytes)], + shared.mime, + ); + if (sent.status < 200 || sent.status >= 300) throw new ConnectorHttpError(sent.status, sent.body); + check( + ( + await requestConnector( + 'slack', + ['slackFiles'], + { + url: `${BASE}/files.completeUploadExternal`, + json: { files: [{ id: slot.file_id, title: shared.title }], channel_id: channel }, + }, + client, + ) + ).data, + ); +} + +/** Posts the page to the channel: a message with its link, or the file. */ +export async function shareToSlack( + client: ConnectorsClient, + host: AccountsHost, + channel: Channel, + shared: Shared, + linkMessage: (title: string, link: string) => string, +): Promise { + if (shared.kind === 'link') await post(client, channel.id, linkMessage(shared.title, shared.link)); + else await postFile(client, host, channel.id, shared); +} diff --git a/app/src/features/accounts/share/teams.ts b/app/src/features/accounts/share/teams.ts new file mode 100644 index 00000000..57b3c5a0 --- /dev/null +++ b/app/src/features/accounts/share/teams.ts @@ -0,0 +1,122 @@ +// Teams: the person's teams and their channels, and a channel message. A picture goes in the message itself as a hosted +// image; a PDF is put in the channel's Files folder and the message links to it. The token is the host's and posts as +// the person. + +import { requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; +import { base64Of, bytesPart } from '../host'; +import type { Channel, Shared, Team } from './types'; + +const BASE = 'https://graph.microsoft.com/v1.0'; + +interface List { + value?: T[]; + '@odata.nextLink'?: string; +} + +async function listAll(client: ConnectorsClient, access: string, url: string): Promise { + const found: T[] = []; + let next: string | undefined = url; + for (let page = 0; next && page < 20; page += 1) { + const reply: { data?: List } = await requestConnector>('microsoft', [access], { url: next }, client); + found.push(...(reply.data?.value ?? [])); + next = reply.data?.['@odata.nextLink']; + } + return found; +} + +export async function listTeams(client: ConnectorsClient): Promise { + const teams = await listAll<{ id: string; displayName: string }>(client, 'teamsRead', `${BASE}/me/joinedTeams`); + return teams.map((team) => ({ id: team.id, name: team.displayName })).sort((a, b) => a.name.localeCompare(b.name)); +} + +export async function listTeamChannels(client: ConnectorsClient, team: Team): Promise { + const channels = await listAll<{ id: string; displayName: string }>( + client, + 'teamsRead', + `${BASE}/teams/${encodeURIComponent(team.id)}/channels`, + ); + return channels.map((channel) => ({ id: channel.id, name: channel.displayName, team: team.id })); +} + +const escape = (text: string): string => + text.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); + +interface Folder { + id: string; + parentReference?: { driveId?: string }; +} + +/** Puts the file in the channel's Files folder and answers its web address ('' when the answer has none). */ +async function putFile( + client: ConnectorsClient, + host: AccountsHost, + channel: Channel, + shared: Extract, +): Promise { + const team = encodeURIComponent(channel.team ?? ''); + const folder = ( + await requestConnector( + 'microsoft', + ['onedriveFiles'], + { url: `${BASE}/teams/${team}/channels/${encodeURIComponent(channel.id)}/filesFolder` }, + client, + ) + ).data; + const drive = folder?.parentReference?.driveId; + if (!folder || !drive) throw new Error('The channel has no Files folder.'); + const sent = await host.upload( + 'microsoft', + ['onedriveFiles'], + { + method: 'PUT', + // checks-disable-next-line length: one address built from its encoded parts + url: `${BASE}/drives/${encodeURIComponent(drive)}/items/${encodeURIComponent(folder.id)}:/${encodeURIComponent(shared.name)}:/content`, + }, + [bytesPart(shared.bytes)], + shared.mime, + ); + try { + return (JSON.parse(sent.body) as { webUrl?: string }).webUrl ?? ''; + } catch { + return ''; + } +} + +/** Posts the page to the channel. */ +export async function shareToTeams( + client: ConnectorsClient, + host: AccountsHost, + channel: Channel, + shared: Shared, + linkMessage: (title: string, link: string) => string, +): Promise { + const title = escape(shared.title); + let content: string; + const extra: Record = {}; + if (shared.kind === 'link') { + content = `

    ${escape(linkMessage(shared.title, shared.link))}

    `; + } else if (shared.mime === 'image/png') { + content = `

    ${title}

    ${title}`; + extra.hostedContents = [ + { + '@microsoft.graph.temporaryId': '1', + contentBytes: base64Of(shared.bytes), + contentType: shared.mime, + }, + ]; + } else { + const url = await putFile(client, host, channel, shared); + content = url ? `

    ${escape(shared.name)}

    ` : `

    ${escape(shared.name)}

    `; + } + await requestConnector( + 'microsoft', + ['teamsPost'], + { + url: `${BASE}/teams/${encodeURIComponent(channel.team ?? '')}/channels/${encodeURIComponent(channel.id)}/messages`, + json: { body: { contentType: 'html', content }, ...extra }, + }, + client, + ); +} diff --git a/app/src/features/accounts/share/types.ts b/app/src/features/accounts/share/types.ts new file mode 100644 index 00000000..74e35db1 --- /dev/null +++ b/app/src/features/accounts/share/types.ts @@ -0,0 +1,21 @@ +// What the two chat services share: a destination the person chooses, and a page to post there. + +export interface Channel { + /** The service's ID for it. */ + id: string; + name: string; + /** Teams: the team the channel is in. */ + team?: string; +} + +export interface Team { + id: string; + name: string; +} + +export type ShareKind = 'pdf' | 'png' | 'link'; + +/** What is posted: a file, or a link with the page's title. */ +export type Shared = + | { kind: 'file'; title: string; name: string; mime: string; bytes: Uint8Array } + | { kind: 'link'; title: string; link: string }; diff --git a/app/src/features/accounts/state.ts b/app/src/features/accounts/state.ts new file mode 100644 index 00000000..ea32c7c5 --- /dev/null +++ b/app/src/features/accounts/state.ts @@ -0,0 +1,30 @@ +// What a sync remembers between runs, kept by the host in this device's folder (accounts_call state.*): a cursor, a +// delta link, the IDs of copies at a service, a conflict log. Nothing here is a token. A name is lowercase letters, +// digits, dots, and dashes. Reading never throws: a missing or damaged file reads as the fallback. + +import { accountsHost } from './host'; +import type { AccountsHost } from './host'; + +/** The saved value with the fallback's keys filled in, or the fallback. */ +export async function loadState( + name: string, + fallback: T, + host: AccountsHost = accountsHost(), +): Promise { + try { + const saved = await host.stateGet>(name); + const merged = saved && typeof saved === 'object' && !Array.isArray(saved) ? { ...fallback, ...saved } : fallback; + // A copy, so a sync that changes what it loaded never changes the fallback or the host's own copy. + return structuredClone(merged); + } catch { + return structuredClone(fallback); + } +} + +export async function saveState(name: string, value: unknown, host: AccountsHost = accountsHost()): Promise { + await host.stateSet(name, value); +} + +export async function forgetState(name: string, host: AccountsHost = accountsHost()): Promise { + await host.stateDelete(name); +} diff --git a/app/src/features/accounts/tasks/auto.ts b/app/src/features/accounts/tasks/auto.ts new file mode 100644 index 00000000..9d22bbfc --- /dev/null +++ b/app/src/features/accounts/tasks/auto.ts @@ -0,0 +1,31 @@ +// The quiet sync. Once a service has been synced by hand, OpenNote keeps it in step while the app is open: shortly after +// start and then every quarter of an hour, with no message and no sound. A failure (offline, signed out) is ignored, and +// the next time just tries again. Turning the flag off, or disconnecting the account, stops it. +import { isEnabled } from '../../../app/flags'; +import { isConnected } from '../run'; +import { hasSynced, runSync } from './commands'; +import type { TasksService } from './commands'; + +const EVERY_MS = 15 * 60_000; +const FIRST_MS = 60_000; +let started = false; + +async function once(): Promise { + if (!isEnabled('accounts.tasksSync')) return; + for (const service of ['microsoft', 'google'] as const satisfies readonly TasksService[]) { + try { + if ((await hasSynced(service)) && (await isConnected(service))) await runSync(service); + } catch { + // Not now. The next run tries again. + } + } +} + +export function startAutoSync(): void { + if (started || typeof window === 'undefined') return; + started = true; + window.setTimeout(() => { + void once(); + window.setInterval(() => void once(), EVERY_MS); + }, FIRST_MS); +} diff --git a/app/src/features/accounts/tasks/commands.ts b/app/src/features/accounts/tasks/commands.ts new file mode 100644 index 00000000..6949abb3 --- /dev/null +++ b/app/src/features/accounts/tasks/commands.ts @@ -0,0 +1,99 @@ +// Sync tasks with Microsoft To Do or Google Tasks, and the log of changes that were made in both places. Each sync +// copies changes both ways. It is quiet: one short line when it is done, and the log for the rare conflict. +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { connectors } from '../../connectors'; +import { accountsHost } from '../host'; +import { attempt, isConnected, tellNotConnected } from '../run'; +import { loadState, saveState } from '../state'; +import { pickOne } from '../ui/prompts'; +import { googleTasks } from './google'; +import { upcomingTasks } from './local'; +import { syncTasks } from './sync'; +import { microsoftTodo } from './todo'; +import { EMPTY_STATE } from './types'; +import type { SyncResult, TasksApi, TasksState } from './types'; + +export type TasksService = 'microsoft' | 'google'; + +const stateName = (service: TasksService): string => `tasks.${service}`; +const CONNECTOR: Record = { microsoft: 'microsoft', google: 'google' }; + +export function apiFor(service: TasksService): TasksApi { + const client = connectors(); + return service === 'google' + ? googleTasks(client) + : microsoftTodo(client, Intl.DateTimeFormat().resolvedOptions().timeZone); +} + +/** Whether this service has been synced before, which is what lets the quiet automatic sync run. */ +export async function hasSynced(service: TasksService): Promise { + return (await loadState(stateName(service), EMPTY_STATE, accountsHost())).lastSync !== null; +} + +/** One sync, saved. Throws what the service says. */ +export async function runSync(service: TasksService): Promise { + const host = accountsHost(); + const state = await loadState(stateName(service), EMPTY_STATE, host); + const done = await syncTasks({ api: apiFor(service), local: upcomingTasks(), state }); + await saveState(stateName(service), done.state, host); + return done.result; +} + +export async function syncNow(service: TasksService): Promise { + const label = service === 'google' ? 'Google Tasks' : 'Microsoft To Do'; + if (!(await isConnected(CONNECTOR[service]))) return tellNotConnected(label); + await attempt(label, async () => { + announce(t('accounts.tasks.working', { service: label })); + const result = await runSync(service); + const message = + result.pulled + result.pushed + result.removed === 0 + ? t('accounts.tasks.upToDate', { service: label }) + : t('accounts.tasks.done', { + service: label, + pulled: result.pulled, + pushed: result.pushed, + removed: result.removed, + }); + showToast({ + message: + result.conflicts > 0 ? `${message} ${t('accounts.tasks.conflicts', { count: result.conflicts })}` : message, + }); + announce(message); + }); +} + +/** The changes made in both places, newest first. */ +export async function showConflicts(): Promise { + const choices = []; + for (const service of ['microsoft', 'google'] as const) { + const state = await loadState(stateName(service), EMPTY_STATE, accountsHost()); + for (const [at, one] of state.conflicts.entries()) { + choices.push({ + id: `${service}:${at}`, + label: one.title, + detail: t('accounts.tasks.log.entry', { + service: service === 'google' ? 'Google Tasks' : 'Microsoft To Do', + when: new Date(one.at).toLocaleString(), + kept: t(one.kept === 'here' ? 'accounts.tasks.log.keptHere' : 'accounts.tasks.log.keptThere'), + other: [ + one.other.done ? t('accounts.tasks.log.done') : '', + one.other.date ?? '', + one.other.time ?? '', + one.other.title, + ] + .filter(Boolean) + .join(' '), + }), + value: true as const, + }); + } + } + await pickOne({ + title: t('accounts.tasks.log.title'), + description: t('accounts.tasks.log.description'), + confirmLabel: t('accounts.tasks.log.close'), + choices, + empty: t('accounts.tasks.log.none'), + }); +} diff --git a/app/src/features/accounts/tasks/google.ts b/app/src/features/accounts/tasks/google.ts new file mode 100644 index 00000000..80fae2ee --- /dev/null +++ b/app/src/features/accounts/tasks/google.ts @@ -0,0 +1,91 @@ +// Google Tasks: the default list's tasks (finished ones too), and creating, changing, and deleting one. Google keeps a due +// date and no time of day, so the time is not synced there, and a finished task keeps its date. +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { RemoteTask, TaskFields, TasksApi } from './types'; + +const BASE = 'https://tasks.googleapis.com/tasks/v1/lists/@default/tasks'; +const ACCESS = ['tasksWrite'] as const; +const MAX_PAGES = 20; + +interface GoogleTask { + id: string; + title?: string; + status?: string; + due?: string; + updated?: string; + deleted?: boolean; +} + +interface GoogleList { + items?: GoogleTask[]; + nextPageToken?: string; +} + +function toRemote(task: GoogleTask): RemoteTask { + return { + id: task.id, + title: task.title ?? '', + done: task.status === 'completed', + date: task.due ? task.due.slice(0, 10) : null, + time: null, + updated: task.updated ?? new Date(0).toISOString(), + }; +} + +function body(fields: TaskFields): Record { + return { + title: fields.title, + status: fields.done ? 'completed' : 'needsAction', + // A cleared date is sent as null, and a task that is not finished must not carry a completion time. + due: fields.date ? `${fields.date}T00:00:00.000Z` : null, + ...(fields.done ? {} : { completed: null }), + }; +} + +export function googleTasks(client: ConnectorsClient): TasksApi { + const send = async (method: 'POST' | 'PATCH', url: string, fields: TaskFields): Promise => { + const reply = await client.request('google', [...ACCESS], { + method, + url, + contentType: 'application/json; charset=UTF-8', + body: JSON.stringify(body(fields)), + }); + if (reply.status < 200 || reply.status >= 300) throw new ConnectorHttpError(reply.status, reply.body); + return toRemote(JSON.parse(reply.body) as GoogleTask); + }; + return { + service: 'google', + label: 'Google Tasks', + async list() { + const found: RemoteTask[] = []; + let token: string | undefined; + for (let page = 0; page < MAX_PAGES; page += 1) { + const reply = await requestConnector( + 'google', + [...ACCESS], + { + url: BASE, + query: { showCompleted: true, showHidden: true, maxResults: 100, pageToken: token }, + }, + client, + ); + for (const task of reply.data?.items ?? []) if (!task.deleted) found.push(toRemote(task)); + token = reply.data?.nextPageToken; + if (!token) break; + } + return found; + }, + create: (fields) => send('POST', BASE, fields), + update: (id, fields) => send('PATCH', `${BASE}/${encodeURIComponent(id)}`, fields), + async remove(id) { + const reply = await client.request('google', [...ACCESS], { + method: 'DELETE', + url: `${BASE}/${encodeURIComponent(id)}`, + }); + if ((reply.status < 200 || reply.status >= 300) && reply.status !== 404) { + throw new ConnectorHttpError(reply.status, reply.body); + } + }, + }; +} diff --git a/app/src/features/accounts/tasks/local.ts b/app/src/features/accounts/tasks/local.ts new file mode 100644 index 00000000..a9482416 --- /dev/null +++ b/app/src/features/accounts/tasks/local.ts @@ -0,0 +1,55 @@ +// The person's own tasks in Upcoming, as the sync sees them: the items that are not read from a page, not from a calendar +// file or a course, not repeating, and not events. Those are the ones the person adds and ticks off themselves. +import { readUpcoming, updateUpcomingItems } from '../../tools'; +import type { Due, UpcomingItem } from '../../tools'; +import type { LocalTask, LocalTasks } from './types'; + +const two = (n: number): string => String(n).padStart(2, '0'); + +function isOwn(item: UpcomingItem): boolean { + return !item.page && !item.source && !item.repeat && (item.kind ?? 'task') === 'task'; +} + +export function toLocal(item: UpcomingItem): LocalTask { + return { + id: item.id, + title: item.title, + done: item.done, + date: item.due ? `${item.due.date.year}-${two(item.due.date.month)}-${two(item.due.date.day)}` : null, + time: item.due?.time ? `${two(item.due.time.hour)}:${two(item.due.time.minute)}` : null, + }; +} + +function dueOf(task: LocalTask): Due | null { + const date = /^(\d{4})-(\d{2})-(\d{2})$/.exec(task.date ?? ''); + if (!date) return null; + const time = /^(\d{2}):(\d{2})$/.exec(task.time ?? ''); + return { + date: { year: Number(date[1]), month: Number(date[2]), day: Number(date[3]) }, + time: time ? { hour: Number(time[1]), minute: Number(time[2]) } : null, + }; +} + +export function upcomingTasks(): LocalTasks { + return { + read: () => readUpcoming().items.filter(isOwn).map(toLocal), + write(put, remove) { + updateUpcomingItems((items) => { + const gone = new Set(remove); + const incoming = new Map(put.map((task) => [task.id, task])); + const next = items + .filter((item) => !gone.has(item.id)) + .map((item) => { + const task = incoming.get(item.id); + if (!task) return item; + incoming.delete(item.id); + return { ...item, title: task.title, done: task.done, due: dueOf(task) }; + }); + for (const task of incoming.values()) { + next.push({ id: task.id, title: task.title, done: task.done, due: dueOf(task), kind: 'task' }); + } + return next; + }); + }, + }; +} diff --git a/app/src/features/accounts/tasks/sync.ts b/app/src/features/accounts/tasks/sync.ts new file mode 100644 index 00000000..a7548d4d --- /dev/null +++ b/app/src/features/accounts/tasks/sync.ts @@ -0,0 +1,146 @@ +// The two-way sync of tasks. It remembers, for each linked task, what both sides held after the last sync (the base). A +// side that differs from the base has changed. A change on one side only is copied to the other. When both changed, the +// newer change wins, and the other copy is written to the conflict log so nothing is lost silently. A task deleted on +// one side is deleted on the other only if the other has not changed since; otherwise the change brings it back. +// +// The function does the work against a service's API and the local list, both passed in, and returns the new state for +// the caller to save, so the whole thing is tested without a service or a screen. +import type { Conflict, Link, LocalTask, LocalTasks, SyncResult, TaskFields, TasksApi, TasksState } from './types'; + +const MAX_CONFLICTS = 50; + +export const fieldsOf = (task: TaskFields): TaskFields => ({ + title: task.title, + done: task.done, + date: task.date, + time: task.time, +}); + +export const sameFields = (a: TaskFields, b: TaskFields): boolean => + a.title === b.title && a.done === b.done && a.date === b.date && a.time === b.time; + +/** The local ID of a task that came from the service. */ +export const localIdFor = (service: string, remote: string): string => `${service}:${remote}`; + +export interface SyncOptions { + api: TasksApi; + local: LocalTasks; + state: TasksState; + now?: number; +} + +export async function syncTasks(options: SyncOptions): Promise<{ state: TasksState; result: SyncResult }> { + const { api, local } = options; + const now = options.now ?? Date.now(); + const stamp = new Date(now).toISOString(); + const state: TasksState = structuredClone(options.state); + const result: SyncResult = { pulled: 0, pushed: 0, removed: 0, conflicts: 0 }; + const remote = new Map((await api.list()).map((task) => [task.id, task])); + const here = new Map(local.read().map((task) => [task.id, task])); + const put: LocalTask[] = []; + const drop: string[] = []; + + const modified = (id: string): number => { + const known = local.modifiedAt?.(id); + if (typeof known === 'number') return known; + state.seen[id] ??= now; + return state.seen[id] ?? now; + }; + const conflict = (entry: Conflict) => { + state.conflicts = [entry, ...state.conflicts].slice(0, MAX_CONFLICTS); + result.conflicts += 1; + }; + const link = (id: string, remoteId: string, base: TaskFields) => { + state.links[id] = { remote: remoteId, base: fieldsOf(base) } satisfies Link; + delete state.seen[id]; + }; + + const linkedRemote = new Set(Object.values(state.links).map((one) => one.remote)); + + for (const [id, entry] of Object.entries(state.links)) { + const mine = here.get(id); + const theirs = remote.get(entry.remote); + const base = entry.base; + if (!mine && !theirs) { + delete state.links[id]; + continue; + } + if (!mine && theirs) { + // Deleted here. Gone there too, unless it changed there since. + if (sameFields(theirs, base)) { + await api.remove(theirs.id); + delete state.links[id]; + result.removed += 1; + } else { + put.push({ id, ...fieldsOf(theirs) }); + link(id, theirs.id, theirs); + result.pulled += 1; + conflict({ at: stamp, title: theirs.title, kept: 'there', other: fieldsOf(base) }); + } + continue; + } + if (mine && !theirs) { + // Deleted there. Gone here too, unless it changed here since. + if (sameFields(mine, base)) { + drop.push(id); + delete state.links[id]; + result.removed += 1; + } else { + const made = await api.create(fieldsOf(mine)); + link(id, made.id, mine); + linkedRemote.add(made.id); + result.pushed += 1; + conflict({ at: stamp, title: mine.title, kept: 'here', other: fieldsOf(base) }); + } + continue; + } + if (!mine || !theirs) continue; + const changedHere = !sameFields(mine, base); + const changedThere = !sameFields(theirs, base); + if (!changedHere && !changedThere) continue; + if (changedHere && changedThere && sameFields(mine, theirs)) { + link(id, theirs.id, theirs); + continue; + } + let pushHere = changedHere; + if (changedHere && changedThere) { + pushHere = modified(id) >= Date.parse(theirs.updated); + conflict({ + at: stamp, + title: pushHere ? mine.title : theirs.title, + kept: pushHere ? 'here' : 'there', + other: fieldsOf(pushHere ? theirs : mine), + }); + } + if (pushHere) { + const updated = await api.update(theirs.id, fieldsOf(mine)); + link(id, updated.id, mine); + result.pushed += 1; + } else { + put.push({ id, ...fieldsOf(theirs) }); + link(id, theirs.id, theirs); + result.pulled += 1; + } + } + + // New there: bring it here. + for (const task of remote.values()) { + if (linkedRemote.has(task.id)) continue; + const id = localIdFor(api.service, task.id); + put.push({ id, ...fieldsOf(task) }); + link(id, task.id, task); + result.pulled += 1; + } + + // New here: send it there. + for (const task of here.values()) { + if (state.links[task.id]) continue; + const made = await api.create(fieldsOf(task)); + link(task.id, made.id, task); + result.pushed += 1; + } + + if (put.length > 0 || drop.length > 0) local.write(put, drop); + state.lastSync = stamp; + return { state, result }; +} diff --git a/app/src/features/accounts/tasks/tasks.test.ts b/app/src/features/accounts/tasks/tasks.test.ts new file mode 100644 index 00000000..66604ee0 --- /dev/null +++ b/app/src/features/accounts/tasks/tasks.test.ts @@ -0,0 +1,232 @@ +// Two-way task sync against mocks of Microsoft To Do and Google Tasks: a first sync both ways, each field changed on one +// side, a change in both places (the newer one wins and the other is logged), deletes, and reminders on To Do. +// The Upcoming list is the local side. +import { describe, expect, it } from 'vitest'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { addGoogleTasks, addTodo } from '../../../../../tests/mock-servers/tasks'; +import type { GTask } from '../../../../../tests/mock-servers/tasks'; +import { createFakeConnectors } from '../../connectors'; +import { googleTasks } from './google'; +import { toLocal } from './local'; +import { sameFields, syncTasks } from './sync'; +import { microsoftTodo } from './todo'; +import { EMPTY_STATE } from './types'; +import type { LocalTask, LocalTasks, TasksState } from './types'; + +/** A local list in memory, with the time of each change when a test wants to say it. */ +function memoryList( + initial: LocalTask[], + times: Record = {}, +): LocalTasks & { tasks: Map } { + const tasks = new Map(initial.map((task) => [task.id, task])); + return { + tasks, + read: () => [...tasks.values()].map((task) => ({ ...task })), + write(put, remove) { + for (const task of put) tasks.set(task.id, { ...task }); + for (const id of remove) tasks.delete(id); + }, + modifiedAt: (id) => times[id] ?? null, + }; +} + +const task = (id: string, title: string, rest: Partial = {}): LocalTask => ({ + id, + title, + done: false, + date: null, + time: null, + ...rest, +}); + +// checks-disable-next-line modifiability: one test group that shares its fixtures; split it when it grows again +describe('Google Tasks', () => { + function rig( + initial: GTask[] = [ + { + id: 'g1', + title: 'Read chapter 4', + status: 'needsAction', + due: '2026-10-20T00:00:00.000Z', + updated: '2026-10-01T10:00:00Z', + }, + ], + ) { + const server = new MockServer(); + const mock = addGoogleTasks(server, initial); + const { client } = createFakeConnectors({ connected: { google: 'sam@example.com' }, respond: server.respond }); + return { api: googleTasks(client), mock, server }; + } + + it('brings tasks over and sends new ones, then does nothing when nothing changed', async () => { + const { api, mock } = rig(); + const local = memoryList([task('mine', 'Buy graph paper')]); + const first = await syncTasks({ api, local, state: EMPTY_STATE, now: Date.parse('2026-10-02T09:00:00Z') }); + expect(first.result).toMatchObject({ pulled: 1, pushed: 1, removed: 0 }); + expect(local.tasks.get('google:g1')).toMatchObject({ title: 'Read chapter 4', date: '2026-10-20', done: false }); + expect([...mock.tasks.values()].map((one) => one.title).sort()).toEqual(['Buy graph paper', 'Read chapter 4']); + const second = await syncTasks({ api, local, state: first.state }); + expect(second.result).toEqual({ pulled: 0, pushed: 0, removed: 0, conflicts: 0 }); + }); + + it('copies a change made here, and a change made there', async () => { + const { api, mock } = rig(); + const local = memoryList([]); + let state = (await syncTasks({ api, local, state: EMPTY_STATE })).state; + // Ticked off here. + const pulled = local.tasks.get('google:g1'); + if (!pulled) throw new Error('not pulled'); + local.tasks.set(pulled.id, { ...pulled, done: true }); + state = (await syncTasks({ api, local, state })).state; + expect(mock.tasks.get('g1')?.status).toBe('completed'); + // Renamed and moved there. + const there = mock.tasks.get('g1'); + if (there) + Object.assign(there, { + title: 'Read chapter 5', + due: '2026-10-22T00:00:00.000Z', + updated: new Date().toISOString(), + }); + const done = await syncTasks({ api, local, state }); + expect(done.result.pulled).toBe(1); + expect(local.tasks.get('google:g1')).toMatchObject({ title: 'Read chapter 5', date: '2026-10-22', done: true }); + }); + + it('keeps the newer change when both sides changed, and logs the other', async () => { + const { api, mock } = rig(); + const local = memoryList([]); + const state = (await syncTasks({ api, local, state: EMPTY_STATE })).state; + const mine = local.tasks.get('google:g1'); + if (!mine) throw new Error('not pulled'); + const theirs = mock.tasks.get('g1'); + if (!theirs) throw new Error('missing'); + + // Here, 10:00. There, 11:00. There is newer. + local.tasks.set(mine.id, { ...mine, title: 'Local title' }); + Object.assign(theirs, { title: 'Remote title', updated: '2026-10-03T11:00:00Z' }); + const older = memoryList([...local.tasks.values()], { 'google:g1': Date.parse('2026-10-03T10:00:00Z') }); + const lost = await syncTasks({ api, local: older, state }); + expect(lost.result.conflicts).toBe(1); + expect(older.tasks.get('google:g1')?.title).toBe('Remote title'); + expect(lost.state.conflicts[0]).toMatchObject({ kept: 'there', other: { title: 'Local title' } }); + + // Here, 12:00. There, 11:00. Here is newer. + older.tasks.set('google:g1', { ...(older.tasks.get('google:g1') as LocalTask), title: 'Newest local' }); + Object.assign(theirs, { title: 'Remote again', updated: '2026-10-03T11:00:00Z' }); + const newer = memoryList([...older.tasks.values()], { 'google:g1': Date.parse('2026-10-03T12:00:00Z') }); + const won = await syncTasks({ api, local: newer, state: lost.state }); + expect(mock.tasks.get('g1')?.title).toBe('Newest local'); + expect(won.state.conflicts[0]).toMatchObject({ kept: 'here', other: { title: 'Remote again' } }); + }); + + it('deletes on one side only when the other has not changed, and brings a changed task back', async () => { + const { api, mock } = rig([ + { id: 'g1', title: 'A', status: 'needsAction', due: null, updated: '2026-10-01T10:00:00Z' }, + { id: 'g2', title: 'B', status: 'needsAction', due: null, updated: '2026-10-01T10:00:00Z' }, + ]); + const local = memoryList([]); + const state = (await syncTasks({ api, local, state: EMPTY_STATE })).state; + // Task A is deleted here. Task B is deleted here but changed there. + local.tasks.delete('google:g1'); + local.tasks.delete('google:g2'); + const b = mock.tasks.get('g2'); + if (b) Object.assign(b, { title: 'B changed', updated: new Date().toISOString() }); + const done = await syncTasks({ api, local, state }); + expect(mock.tasks.has('g1')).toBe(false); + expect(local.tasks.get('google:g2')?.title).toBe('B changed'); + expect(done.result).toMatchObject({ removed: 1, pulled: 1, conflicts: 1 }); + + // Deleted there: gone here when unchanged. + mock.tasks.delete('g2'); + const last = await syncTasks({ api, local, state: done.state }); + expect(local.tasks.has('google:g2')).toBe(false); + expect(last.result.removed).toBe(1); + }); +}); + +describe('Microsoft To Do', () => { + const zone = 'America/Los_Angeles'; + + it('syncs the due date and the reminder as the time of day', async () => { + const server = new MockServer(); + const mock = addTodo(server, [ + { + id: 't0', + title: 'Lab report', + status: 'notStarted', + dueDateTime: { dateTime: '2026-10-21T00:00:00.0000000', timeZone: 'UTC' }, + reminderDateTime: { dateTime: '2026-10-21T08:30:00.0000000', timeZone: zone }, + isReminderOn: true, + lastModifiedDateTime: '2026-10-01T10:00:00Z', + }, + ]); + const { client } = createFakeConnectors({ connected: { microsoft: 'sam@example.com' }, respond: server.respond }); + const api = microsoftTodo(client, zone); + const local = memoryList([task('mine', 'Call the lab', { date: '2026-10-25', time: '14:15' })]); + const first = await syncTasks({ api, local, state: EMPTY_STATE }); + expect(local.tasks.get('microsoft:t0')).toMatchObject({ date: '2026-10-21', time: '08:30' }); + const sent = [...mock.tasks.values()].find((one) => one.title === 'Call the lab'); + expect(sent).toMatchObject({ + isReminderOn: true, + dueDateTime: { dateTime: '2026-10-25T00:00:00.0000000' }, + reminderDateTime: { dateTime: '2026-10-25T14:15:00.0000000', timeZone: zone }, + }); + expect(server.requests.some((one) => one.headers.prefer === `outlook.timezone="${zone}"`)).toBe(true); + + // Clear the time here: the reminder goes off there. + const mine = local.tasks.get('mine'); + if (mine) local.tasks.set('mine', { ...mine, time: null }); + await syncTasks({ api, local, state: first.state }); + expect([...mock.tasks.values()].find((one) => one.title === 'Call the lab')?.isReminderOn).toBe(false); + }); + + it('reads tasks across result pages', async () => { + const server = new MockServer(); + addTodo( + server, + ['a', 'b', 'c', 'd', 'e'].map((id) => ({ + id, + title: id, + status: 'notStarted' as const, + lastModifiedDateTime: '2026-10-01T10:00:00Z', + })), + ); + const { client } = createFakeConnectors({ connected: { microsoft: 'sam@example.com' }, respond: server.respond }); + expect(await microsoftTodo(client, zone).list()).toHaveLength(5); + }); +}); + +describe('the pieces', () => { + it('compares tasks by what is synced', () => { + expect(sameFields(task('a', 'x'), task('b', 'x'))).toBe(true); + expect(sameFields(task('a', 'x'), task('a', 'x', { done: true }))).toBe(false); + }); + + it('reads an Upcoming item as a task', () => { + expect( + toLocal({ + id: 'u1', + title: 'Essay', + done: false, + due: { date: { year: 2026, month: 3, day: 5 }, time: { hour: 9, minute: 5 } }, + }), + ).toEqual({ id: 'u1', title: 'Essay', done: false, date: '2026-03-05', time: '09:05' }); + }); + + it('keeps no more than fifty conflicts', async () => { + const server = new MockServer(); + addGoogleTasks(server, []); + const { client } = createFakeConnectors({ connected: { google: 'a@example.com' }, respond: server.respond }); + const state: TasksState = { + ...EMPTY_STATE, + conflicts: Array.from({ length: 50 }, (_, at) => ({ + at: String(at), + title: 't', + kept: 'here' as const, + other: task('x', 'y'), + })), + }; + const done = await syncTasks({ api: googleTasks(client), local: memoryList([]), state }); + expect(done.state.conflicts).toHaveLength(50); + }); +}); diff --git a/app/src/features/accounts/tasks/todo.ts b/app/src/features/accounts/tasks/todo.ts new file mode 100644 index 00000000..5fbdcab1 --- /dev/null +++ b/app/src/features/accounts/tasks/todo.ts @@ -0,0 +1,117 @@ +// Microsoft To Do through Graph: the default list ("Tasks"), its tasks across result pages, and creating, changing, and +// deleting one. A due date is a date. The time of day is the task's reminder: set a time here and To Do reminds you at +// that time, and a reminder set in To Do shows here as the time. Times are asked for in the viewer's own zone. +import { ConnectorHttpError, requestConnector } from '../../connectors'; +import type { ConnectorsClient } from '../../connectors'; +import type { RemoteTask, TaskFields, TasksApi } from './types'; + +const GRAPH = 'https://graph.microsoft.com/v1.0/me/todo/lists'; +const ACCESS = ['tasksWrite'] as const; +const MAX_PAGES = 30; + +interface TodoTask { + id: string; + title?: string; + status?: string; + dueDateTime?: { dateTime: string; timeZone: string } | null; + reminderDateTime?: { dateTime: string; timeZone: string } | null; + isReminderOn?: boolean; + lastModifiedDateTime?: string; +} + +interface Page { + value?: T[]; + '@odata.nextLink'?: string; +} + +function toRemote(task: TodoTask): RemoteTask { + const reminder = task.isReminderOn && task.reminderDateTime ? task.reminderDateTime.dateTime : null; + return { + id: task.id, + title: task.title ?? '', + done: task.status === 'completed', + date: task.dueDateTime ? task.dueDateTime.dateTime.slice(0, 10) : reminder ? reminder.slice(0, 10) : null, + time: reminder ? reminder.slice(11, 16) : null, + updated: task.lastModifiedDateTime ?? new Date(0).toISOString(), + }; +} + +function body(fields: TaskFields, zone: string): Record { + return { + title: fields.title, + status: fields.done ? 'completed' : 'notStarted', + dueDateTime: fields.date ? { dateTime: `${fields.date}T00:00:00.0000000`, timeZone: 'UTC' } : null, + isReminderOn: Boolean(fields.date && fields.time && !fields.done), + reminderDateTime: + fields.date && fields.time ? { dateTime: `${fields.date}T${fields.time}:00.0000000`, timeZone: zone } : null, + }; +} + +export function microsoftTodo(client: ConnectorsClient, zone: string): TasksApi { + const headers = { Prefer: `outlook.timezone="${zone}"` }; + let listUrl: string | null = null; + + async function tasksUrl(): Promise { + if (listUrl) return listUrl; + const lists = await requestConnector>( + 'microsoft', + [...ACCESS], + { url: GRAPH }, + client, + ); + const all = lists.data?.value ?? []; + const chosen = all.find((one) => one.wellknownListName === 'defaultList') ?? all[0]; + if (!chosen) throw new ConnectorHttpError(404, JSON.stringify({ error: { message: 'No task list was found.' } })); + listUrl = `${GRAPH}/${encodeURIComponent(chosen.id)}/tasks`; + return listUrl; + } + + const send = async (method: 'POST' | 'PATCH', url: string, fields: TaskFields): Promise => { + const reply = await client.request('microsoft', [...ACCESS], { + method, + url, + headers, + contentType: 'application/json', + body: JSON.stringify(body(fields, zone)), + }); + if (reply.status < 200 || reply.status >= 300) throw new ConnectorHttpError(reply.status, reply.body); + return toRemote(JSON.parse(reply.body) as TodoTask); + }; + + return { + service: 'microsoft', + label: 'Microsoft To Do', + async list() { + const found: RemoteTask[] = []; + let url: string | null = await tasksUrl(); + let query: Record | undefined = { $top: 100 }; + for (let page = 0; url && page < MAX_PAGES; page += 1) { + const reply: { data?: Page } = await requestConnector>( + 'microsoft', + [...ACCESS], + { url, headers, ...(query ? { query } : {}) }, + client, + ); + for (const task of reply.data?.value ?? []) found.push(toRemote(task)); + url = reply.data?.['@odata.nextLink'] ?? null; + query = undefined; + } + return found; + }, + async create(fields) { + return send('POST', await tasksUrl(), fields); + }, + async update(id, fields) { + return send('PATCH', `${await tasksUrl()}/${encodeURIComponent(id)}`, fields); + }, + async remove(id) { + const reply = await client.request('microsoft', [...ACCESS], { + method: 'DELETE', + url: `${await tasksUrl()}/${encodeURIComponent(id)}`, + }); + if ((reply.status < 200 || reply.status >= 300) && reply.status !== 404) { + throw new ConnectorHttpError(reply.status, reply.body); + } + }, + }; +} diff --git a/app/src/features/accounts/tasks/types.ts b/app/src/features/accounts/tasks/types.ts new file mode 100644 index 00000000..a209d260 --- /dev/null +++ b/app/src/features/accounts/tasks/types.ts @@ -0,0 +1,73 @@ +// Two-way sync of tasks with Microsoft To Do and Google Tasks: the fields that are kept in step, in one shape for both +// services and for the Upcoming list. + +/** What is kept in step: the title, the box, the due date, and the time of day (To Do's reminder). */ +export interface TaskFields { + title: string; + done: boolean; + /** The due date as YYYY-MM-DD, or null. */ + date: string | null; + /** The time of day as HH:MM in the viewer's zone, or null. On To Do this is the reminder. */ + time: string | null; +} + +export interface LocalTask extends TaskFields { + id: string; +} + +export interface RemoteTask extends TaskFields { + id: string; + /** When the service last changed it, as an ISO time. */ + updated: string; +} + +export interface TasksApi { + service: 'microsoft' | 'google'; + label: string; + /** Every task of the default list, finished ones too. */ + list(): Promise; + create(fields: TaskFields): Promise; + update(id: string, fields: TaskFields): Promise; + remove(id: string): Promise; +} + +/** The person's own tasks on this device. */ +export interface LocalTasks { + read(): LocalTask[]; + /** Adds or replaces tasks by ID, and removes the others named. */ + write(put: readonly LocalTask[], remove: readonly string[]): void; + /** When the task was last changed here, if the list knows. */ + modifiedAt?(id: string): number | null; +} + +/** A task changed in both places between two syncs. The newer change was kept. */ +export interface Conflict { + at: string; + title: string; + kept: 'here' | 'there'; + /** What the other copy said, so nothing is lost without a record. */ + other: TaskFields; +} + +export interface Link { + remote: string; + /** What both sides held after the last sync. */ + base: TaskFields; +} + +export interface TasksState { + links: Record; + conflicts: Conflict[]; + lastSync: string | null; + /** When this device's change was first noticed, for a task that has not synced yet. */ + seen: Record; +} + +export const EMPTY_STATE: TasksState = { links: {}, conflicts: [], lastSync: null, seen: {} }; + +export interface SyncResult { + pulled: number; + pushed: number; + removed: number; + conflicts: number; +} diff --git a/app/src/features/accounts/ui/accounts.module.css b/app/src/features/accounts/ui/accounts.module.css new file mode 100644 index 00000000..67a36300 --- /dev/null +++ b/app/src/features/accounts/ui/accounts.module.css @@ -0,0 +1,60 @@ +/* The small dialogs of the account features: a list to choose from, a line to type, and a progress note. Tokens + only (BRAND.md section 14). */ +@layer components { + .list { + display: grid; + gap: var(--space-2); + margin: 0; + padding: 0; + list-style: none; + max-block-size: 50vh; + overflow-y: auto; + } + + .choice { + display: grid; + gap: var(--space-1); + inline-size: 100%; + min-block-size: var(--size-target-pointer); + padding: var(--space-2) var(--space-3); + border: 1px solid var(--color-border-default); + border-radius: var(--radius-md); + background: var(--color-surface-raised); + color: var(--color-text-primary); + font: inherit; + text-align: start; + cursor: pointer; + } + + :global(:root[data-density='touch']) .choice { + min-block-size: var(--size-target-touch); + } + + .choice[aria-checked='true'] { + border-color: var(--color-accent-primary); + background: var(--color-surface-selected); + } + + .choice:focus-visible { + outline: var(--size-focus-ring-width) solid var(--color-focus-ring); + outline-offset: var(--size-focus-ring-offset); + } + + .detail, + .help { + margin: 0; + color: var(--color-text-secondary); + font-size: var(--type-small-size); + line-height: var(--type-small-line); + } + + .empty { + margin: 0; + color: var(--color-text-secondary); + } + + .stack { + display: grid; + gap: var(--space-3); + } +} diff --git a/app/src/features/accounts/ui/dialog.tsx b/app/src/features/accounts/ui/dialog.tsx new file mode 100644 index 00000000..5e8fc20f --- /dev/null +++ b/app/src/features/accounts/ui/dialog.tsx @@ -0,0 +1,24 @@ +// Mounts a React tree for a dialog that a command opens, and unmounts it when it closes. The commands live in the +// start-up bundle, and the dialogs load on first use. +import type { ReactNode } from 'react'; +import { createRoot } from 'react-dom/client'; + +/** Renders `render(close)` into the page. Resolves when `close` runs. */ +export function openDialog(render: (close: () => void) => ReactNode): Promise { + const host = document.body.appendChild(document.createElement('div')); + const root = createRoot(host); + return new Promise((resolve) => { + let done = false; + const close = () => { + if (done) return; + done = true; + // Unmounting first returns focus before the caller's code after `await openDialog(...)` runs. + queueMicrotask(() => { + root.unmount(); + host.remove(); + }); + resolve(); + }; + root.render(render(close)); + }); +} diff --git a/app/src/features/accounts/ui/pickFile.ts b/app/src/features/accounts/ui/pickFile.ts new file mode 100644 index 00000000..fa75edc4 --- /dev/null +++ b/app/src/features/accounts/ui/pickFile.ts @@ -0,0 +1,19 @@ +// Asks for one file with the browser's own picker, which WebView2 shows as the Windows Open dialog. Resolves with the +// file, or null when the person cancels. The file's bytes stay in the app: nothing is sent anywhere by this. + +export function pickFile(accept: string): Promise { + return new Promise((resolve) => { + const input = document.createElement('input'); + input.type = 'file'; + input.accept = accept; + input.style.display = 'none'; + const done = (file: File | null) => { + input.remove(); + resolve(file); + }; + input.addEventListener('change', () => done(input.files?.[0] ?? null), { once: true }); + input.addEventListener('cancel', () => done(null), { once: true }); + document.body.appendChild(input); + input.click(); + }); +} diff --git a/app/src/features/accounts/ui/prompts.test.tsx b/app/src/features/accounts/ui/prompts.test.tsx new file mode 100644 index 00000000..f8d82a68 --- /dev/null +++ b/app/src/features/accounts/ui/prompts.test.tsx @@ -0,0 +1,61 @@ +// The two questions the account commands ask: which one, and what to type. They are dialogs that resolve with the +// answer or null, and they work from the keyboard. +import { cleanup, fireEvent, screen, waitFor } from '@testing-library/react'; +import { afterEach, describe, expect, it } from 'vitest'; +import { expectNoAxeViolations } from '../../../test'; +import { askText, pickOne } from './prompts'; + +afterEach(cleanup); + +const choices = [ + { id: 'a', label: 'Standup', detail: 'Today, 10:00 AM', value: 1 }, + { id: 'b', label: 'Design review', detail: 'Tomorrow, 2:00 PM', value: 2 }, +]; + +describe('pickOne', () => { + it('resolves with the value of the chosen item, and the first item is chosen at the start', async () => { + const first = pickOne({ title: 'Which meeting?', choices, confirmLabel: 'Make the note' }); + const dialog = await screen.findByRole('dialog', { name: 'Which meeting?' }); + expect(screen.getByRole('radio', { name: /Standup/ }).getAttribute('aria-checked')).toBe('true'); + await expectNoAxeViolations(dialog); + fireEvent.click(screen.getByRole('button', { name: 'Make the note' })); + expect(await first).toBe(1); + + const second = pickOne({ title: 'Which meeting?', choices, confirmLabel: 'Make the note' }); + await screen.findByRole('dialog', { name: 'Which meeting?' }); + fireEvent.keyDown(screen.getByRole('radio', { name: /Standup/ }), { key: 'ArrowDown' }); + await waitFor(() => + expect(screen.getByRole('radio', { name: /Design review/ }).getAttribute('aria-checked')).toBe('true'), + ); + fireEvent.click(screen.getByRole('button', { name: 'Make the note' })); + expect(await second).toBe(2); + }); + + it('resolves null on Cancel, and says so instead of offering a choice when there is nothing to choose', async () => { + const none = pickOne({ + title: 'Which meeting?', + choices: [], + confirmLabel: 'Make the note', + empty: 'Nothing here.', + }); + await screen.findByText('Nothing here.'); + expect(screen.queryByRole('button', { name: 'Make the note' })).toBeNull(); + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })); + expect(await none).toBeNull(); + }); +}); + +describe('askText', () => { + it('resolves with the trimmed text, and null when it is left empty', async () => { + const typed = askText({ title: 'Name it', label: 'Name', confirmLabel: 'Save' }); + const field = await screen.findByRole('textbox', { name: 'Name' }); + fireEvent.change(field, { target: { value: ' Biology ' } }); + fireEvent.click(screen.getByRole('button', { name: 'Save' })); + expect(await typed).toBe('Biology'); + + const empty = askText({ title: 'Name it', label: 'Name', confirmLabel: 'Save' }); + await screen.findByRole('textbox', { name: 'Name' }); + fireEvent.click(screen.getByRole('button', { name: 'Save' })); + expect(await empty).toBeNull(); + }); +}); diff --git a/app/src/features/accounts/ui/prompts.tsx b/app/src/features/accounts/ui/prompts.tsx new file mode 100644 index 00000000..89586db0 --- /dev/null +++ b/app/src/features/accounts/ui/prompts.tsx @@ -0,0 +1,110 @@ +// The two questions the account commands ask: which one of these, and what to type. Each is a dialog that resolves +// with the answer, or null when the person cancels. They are keyboard-first: arrow keys move through a list, Enter +// chooses, and Escape cancels. +import { useState } from 'react'; +import { t } from '../../../strings/t'; +import { Dialog, RadioCard, RadioGroup, TextField } from '../../../ui'; +import styles from './accounts.module.css'; +import { openDialog } from './dialog'; + +export interface Choice { + id: string; + label: string; + detail?: string; + value: T; +} + +export interface PickOptions { + title: string; + description?: string; + choices: readonly Choice[]; + /** What the primary button says. */ + confirmLabel: string; + /** Shown instead of the list when there is nothing to choose. */ + empty?: string; +} + +/** Asks the person to choose one of the items. Resolves with its value, or null. */ +export function pickOne(options: PickOptions): Promise { + let answer: T | null = null; + return openDialog((close) => { + function Picker() { + const [chosen, setChosen] = useState(options.choices[0]?.id ?? ''); + const finish = () => { + answer = options.choices.find((choice) => choice.id === chosen)?.value ?? null; + close(); + }; + return ( + 0 + ? [{ id: 'choose', label: options.confirmLabel, variant: 'primary' as const, onPress: finish }] + : []), + ]} + > + {options.choices.length === 0 ? ( +

    {options.empty ?? ''}

    + ) : ( + + {options.choices.map((choice) => ( + + ))} + + )} +
    + ); + } + return ; + }).then(() => answer); +} + +export interface AskOptions { + title: string; + description?: string; + label: string; + help?: string; + initial?: string; + confirmLabel: string; + /** Hides what is typed. */ + secret?: boolean; +} + +/** Asks for one line of text. Resolves with the trimmed text, or null when canceled or left empty. */ +export function askText(options: AskOptions): Promise { + let answer: string | null = null; + return openDialog((close) => { + function Asker() { + const [value, setValue] = useState(options.initial ?? ''); + const finish = () => { + answer = value.trim() === '' ? null : value.trim(); + close(); + }; + return ( + + + + ); + } + return ; + }).then(() => answer); +} diff --git a/app/src/features/accounts/youtube/commands.ts b/app/src/features/accounts/youtube/commands.ts new file mode 100644 index 00000000..fc52c30f --- /dev/null +++ b/app/src/features/accounts/youtube/commands.ts @@ -0,0 +1,72 @@ +// Upload to YouTube: choose private or unlisted, name the video, and send it. The page's video is made by the caller (the +// recording is the page lane's), into the transfer folder, and removed afterwards whatever happens. Nothing goes to +// YouTube until the person presses the last button, and nothing is ever made public from here. +import { t } from '../../../strings/t'; +import { announce, showToast } from '../../../ui'; +import { connectors } from '../../connectors'; +import { accountsHost } from '../host'; +import { attempt, isConnected, tellNotConnected } from '../run'; +import { askText, pickOne } from '../ui/prompts'; +import { uploadVideo } from './upload'; +import type { YoutubePrivacy } from './upload'; + +export interface UploadRequest { + /** A title to start from. */ + title: string; + /** Makes the video under this name in the transfer folder and answers its size. */ + makeVideo(name: string): Promise; +} + +export async function uploadRecording(request: UploadRequest): Promise { + if (!(await isConnected('google'))) return tellNotConnected('YouTube'); + const privacy = await pickOne({ + title: t('accounts.youtube.privacy.title'), + description: t('accounts.youtube.privacy.description'), + confirmLabel: t('accounts.youtube.privacy.confirm'), + choices: [ + { + id: 'private', + label: t('accounts.youtube.privacy.private'), + detail: t('accounts.youtube.privacy.privateDetail'), + value: 'private', + }, + { + id: 'unlisted', + label: t('accounts.youtube.privacy.unlisted'), + detail: t('accounts.youtube.privacy.unlistedDetail'), + value: 'unlisted', + }, + ], + }); + if (!privacy) return; + const title = await askText({ + title: t('accounts.youtube.name.title'), + description: t('accounts.youtube.name.description'), + label: t('accounts.youtube.name.label'), + initial: request.title, + confirmLabel: t('accounts.youtube.name.confirm'), + }); + if (!title) return; + const host = accountsHost(); + const file = `youtube-${Date.now()}.mp4`; + await attempt('YouTube', async () => { + try { + announce(t('accounts.youtube.making')); + const size = await request.makeVideo(file); + showToast({ id: 'youtube-upload', message: t('accounts.youtube.sending') }); + const done = await uploadVideo({ client: connectors(), host }, { file, size, title, privacy }, (share) => + announce(t('accounts.youtube.progress', { percent: Math.round(share * 100) })), + ); + const message = t(privacy === 'private' ? 'accounts.youtube.donePrivate' : 'accounts.youtube.doneUnlisted'); + showToast({ id: 'youtube-upload', message }); + announce(message); + try { + await navigator.clipboard.writeText(done.url); + } catch { + // The link is in YouTube Studio either way. + } + } finally { + await host.tempRemove(file).catch(() => undefined); + } + }); +} diff --git a/app/src/features/accounts/youtube/upload.ts b/app/src/features/accounts/youtube/upload.ts new file mode 100644 index 00000000..3659f11d --- /dev/null +++ b/app/src/features/accounts/youtube/upload.ts @@ -0,0 +1,85 @@ +// Uploading a video to YouTube the way YouTube's resumable upload works: one request that says what the video is and +// answers with an upload address, then the file in pieces, each with the range it covers. YouTube answers 308 while it +// wants more and 200 or 201 with the video's ID when it has it all. A video is only ever private or unlisted from here. + +import { ConnectorHttpError } from '../../connectors'; +import type { ConnectorResponse, ConnectorsClient } from '../../connectors'; +import type { AccountsHost } from '../host'; + +export type YoutubePrivacy = 'private' | 'unlisted'; + +/** A multiple of 256 KiB, which YouTube requires of every piece but the last. */ +export const CHUNK = 8 * 1024 * 1024; + +const START = 'https://www.googleapis.com/upload/youtube/v3/videos'; +const ACCESS = ['youtubeUpload'] as const; + +export interface UploadInput { + /** The file in the transfer folder. */ + file: string; + size: number; + title: string; + description?: string; + privacy: YoutubePrivacy; +} + +export interface Uploaded { + id: string; + url: string; +} + +function failure(response: ConnectorResponse): ConnectorHttpError { + return new ConnectorHttpError(response.status, response.body); +} + +/** Sends one piece, and once more when YouTube was busy. */ +async function sendPiece( + host: AccountsHost, + address: string, + input: UploadInput, + start: number, +): Promise { + const length = Math.min(CHUNK, input.size - start); + const request = { + method: 'PUT' as const, + url: address, + headers: { 'Content-Range': `bytes ${start}-${start + length - 1}/${input.size}` }, + }; + const parts = [{ kind: 'file' as const, file: input.file, offset: start, length }]; + const first = await host.upload('google', ACCESS, request, parts, 'video/mp4'); + if (first.status < 500) return first; + return host.upload('google', ACCESS, request, parts, 'video/mp4'); +} + +export async function uploadVideo( + deps: { client: ConnectorsClient; host: AccountsHost }, + input: UploadInput, + progress: (share: number) => void = () => undefined, +): Promise { + if (input.size <= 0) throw new ConnectorHttpError(400, JSON.stringify({ error: 'The video is empty.' })); + const started = await deps.client.request('google', [...ACCESS], { + method: 'POST', + url: `${START}?uploadType=resumable&part=snippet,status`, + headers: { 'X-Upload-Content-Type': 'video/mp4', 'X-Upload-Content-Length': String(input.size) }, + contentType: 'application/json; charset=UTF-8', + body: JSON.stringify({ + snippet: { title: input.title.slice(0, 100), description: input.description ?? '' }, + status: { privacyStatus: input.privacy, selfDeclaredMadeForKids: false }, + }), + }); + if (started.status < 200 || started.status >= 300) throw failure(started); + const address = started.location; + if (!address) throw new ConnectorHttpError(502, ''); + let at = 0; + for (;;) { + const answer = await sendPiece(deps.host, address, input, at); + at += Math.min(CHUNK, input.size - at); + progress(Math.min(1, at / input.size)); + if (answer.status === 200 || answer.status === 201) { + const id = (JSON.parse(answer.body || '{}') as { id?: string }).id; + if (!id) throw new ConnectorHttpError(502, answer.body); + return { id, url: `https://www.youtube.com/watch?v=${id}` }; + } + if (answer.status !== 308 || at >= input.size) throw failure(answer); + } +} diff --git a/app/src/features/accounts/youtube/youtube.test.ts b/app/src/features/accounts/youtube/youtube.test.ts new file mode 100644 index 00000000..6343c1fa --- /dev/null +++ b/app/src/features/accounts/youtube/youtube.test.ts @@ -0,0 +1,62 @@ +// Upload to YouTube against a mock of its resumable upload: the start request says private or unlisted, the file goes in +// pieces with a range each, a busy answer is tried once more, and a refusal is said. +import { describe, expect, it } from 'vitest'; +import { MockServer } from '../../../../../tests/mock-servers/server'; +import { addYoutubeUpload } from '../../../../../tests/mock-servers/video'; +import { ConnectorHttpError, createFakeConnectors } from '../../connectors'; +import { createFakeAccountsHost } from '../fakeHost'; +import { CHUNK, uploadVideo } from './upload'; + +async function rig(server: MockServer, size: number) { + const connectors = createFakeConnectors({ connected: { google: 'sam@example.com' }, respond: server.respond }); + const host = createFakeAccountsHost(() => connectors.client); + await host.tempWrite('movie.mp4', { kind: 'text', text: 'v'.repeat(size) }); + return { client: connectors.client, host }; +} + +describe('uploading a video', () => { + it('asks for private or unlisted and sends a small file in one piece', async () => { + const server = new MockServer(); + const state = addYoutubeUpload(server); + const { client, host } = await rig(server, 1000); + const done = await uploadVideo( + { client, host }, + { file: 'movie.mp4', size: 1000, title: 'Cell lecture', privacy: 'unlisted' }, + ); + expect(done).toEqual({ id: 'vid12345678', url: 'https://www.youtube.com/watch?v=vid12345678' }); + expect(state.started).toEqual([{ title: 'Cell lecture', privacy: 'unlisted', length: '1000' }]); + expect(state.ranges).toEqual(['bytes 0-999/1000']); + }); + + it('sends a large file in pieces and reports the share done', async () => { + const server = new MockServer(); + const state = addYoutubeUpload(server); + const size = CHUNK + 500; + const { client, host } = await rig(server, size); + const shares: number[] = []; + await uploadVideo({ client, host }, { file: 'movie.mp4', size, title: 'Long', privacy: 'private' }, (share) => + shares.push(share), + ); + expect(state.ranges).toEqual([`bytes 0-${CHUNK - 1}/${size}`, `bytes ${CHUNK}-${size - 1}/${size}`]); + expect(shares[shares.length - 1]).toBe(1); + }); + + it('tries a piece again when YouTube was busy, and says so when it refuses', async () => { + const server = new MockServer(); + const state = addYoutubeUpload(server); + state.failOnce = 503; + const { client, host } = await rig(server, 100); + await uploadVideo({ client, host }, { file: 'movie.mp4', size: 100, title: 'x', privacy: 'private' }); + expect(state.ranges).toHaveLength(1); + + const refusing = new MockServer(); + refusing.route('POST www.googleapis.com/upload/youtube/v3/videos', () => ({ + status: 403, + json: { error: { message: 'The quota was used up.' } }, + })); + const other = await rig(refusing, 10); + await expect( + uploadVideo(other, { file: 'movie.mp4', size: 10, title: 'x', privacy: 'private' }), + ).rejects.toBeInstanceOf(ConnectorHttpError); + }); +}); diff --git a/app/src/features/audio/flags.ts b/app/src/features/audio/flags.ts new file mode 100644 index 00000000..2739a61d --- /dev/null +++ b/app/src/features/audio/flags.ts @@ -0,0 +1,34 @@ +// Phase 9's flags (audio recording). Each feature is on once it works end to end in the app; the rest stay off, +// hidden rather than disabled. app/flags.ts joins these to the other flags, so they load at start-up. +import type { FlagDef, FlagId } from '../../app/flags'; + +export type AudioFlagId = Extract; + +const ISSUES = 'https://github.com/XrxcGH/OpenNote/issues?q=label%3Aflag%3A'; +const on = { dev: true, nightly: true, beta: true, stable: true }; + +const flag = (id: AudioFlagId, description: string, enabled: FlagDef['enabled']): FlagDef => ({ + id, + description, + issue: `${ISSUES}${encodeURIComponent(id)}`, + enabled, +}); + +export const AUDIO_FLAGS: readonly FlagDef[] = [ + flag('audio.record', 'Recording from the microphone, the recording block, and playback.', on), + flag('audio.stamps', 'Time stamps on typed text, and tapping a word to hear it.', on), + flag('audio.flags', 'Flags that mark moments of a recording.', on), + flag('audio.trim', 'Trimming silence and removing parts of a recording.', on), + flag('audio.systemAudio', 'Recording the PC’s sound along with the microphone, for meetings.', on), + flag('audio.meetingPrompt', 'The setting and prompt that offer to record when another app uses the microphone.', on), + flag('audio.enhance', 'Enhance voice on a copy of a recording, and switching between the two.', on), + flag('audio.storage', 'The list of what recordings take, with Compress and Remove audio.', on), + flag('audio.snap', 'Snapping the screen, a window, or a region onto the page while recording.', on), + flag('audio.import', 'Audio and video files dropped onto a page become recordings.', on), + flag('audio.export', 'Saving a recording as an audio file.', on), + flag('transcripts.block', 'The transcript of a recording, with a summary, times, and click to listen.', on), + flag('transcripts.speakers', 'Speaker labels in a transcript, and renaming them.', on), + flag('transcripts.notes', 'Copying transcript lines into the notes as a quote with a time link.', on), + flag('transcripts.actions', 'Finding action items and making chapters from a transcript.', on), + flag('transcripts.recap', 'Copying a meeting recap.', on), +]; diff --git a/app/src/features/canvas/CanvasView.tsx b/app/src/features/canvas/CanvasView.tsx new file mode 100644 index 00000000..55da6a28 --- /dev/null +++ b/app/src/features/canvas/CanvasView.tsx @@ -0,0 +1,494 @@ +// checks-disable-file modifiability: one dialog whose parts share its state; split it when it grows again +// The canvas of the open page: an open board of cards (pages, notes, images, PDFs, web addresses, and groups) +// joined by labeled arrows. Cards move by dragging or with the arrow keys, Enter opens one, and the Connect form +// makes arrows without a pointer. The board is saved in the page, so it goes wherever the notebook goes. +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import type { KeyboardEvent, PointerEvent } from 'react'; +import { useLocation } from '../../app/location'; +import { commandContext } from '../../commands/registry'; +import { newId } from '../../editor/ids'; +import type { OpenPage } from '../../services/pages/types'; +import type { PageSuggestion } from '../../services/search/types'; +import type { NodeId } from '../../services/notes/types'; +import type { OverlayProps } from '../../shell/commandbar/overlays'; +import { t } from '../../strings/t'; +import { Button, Dialog, TextField, announce, showToast } from '../../ui'; +import { maybeSearchClient, openAndReveal } from '../search'; +import styles from './canvas.module.css'; +import { + CARD_KINDS, + CARD_SIZE, + align, + cardKind, + cardText, + center, + edgePoint, + emptyCanvas, + exportJson, + importJson, + pageOf, + place, + readCanvas, + removeCards, + viewPatch, +} from './model'; +import type { Alignment, Canvas, CanvasNode, CardKind } from './model'; + +const WEB = /^https?:\/\//i; + +function Pick({ + label, + value, + options, + onChange, +}: { + label: string; + value: string; + options: { value: string; text: string }[]; + onChange(value: string): void; +}) { + return ( + + ); +} + +export default function CanvasView({ onClose }: OverlayProps) { + const here = useLocation(); + const pageId = here.view === 'workspace' ? here.pageId : null; + const [canvas, setCanvas] = useState(emptyCanvas); + const [ready, setReady] = useState(false); + const [selection, setSelection] = useState([]); + const [titles, setTitles] = useState>(new Map()); + const [kind, setKind] = useState('note'); + const [entry, setEntry] = useState(''); + const [found, setFound] = useState([]); + const [editing, setEditing] = useState(null); + const [from, setFrom] = useState(''); + const [to, setTo] = useState(''); + const [label, setLabel] = useState(''); + const [json, setJson] = useState(''); + const open = useRef(null); + const latest = useRef(canvas); + const timer = useRef | null>(null); + + const send = useCallback(() => { + if (timer.current) clearTimeout(timer.current); + timer.current = null; + void open.current?.send({ edits: [{ edit: 'setPage', view: viewPatch(latest.current) }] }).catch(() => undefined); + }, []); + const change = useCallback( + (next: Canvas, now = true) => { + latest.current = next; + setCanvas(next); + if (timer.current) clearTimeout(timer.current); + if (now) send(); + else timer.current = setTimeout(send, 400); + }, + [send], + ); + + useEffect(() => { + if (!pageId) return; + let live = true; + void commandContext('palette') + .platform.pages.open(pageId, { viewport: null }) + .then((page) => { + if (!live) return void page.close(); + open.current = page; + const loaded = readCanvas(page.initial.view); + latest.current = loaded; + setCanvas(loaded); + setReady(true); + }) + .catch(() => undefined); + return () => { + live = false; + if (timer.current) send(); + void open.current?.close(); + open.current = null; + }; + }, [pageId, send]); + + useEffect(() => { + const { notes } = commandContext('palette'); + const ids = canvas.nodes.map(pageOf).filter((id): id is string => id !== null && !titles.has(id)); + if (ids.length === 0) return; + void Promise.all(ids.map((id) => notes.get(id as NodeId))).then((nodes) => + setTitles( + (was) => new Map([...was, ...nodes.flatMap((node, at) => (node ? [[ids[at], node.title] as const] : []))]), + ), + ); + }, [canvas.nodes, titles]); + + useEffect(() => { + const client = maybeSearchClient(); + if (kind !== 'page' || entry.trim() === '' || !client) return; + let live = true; + const timeout = setTimeout( + () => + void client + .suggestPages(entry, 5) + .then((list) => live && setFound(list)) + .catch(() => undefined), + 150, + ); + return () => { + live = false; + clearTimeout(timeout); + }; + }, [kind, entry]); + + const suggestions = kind === 'page' && entry.trim() !== '' ? found : []; + const add = (node: Pick & Partial) => { + const created: CanvasNode = { id: newId(), ...place(canvas), ...CARD_SIZE, ...node }; + change({ ...canvas, nodes: [...canvas.nodes, created] }); + setEntry(''); + setFound([]); + announce(t('qolSearch.canvas.added')); + }; + const addFromEntry = () => { + const value = entry.trim(); + if (kind === 'note') add({ type: 'text', text: value || t('qolSearch.canvas.newNote') }); + else if (kind === 'group') + add({ type: 'group', label: value || t('qolSearch.canvas.newGroup'), width: 360, height: 220 }); + else if (kind === 'web' && value) add({ type: 'link', url: WEB.test(value) ? value : `https://${value}` }); + else if ((kind === 'image' || kind === 'pdf') && value) add({ type: 'file', file: value }); + }; + const pick = (page: PageSuggestion) => add({ type: 'file', file: `opennote://page/${page.page}`, label: page.title }); + + const move = (id: string, dx: number, dy: number, now = true) => + change( + { + ...canvas, + nodes: canvas.nodes.map((node) => + node.id === id ? { ...node, x: Math.round(node.x + dx), y: Math.round(node.y + dy) } : node, + ), + }, + now, + ); + const drag = useRef<{ id: string; x: number; y: number } | null>(null); + const onDown = (event: PointerEvent, node: CanvasNode) => { + if ((event.target as HTMLElement).closest('button, textarea, input')) return; + drag.current = { id: node.id, x: event.clientX, y: event.clientY }; + event.currentTarget.setPointerCapture(event.pointerId); + setSelection((was) => + event.shiftKey || event.ctrlKey + ? was.includes(node.id) + ? was.filter((id) => id !== node.id) + : [...was, node.id] + : [node.id], + ); + }; + const onMove = (event: PointerEvent) => { + const d = drag.current; + if (!d) return; + move(d.id, event.clientX - d.x, event.clientY - d.y, false); + d.x = event.clientX; + d.y = event.clientY; + }; + const onUp = () => { + if (drag.current) send(); + drag.current = null; + }; + + const openCard = (node: CanvasNode) => { + const target = cardKind(node); + const page = pageOf(node); + if (page) { + onClose(); + void openAndReveal(commandContext('palette').notes, page, null); + } else if (node.url && WEB.test(node.url)) { + void commandContext('palette').platform.shell.openExternal({ kind: 'link', url: node.url }); + } else if (target === 'image' || target === 'pdf') { + const address = node.file ?? ''; + if (WEB.test(address)) void commandContext('palette').platform.shell.openExternal({ kind: 'link', url: address }); + } + }; + const onKey = (event: KeyboardEvent, node: CanvasNode) => { + if ((event.target as HTMLElement).closest('textarea, input')) return; + const step = event.shiftKey ? 50 : 10; + const arrows: Record = { + ArrowLeft: [-step, 0], + ArrowRight: [step, 0], + ArrowUp: [0, -step], + ArrowDown: [0, step], + }; + const delta = arrows[event.key]; + if (delta) { + event.preventDefault(); + move(node.id, delta[0], delta[1], false); + } else if (event.key === 'Enter') openCard(node); + else if (event.key === ' ') { + event.preventDefault(); + setSelection((was) => (was.includes(node.id) ? was.filter((id) => id !== node.id) : [...was, node.id])); + } else if (event.key === 'Delete') { + change(removeCards(canvas, [node.id])); + announce(t('qolSearch.canvas.removed')); + } + }; + const editText = (node: CanvasNode, value: string) => + change( + { + ...canvas, + nodes: canvas.nodes.map((n) => + n.id !== node.id + ? n + : n.type === 'text' + ? { ...n, text: value } + : n.type === 'link' + ? { ...n, url: value } + : { ...n, label: value }, + ), + }, + false, + ); + const names = useMemo( + () => + new Map( + canvas.nodes.map((node) => [node.id, cardText(node, titles) || t(`qolSearch.canvas.kinds.${cardKind(node)}`)]), + ), + [canvas.nodes, titles], + ); + + const connect = () => { + if (!from || !to || from === to) return; + change({ + ...canvas, + edges: [ + ...canvas.edges, + { id: newId(), fromNode: from, toNode: to, toEnd: 'arrow', ...(label.trim() ? { label: label.trim() } : {}) }, + ], + }); + setLabel(''); + announce(t('qolSearch.canvas.connected', { from: names.get(from) ?? '', to: names.get(to) ?? '' })); + }; + const lineUp = (mode: Alignment) => change(align(canvas, selection, mode)); + + const width = Math.max(900, ...canvas.nodes.map((node) => node.x + node.width + 80)); + const height = Math.max(420, ...canvas.nodes.map((node) => node.y + node.height + 80)); + const nodeOf = (id: string) => canvas.nodes.find((node) => node.id === id); + const cards = canvas.nodes.map((node) => ({ id: node.id, text: names.get(node.id) ?? '' })); + const cardOptions = [ + { value: '', text: t('qolSearch.canvas.chooseCard') }, + ...cards.map((card) => ({ value: card.id, text: card.text.slice(0, 60) })), + ]; + + return ( + + {!pageId &&

    {t('qolSearch.canvas.noPage')}

    } + {pageId && !ready &&

    {t('qolSearch.collections.loading')}

    } + {pageId && ready && ( + <> +
    + ({ value, text: t(`qolSearch.canvas.kinds.${value}`) }))} + onChange={(value) => { + setKind(value as CardKind); + setEntry(''); + }} + /> + + {kind !== 'page' && ( + + )} +
    + {suggestions.length > 0 && ( +
      + {suggestions.map((page) => ( +
    • + +
    • + ))} +
    + )} +
    + + + + +
    +
    + {(['row', 'column', 'grid'] as const).map((mode) => ( + + ))} + + {t('qolSearch.canvas.selected', { count: selection.length })} +
    +
    +
    + + {canvas.nodes.map((node) => { + const type = cardKind(node); + const text = names.get(node.id) ?? ''; + const chosen = selection.includes(node.id); + return ( +
    onDown(event, node)} + onPointerMove={onMove} + onPointerUp={onUp} + onKeyDown={(event) => onKey(event, node)} + > + {t(`qolSearch.canvas.kinds.${type}`)} + {editing === node.id && type !== 'page' ? ( +