diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml index 52cb6276..6043cf06 100644 --- a/.github/workflows/experiment.yml +++ b/.github/workflows/experiment.yml @@ -69,26 +69,82 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - experiment: + clang_producer: needs: latest_update if: ${{ needs.latest_update.outputs.run == 'true' }} + name: pinned Clang producer + runs-on: ubuntu-latest + timeout-minutes: 150 + steps: + - name: Skip when no C-family lane was requested + if: ${{ github.event_name == 'workflow_dispatch' && inputs.language != 'all' && inputs.language != 'c' && inputs.language != 'cpp' }} + run: echo "The requested language does not consume the Clang producer." + + - name: Checkout + if: ${{ github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == 'c' || inputs.language == 'cpp' }} + uses: actions/checkout@v7 + + - name: Setup Node + if: ${{ github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == 'c' || inputs.language == 'cpp' }} + uses: actions/setup-node@v7 + with: + node-version: 22.x + + - name: Restore the pinned Clang producer + id: clang_producer + if: ${{ github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == 'c' || inputs.language == 'cpp' }} + uses: actions/cache/restore@v6 + with: + path: tests/experiment/.work/tools + key: clang-producer-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts', 'tests/experiment/src/clang-producer.mjs') }} + + - name: Provision the pinned Clang producer + if: ${{ github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == 'c' || inputs.language == 'cpp' }} + run: node tests/experiment/src/clang-producer.mjs + env: + # An exact hit is immutable. If validation rejects it, rebuilding + # cannot repair that key, so refuse rather than claiming success. + SAMCHON_GRAPH_CLANG_PRODUCER_ALLOW_BUILD: ${{ steps.clang_producer.outputs.cache-hit != 'true' && '1' || '0' }} + + # This job owns the build and saves it before either corpus consumer can + # start. A later C or C++ experiment failure cannot discard a successful + # compiler build, and the two consumers cannot race on one cold key. + - name: Save the pinned Clang producer + if: ${{ steps.clang_producer.outputs.cache-hit != 'true' && (github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == 'c' || inputs.language == 'cpp') }} + continue-on-error: true + uses: actions/cache/save@v6 + with: + path: tests/experiment/.work/tools + key: ${{ steps.clang_producer.outputs.cache-primary-key }} + + # GitHub artifacts restore ordinary files as 0644. Pack first so the + # executable bits on clangd survive the same-run handoff. + - name: Pack the verified Clang producer + if: ${{ github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == 'c' || inputs.language == 'cpp' }} + run: tar -C tests/experiment/.work/tools -cf pinned-clang-producer.tar . + + # Cache writes are best-effort and immutable. The artifact is the + # guaranteed same-run handoff, including on a fork PR that cannot save a + # repository cache. + - name: Upload the verified Clang producer + if: ${{ github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == 'c' || inputs.language == 'cpp' }} + uses: actions/upload-artifact@v7 + with: + name: pinned-clang-producer + path: pinned-clang-producer.tar + if-no-files-found: error + retention-days: 1 + compression-level: 0 + + experiment: + needs: [latest_update, clang_producer] + if: ${{ always() && needs.latest_update.outputs.run == 'true' }} name: ${{ matrix.language }} LSP runs-on: ubuntu-latest - # Fourteen rows install a released producer and finish in minutes; the - # ninety-minute bound is theirs and stays exactly where it is. C and C++ - # build a compiler from source, and that is a property of the row rather - # than a defect inside it. Measured: three minutes of checkout, install and - # package build, then 56 minutes to a linked `clangd` on one runner and 107 - # on another in the same workflow, then the real-corpus lifecycle run. - # Ninety did not fit. 150 covers the fast runner comfortably and the slow - # one barely, and it is still a bound, so a lane that hangs is caught. Only - # the two rows that build a compiler get it. - # - # The restore below is what should keep an ordinary push away from that - # cost, and it is an expectation rather than a property: Actions caches are - # branch-scoped and evicted, so a first push on a fresh branch still pays - # the whole build, on whichever runner it draws. - timeout-minutes: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && 150 || 90 }} + # Compiler construction has its own 150-minute predecessor above. Every + # matrix row now spends this budget on setup and the real-corpus lifecycle + # only; C and C++ cannot consume it rebuilding LLVM on a cold cache miss. + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -152,50 +208,27 @@ jobs: if: ${{ github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == matrix.language }} run: pnpm build - # C and C++ are the only rows whose producer is built rather than - # downloaded, and it is a pinned commit: the same bytes, reproduced from - # scratch, on every push. Restoring them instead is not a shortcut around - # the build but a removal of work that had no reason to happen twice. - # - # The key hashes every file the built bytes depend on. `catalog.mjs` is - # the one `setup` actually reads its commit from, so it has to be here — - # keying on the adapter's constant alone would leave the two bound only - # by a text assertion in a different workflow, and a divergence would hit - # the key, fail the `--version` check, rebuild, and then not re-save, - # because an exact hit has nothing to write. That is silent, permanent, - # full-cost rebuilding, so the binding is made structural instead. - # - # Restore and save are split so the save runs even when a later step - # fails. A campaign iterating on these lanes is exactly the case where - # the producer builds and the experiment does not, and `cache`'s combined - # form would discard an hour or more of correct build because a corpus - # assertion afterwards went red. - - name: Restore the pinned Clang producer - id: clang_producer + # The producer job uploads the exact tree it verified, independently of + # whether cross-run cache saving was permitted. + - name: Download the verified Clang producer if: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && (github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == matrix.language) }} - uses: actions/cache/restore@v6 + uses: actions/download-artifact@v8 with: - path: tests/experiment/.work/tools - key: clang-producer-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts', 'tests/experiment/src/catalog.mjs', 'tests/experiment/src/setup-language.mjs') }} + name: pinned-clang-producer + path: tests/experiment/.work/clang-producer-artifact + + - name: Unpack the verified Clang producer + if: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && (github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == matrix.language) }} + run: | + mkdir -p tests/experiment/.work/tools + tar -C tests/experiment/.work/tools -xf tests/experiment/.work/clang-producer-artifact/pinned-clang-producer.tar - name: Install language server if: ${{ github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == matrix.language }} run: pnpm --filter @samchon/graph-experiment run setup -- --language ${{ matrix.language }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # Saved before the experiment runs, not after it, and only on a miss. - # The producer is proved by this point — `setup` refuses to finish unless - # the installed binary reports the pinned commit — and what follows is a - # corpus run whose failure says nothing about the compiler that was - # built. Waiting until the end would tie a correct build's survival to an - # unrelated assertion. - - name: Save the pinned Clang producer - if: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && steps.clang_producer.outputs.cache-hit != 'true' && (github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == matrix.language) }} - uses: actions/cache/save@v6 - with: - path: tests/experiment/.work/tools - key: ${{ steps.clang_producer.outputs.cache-primary-key }} + SAMCHON_GRAPH_CLANG_PRODUCER_ALLOW_BUILD: "0" # Node sizes its old space from a heuristic, not from the runner, and on # these 16 GiB hosts it settles near 4 GiB, which a real C project's @@ -257,6 +290,7 @@ jobs: # producer has exactly one witness, and it writes to stderr. SAMCHON_GRAPH_LSP_SERVER_LOG: "1" SAMCHON_GRAPH_RUST_ANALYZER_HIR: ${{ github.workspace }}/tests/experiment/.work/tools/bin/samchon-rust-analyzer + SAMCHON_GRAPH_ROSLYN_WORKSPACE: ${{ github.workspace }}/tests/experiment/.work/tools/bin/samchon-roslyn - name: Upload result if: ${{ always() && (github.event_name != 'workflow_dispatch' || inputs.language == 'all' || inputs.language == matrix.language) }} diff --git a/.github/workflows/index-time.yml b/.github/workflows/index-time.yml index c29db87f..3be8dab9 100644 --- a/.github/workflows/index-time.yml +++ b/.github/workflows/index-time.yml @@ -95,9 +95,69 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - measure: + clang_producer: needs: latest_update if: ${{ needs.latest_update.outputs.run == 'true' }} + name: pinned Clang producer + runs-on: ubuntu-latest + timeout-minutes: 150 + steps: + - name: Skip when no C-family project was requested + if: ${{ github.event_name == 'workflow_dispatch' && inputs.project != 'all' && inputs.project != 'redis' && inputs.project != 'leveldb' }} + run: echo "The requested project does not consume the Clang producer." + + - name: Checkout + if: ${{ github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == 'redis' || inputs.project == 'leveldb' }} + uses: actions/checkout@v7 + + - name: Setup Node + if: ${{ github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == 'redis' || inputs.project == 'leveldb' }} + uses: actions/setup-node@v7 + with: + node-version: 22.x + + - name: Restore the pinned Clang producer + id: clang_producer + if: ${{ github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == 'redis' || inputs.project == 'leveldb' }} + uses: actions/cache/restore@v6 + with: + path: tests/experiment/.work/tools + key: clang-producer-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts', 'tests/experiment/src/clang-producer.mjs') }} + + - name: Provision the pinned Clang producer + if: ${{ github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == 'redis' || inputs.project == 'leveldb' }} + run: node tests/experiment/src/clang-producer.mjs + env: + SAMCHON_GRAPH_CLANG_PRODUCER_ALLOW_BUILD: ${{ steps.clang_producer.outputs.cache-hit != 'true' && '1' || '0' }} + + # The compiler is durable before either measured consumer starts. A + # failed or timed-out cell cannot discard it, and redis/leveldb cannot + # both build the same cold key in this workflow. + - name: Save the pinned Clang producer + if: ${{ steps.clang_producer.outputs.cache-hit != 'true' && (github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == 'redis' || inputs.project == 'leveldb') }} + continue-on-error: true + uses: actions/cache/save@v6 + with: + path: tests/experiment/.work/tools + key: ${{ steps.clang_producer.outputs.cache-primary-key }} + + - name: Pack the verified Clang producer + if: ${{ github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == 'redis' || inputs.project == 'leveldb' }} + run: tar -C tests/experiment/.work/tools -cf pinned-clang-producer.tar . + + - name: Upload the verified Clang producer + if: ${{ github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == 'redis' || inputs.project == 'leveldb' }} + uses: actions/upload-artifact@v7 + with: + name: pinned-clang-producer + path: pinned-clang-producer.tar + if-no-files-found: error + retention-days: 1 + compression-level: 0 + + measure: + needs: [latest_update, clang_producer] + if: ${{ always() && needs.latest_update.outputs.run == 'true' }} # A skipped lane says so in its own name. Dispatching one project leaves the # other twelve jobs running nothing but an echo, with every later step gated # off, and they finish green — indistinguishable in the job list from lanes @@ -116,17 +176,10 @@ jobs: # cleanup, and artifact upload. The process cap below is the measurement # verdict; this job cap is only the outer lifecycle backstop. # - # C and C++ provision a compiler built from source, which is a property of - # those two rows rather than a defect inside them, and the restore above is - # an expectation rather than a property: a cold key — a fresh branch, an - # eviction, or the first run after the producer pin moves — pays the whole - # build. Measured at 56 minutes to a linked `clangd` on one runner and 107 - # on another in the same workflow. Under the old flat cap those two rows - # spent an hour and three quarters compiling and were killed with the - # measurement still running and nothing recorded. 210 covers the slow - # runner plus both measurement columns, and is still a bound, so a lane - # that hangs is caught rather than left to the six-hour platform limit. - timeout-minutes: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && 210 || 120 }} + # The 56-to-107-minute LLVM build now belongs to the 150-minute predecessor + # above. Every measured row gets the same two 30-minute process caps plus an + # hour for setup, quiet-host waits, cleanup, diagnosis, and artifact upload. + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -183,26 +236,20 @@ jobs: if: ${{ github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == matrix.project }} run: pnpm build - # The C and C++ producer is an LLVM build, and this workflow used to make - # every run pay for it: leveldb and redis each spent about an hour and - # three quarters compiling clangd inside a two-hour job, reached the - # measurement with minutes left, and were killed by the job cap with - # nothing measured. The experiment matrix already restores that build by - # a key over the exact files its bytes depend on; measuring is not a - # reason to rebuild it, so the same restore/save pair runs here. - # - # The key is identical to the experiment workflow's, deliberately: the - # two lanes provision the same producer from the same sources, so - # whichever runs first pays and the other reads. Restore and save are - # split so a measurement that fails, times out, or is cancelled still - # leaves the compiler it built behind. - - name: Restore the pinned Clang producer - id: clang_producer + # Same-run artifacts, not best-effort cache visibility, hand the verified + # producer to both measured consumers. + - name: Download the verified Clang producer if: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && (github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == matrix.project) }} - uses: actions/cache/restore@v6 + uses: actions/download-artifact@v8 with: - path: tests/experiment/.work/tools - key: clang-producer-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts', 'tests/experiment/src/catalog.mjs', 'tests/experiment/src/setup-language.mjs') }} + name: pinned-clang-producer + path: tests/experiment/.work/clang-producer-artifact + + - name: Unpack the verified Clang producer + if: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && (github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == matrix.project) }} + run: | + mkdir -p tests/experiment/.work/tools + tar -C tests/experiment/.work/tools -xf tests/experiment/.work/clang-producer-artifact/pinned-clang-producer.tar # The same provisioning the experiment matrix uses. It appends each tool's # directory to GITHUB_PATH, so the benchmark's own child processes resolve @@ -212,13 +259,7 @@ jobs: run: pnpm --filter @samchon/graph-experiment run setup -- --language ${{ matrix.language }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Save the pinned Clang producer - if: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && steps.clang_producer.outputs.cache-hit != 'true' && (github.event_name != 'workflow_dispatch' || inputs.project == 'all' || inputs.project == matrix.project) }} - uses: actions/cache/save@v6 - with: - path: tests/experiment/.work/tools - key: ${{ steps.clang_producer.outputs.cache-primary-key }} + SAMCHON_GRAPH_CLANG_PRODUCER_ALLOW_BUILD: "0" # What actually got installed, printed where the run can be read. # `recordTool` has always written this manifest and nothing ever surfaced @@ -336,6 +377,12 @@ jobs: - name: Install renderer dependencies run: pnpm install --frozen-lockfile + # Route summaries derive the expected owner from the runtime's canonical + # provider registry, so the collect job builds that dependency before it + # validates or prints any folded report. + - name: Build provider registry dependency + run: pnpm build + - name: Download reports uses: actions/download-artifact@v8 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b299d5be..a93214fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,25 @@ jobs: go-version: 1.26.5 cache-dependency-path: sidecars/go/go.sum + - name: Setup Java + uses: actions/setup-java@v6 + with: + distribution: temurin + java-version: "21" + cache: maven + cache-dependency-path: "sidecars/scala/**/pom.xml" + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + + - name: Setup Swift + if: runner.os == 'Linux' + uses: swift-actions/setup-swift@v2 + with: + swift-version: "6.0" + - name: Configure pnpm run: | pnpm config set fetch-retries 5 @@ -88,6 +107,26 @@ jobs: working-directory: sidecars/go run: go test ./... + - name: Build Roslyn sidecar + run: dotnet build sidecars/csharp/Samchon.Graph.CSharp.csproj --configuration Release -p:RestoreLockedMode=true + + - name: Build Scala sidecar + run: mvn --batch-mode --file sidecars/scala/pom.xml verify + + - name: Build Swift sidecar + if: runner.os != 'Windows' + shell: bash + run: | + SWIFT_CXX_FLAGS=() + if [[ "$RUNNER_OS" == "Linux" ]]; then + SWIFT_ROOT="$(dirname "$(dirname "$(command -v swift)")")" + SWIFT_CXX_FLAGS+=( + -Xcxx "-I${SWIFT_ROOT}/lib/swift" + -Xcxx "-I${SWIFT_ROOT}/lib/swift/Block" + ) + fi + swift build --package-path sidecars/swift --configuration release "${SWIFT_CXX_FLAGS[@]}" + - name: Test run: pnpm test diff --git a/.gitignore b/.gitignore index 9cbc0a25..7806be43 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,7 @@ packages/graph/sidecars/ # directory, and Go refuses to run an executable it found there. sidecars/go/go sidecars/go/go.exe +sidecars/csharp/bin/ +sidecars/csharp/obj/ +sidecars/scala/**/target/ +sidecars/swift/.build/ diff --git a/README.md b/README.md index 08bdf405..91e46224 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,10 @@ Strict selection is per registered provider and may decline for missing tools, i | `samchon-rust-analyzer-hir` | `rust` | `analyzer` | `contains`, `exports`, `imports`, `calls`, `accesses`, `instantiates`, `type_ref`, `extends`, `implements`, `overrides`, `dispatches`, `decorates`, `tests`, `references` | [upstream](https://github.com/samchon/rust-analyzer) / [route #72](https://github.com/samchon/compiler-knowledge-graph/issues/72) | | `clangd-snapshot` | `c`, `cpp` | `compiler` | `contains`, `exports`, `imports`, `calls`, `accesses`, `instantiates`, `type_ref`, `extends`, `implements`, `overrides`, `dispatches`, `references` | [upstream](https://github.com/samchon/llvm-project) / [route #73](https://github.com/samchon/compiler-knowledge-graph/issues/73) | | `javac-graph` | `java` | `compiler` | `contains`, `exports`, `imports`, `calls`, `accesses`, `instantiates`, `type_ref`, `extends`, `implements`, `overrides`, `dispatches`, `decorates`, `tests`, `references` | [upstream](https://github.com/samchon/scip-java) / [route #74](https://github.com/samchon/compiler-knowledge-graph/issues/74) | -| `scip-kotlinc` | `kotlin` | `semantic-index` | `contains`, `references` | [upstream](https://github.com/scip-code/scip-java) / [route #76](https://github.com/samchon/compiler-knowledge-graph/issues/76) | -| `scip-dotnet` | `csharp` | `semantic-index` | **none** | [upstream](https://github.com/sourcegraph/scip-dotnet) / [route #75](https://github.com/samchon/compiler-knowledge-graph/issues/75) | +| `kotlinc-graph` | `kotlin` | `compiler` | `contains`, `exports`, `imports`, `calls`, `accesses`, `instantiates`, `type_ref`, `extends`, `implements`, `overrides`, `dispatches`, `decorates`, `tests`, `references` | [upstream](https://github.com/samchon/scip-java) / [route #76](https://github.com/samchon/compiler-knowledge-graph/issues/76) | +| `scalac-graph` | `scala` | `compiler` | `contains`, `exports`, `imports`, `calls`, `accesses`, `instantiates`, `type_ref`, `extends`, `implements`, `overrides`, `dispatches`, `decorates`, `references` | [upstream](https://build-server-protocol.github.io/docs/specification) / [route #77](https://github.com/samchon/compiler-knowledge-graph/issues/77) | +| `swift-indexstore` | `swift` | `compiler` | `contains`, `exports`, `imports`, `calls`, `accesses`, `instantiates`, `type_ref`, `extends`, `implements`, `overrides`, `dispatches`, `decorates`, `tests`, `references` | [upstream](https://github.com/swiftlang/indexstore-db) / [route #78](https://github.com/samchon/compiler-knowledge-graph/issues/78) | +| `roslyn-workspace` | `csharp` | `compiler` | `contains`, `exports`, `imports`, `calls`, `accesses`, `instantiates`, `type_ref`, `extends`, `implements`, `overrides`, `dispatches`, `decorates`, `tests`, `references` | [upstream](https://github.com/dotnet/roslyn) / [route #75](https://github.com/samchon/compiler-knowledge-graph/issues/75) | | `scip-python` | `python` | `semantic-index` | `references` | [upstream](https://github.com/sourcegraph/scip-python) / [route #80](https://github.com/samchon/compiler-knowledge-graph/issues/80) | | `scip-ruby` | `ruby` | `semantic-index` | **none** | [upstream](https://github.com/sourcegraph/scip-ruby) / [route #81](https://github.com/samchon/compiler-knowledge-graph/issues/81) | | `scip-dart` | `dart` | `semantic-index` | **none** | [upstream](https://pub.dev/packages/scip_dart) / [route #84](https://github.com/samchon/compiler-knowledge-graph/issues/84) | @@ -94,9 +96,11 @@ These are current implementation modes, not future route claims. Preparation and | `samchon-graph-lua` | `unchanged-snapshot-reuse; full-rebuild-on-change` | LuaLS workspace configuration and the shipped readable exporter. | LuaLS analyzes the workspace and the shipped exporter asks its semantic VM for declaration references. | A changed-input run publishes one references-only whole-workspace graph. | Unchanged inputs reuse the validated snapshot; the current exporter is not a resident incremental session. | | `samchon-rust-analyzer-hir` | `resident-no-op-reuse; invalidated-closure shard deltas; validated restart checkpoints` | Cargo metadata/config, lock/toolchain inputs, target/features/cfg and build-script/proc-macro universe. | The pinned rust-analyzer fork owns one resident HIR database and exports declarations, semantic relationships, diagnostics, coverage and unresolved boundaries from that exact analysis revision. | Content-addressed source shards cross a versioned LSP transaction; the client verifies producer identity, universe, complete manifests, shard digests and graph invariants before atomic publication. | No-op requests reuse the resident snapshot; interface changes invalidate dependent shards, while a complete consumer checkpoint restores the same generation after process restart. | | `clangd-snapshot` | `resident-no-op-reuse; complete changed-TU/configuration replacement; content-addressed deltas` | A valid compilation database plus every source, header, generated input, command, target and working-directory identity used by its translation units. | The pinned clangd fork runs every registered command, keeps headers scoped by translation unit and configuration, and captures complete Clang roles, relations, macros, includes, diagnostics and source digests from the same compiler pass. | Versioned native shards cross one optimistic atomic snapshot; the client verifies producer identity, complete manifests, native and common shard digests, coverage, unresolved boundaries and source identities before publication. | No-op requests reuse the exact resident graph; changed sources or compilation-database commands reindex their owning translation units while a failed batch preserves the last complete generation without publishing it as current. | -| `javac-graph` | `unchanged-input reuse; committed per-target generations with shard-level deltas` | A Maven or Gradle project whose Java compile tasks the producer can attach its javac plugin to, and the JDK those tasks run. | A javac plugin walks the attributed tree the project's own compile task already produced, once per compilation unit. | Committed per-target generations become one validated protocol transaction; every edge endpoint, coverage row and target universe is proved before publication. | The build tool's incremental compilation decides what is rewritten; unchanged shards are carried forward instead of resent, and a moved universe or target set publishes a whole generation. | -| `scip-kotlinc` | `unchanged-snapshot-reuse; full-rebuild-on-change` | Maven or Gradle project metadata, dependency/classpath state and Kotlin compiler inputs. | scip-java drives the selected Maven or Gradle build and its kotlinc producer as one batch. | The complete decoded artifact is merged as a contains/references graph before atomic publication. | Unchanged inputs reuse the validated snapshot; no kotlinc or build session remains resident. | -| `scip-dotnet` | `unchanged-snapshot-reuse; full-rebuild-on-change` | Solution/project/TFM/NuGet/MSBuild inputs and a resolvable SDK. | scip-dotnet loads and analyzes the selected solution through one batch producer run. | The complete decoded artifact publishes declarations but no currently defensible edge family. | Unchanged inputs reuse the validated snapshot; no Roslyn workspace remains resident. | +| `javac-graph` | `javac reuses unchanged inputs and commits per-target shard deltas; JDT retains one resident whole-workspace generation and reports initial, unchanged, incremental, reload or error` | A Maven or Gradle project whose Java compile tasks can host the javac plugin, or a JDT-importable resident workspace; the pinned experiment builds both producers with JDK 21. | The javac plugin walks each attributed compilation unit once inside the project's compile task; the JDT command reconciles each resident compilation unit once and emits one bulk snapshot. | javac target generations and the JDT workspace generation each become one validated protocol transaction; JDT is a fallback semantic owner, never an enrichment pass after javac. | Build-tool incremental compilation carries unchanged javac shards forward. JDT keeps the workspace process resident, freezes source buffers under one read boundary and retains the prior strict generation on errors. | +| `kotlinc-graph` | `resident Gradle connection; per-target content-addressed shard generations; atomic incremental replacement` | A Kotlin/JVM Gradle project using Kotlin 2.3.20, a compatible JDK, resolvable dependencies and the complete build/classpath inputs for every compiled target. | The K2 plugin traverses FIR and IR once inside each ordinary Kotlin Gradle compile task and records compiler-resolved declarations, calls, accesses, types, dispatch, annotations, tests and diagnostics. | The consumer verifies producer/compiler identity, target universes, coverage, unresolved sites, source and disk digests, edge endpoints and complete target manifests before one atomic graph publication. | Exact no-op requests reuse the validated in-memory snapshot without invoking Gradle. Changed source or build inputs reuse one persistent Gradle Tooling connection, KGP daemon, configuration, classpath and incremental caches while carrying unchanged compiler shards forward. | +| `scalac-graph` | `resident BSP connection; per-target SemanticDB and typed-plugin shards; atomic Zinc incremental replacement` | BSP targets whose scalac options load the matching Scala 2/3 typed plugin and emit SemanticDB in the same Zinc-controlled compile. | The repository's BSP server asks Zinc to compile; separate Scala 2 and Scala 3 post-typer plugins emit resolved roles during that same compile while SemanticDB supplies declaration and diagnostic cross-checks. | The consumer verifies BSP target universes, compiler/plugin pairing, SemanticDB URI, md5 and build target, complete coverage, source digests and target manifests before one atomic publication. | Exact no-op requests reuse the validated snapshot. Changed inputs use the same BSP server and Zinc analysis, replace emitted shards and carry validated unchanged shards forward without `clean`. | +| `swift-indexstore` | `resident sidecar process; native SwiftPM incremental builds; explicit-output-unit IndexStoreDB generations` | A SwiftPM package, Swift 6.0-compatible toolchain and its libIndexStore on macOS or Linux. | The package's ordinary `swift build --enable-index-store --build-tests -Xswiftc -index-include-locals` produces compiler records; the sidecar takes the exact source and object paths from SwiftPM's current build description, opens a fresh explicit-output-unit IndexStoreDB view and reads compiler USRs, roles and relations. | The consumer verifies the exact sorted output-unit paths and digests, module/triple/configuration universe, one-pass source enrichment, source digests, coverage and unresolved boundaries before atomic publication. | Exact no-op requests reuse the validated in-memory snapshot. Changed requests reuse SwiftPM build products but reconstruct the frozen IndexStoreDB view; this does not claim SourceKit-LSP cache or scheduler ownership. | +| `roslyn-workspace` | `resident-immutable-solution; shard-delta; atomic-reload` | One unambiguous .sln, .slnx or .csproj selection, restored assets and a matching SDK/MSBuild instance. | One resident MSBuildWorkspace captures immutable Solution generations and Roslyn symbols, semantic models and operations. | Document, generated-source and project metadata shards carry compiler-resolved declarations, relationships, diagnostics, coverage and source digests. | No-op refreshes reuse the exact committed generation; source edits update the immutable Solution and build-input changes reload it atomically. | | `scip-python` | `unchanged-snapshot-reuse; full-rebuild-on-change` | Python project/config/environment/import/stub inputs. | scip-python runs its bundled Pyright-based analysis once for the selected project environment. | The complete decoded artifact publishes a references-only project graph. | Unchanged inputs reuse the validated snapshot; no Pyright analysis session remains resident. | | `scip-ruby` | `unchanged-snapshot-reuse; full-rebuild-on-change` | Gem/Bundler/Sorbet/RBI configuration inputs. | scip-ruby performs one full-project batch using the selected Ruby, Bundler and Sorbet inputs. | The complete decoded artifact publishes declarations but no currently defensible edge family. | Unchanged inputs reuse the validated snapshot; no Ruby or Sorbet index remains resident. | | `scip-dart` | `unchanged-snapshot-reuse; full-rebuild-on-change` | pubspec/lock, analysis options and resolved package configuration. | scip_dart performs one full-project batch using the resolved Dart package universe. | The complete decoded artifact publishes declarations but no currently defensible edge family. | Unchanged inputs reuse the validated snapshot; this is not resident Analysis Server state. | @@ -111,11 +115,13 @@ The troubleshooting table names the ordinary language-server/static fallback for | `ttscgraph` | Install `ttsc@>=0.24.0` in the indexed project. 0.24.0 is the first published release whose `ttscgraph serve` answers graph snapshot protocol v1; 0.23.0 and earlier answer a legacy complete dump and are declined. | [ttsc 0.24.0 first protocol v1 release](https://www.npmjs.com/package/ttsc/v/0.24.0), [native shard producer PR](https://github.com/samchon/ttsc/pull/1056) | `ttscgraph`, `ttscserver` | — | `TTSC_GRAPH_BINARY` | Absolute `TTSC_GRAPH_BINARY`, target-project `ttsc` package/binary, target-project `.bin`, then PATH/global compatibility fallback. | A matching ttsc/TypeScript project and tsconfig/jsconfig/package inputs. | `linux`, `macos`, `windows` | | `samchon-graph-go` | Go 1.25+; the package ships the Go exporter source. Install corroboration with `go install github.com/scip-code/scip-go/cmd/scip-go@v0.2.7`. | [Go downloads](https://go.dev/dl/), [scip-go 0.2.7 source](https://github.com/scip-code/scip-go/tree/v0.2.7) | `samchon-graph-go`, `go`, `scip-go` | — | `SAMCHON_GRAPH_GO`, `SAMCHON_GRAPH_GO_TOOLCHAIN`, `SAMCHON_GRAPH_SCIP_GO` | Project/PATH `samchon-graph-go`, then the shipped source runner through Go; absolute environment overrides take precedence. | Go workspace/module inputs, selected GOOS/GOARCH/cgo environment, embedded files and vendored inputs. | `linux`, `macos`, `windows` | | `samchon-graph-lua` | Install `lua-language-server`; the package ships `sidecars/lua/export.lua`. | [LuaLS releases](https://github.com/LuaLS/lua-language-server/releases) | `lua-language-server` | — | `SAMCHON_GRAPH_LUA`, `SAMCHON_GRAPH_LUA_EXPORTER` | Absolute `SAMCHON_GRAPH_LUA`, then project/PATH LuaLS; the shipped exporter may be replaced by `SAMCHON_GRAPH_LUA_EXPORTER`. | LuaLS workspace configuration and the shipped readable exporter. | `linux`, `macos`, `windows` | -| `samchon-rust-analyzer-hir` | Build the `samchon/rust-analyzer` graph-snapshot fork at commit `2850ecba80311bebd4cdaa9fedc5321533b5b1e7`; point `SAMCHON_GRAPH_RUST_ANALYZER_HIR` at that binary or install it as `samchon-rust-analyzer`. | [native HIR graph producer PR](https://github.com/samchon/rust-analyzer/pull/1), [rust-analyzer build instructions](https://rust-analyzer.github.io/book/contributing/) | `samchon-rust-analyzer`, `rust-analyzer` | — | `SAMCHON_GRAPH_RUST_ANALYZER_HIR` | Absolute `SAMCHON_GRAPH_RUST_ANALYZER_HIR`, then project/PATH `samchon-rust-analyzer`, then a project/PATH `rust-analyzer` only when its version reports the pinned producer commit. | Cargo metadata/config, lock/toolchain inputs, target/features/cfg and build-script/proc-macro universe. | `linux`, `macos`, `windows` | -| `clangd-snapshot` | Build the `samchon/llvm-project` graph-snapshot fork at commit `e33d8f51552a523b5696691738f1ef95f8e3a730`; point `SAMCHON_GRAPH_CLANGD_SNAPSHOT` at `clangd` or install it as `samchon-clangd`, and provide a compilation database. | [native Clang graph producer PR](https://github.com/samchon/llvm-project/pull/1), [LLVM build instructions](https://llvm.org/docs/CMake.html) | `samchon-clangd`, `clangd` | `compile_commands.json`, `build/compile_commands.json` | `SAMCHON_GRAPH_CLANGD_SNAPSHOT` | Absolute `SAMCHON_GRAPH_CLANGD_SNAPSHOT`, then project/PATH `samchon-clangd`, then a project/PATH `clangd` only when its version reports the pinned producer commit. | A valid compilation database plus every source, header, generated input, command, target and working-directory identity used by its translation units. | `linux`, `macos`, `windows` | -| `javac-graph` | Install a `scip-java` build whose `index` command accepts `--graph-output`, plus a JDK 17+ toolchain. A released build without that option is declined to the SCIP lane rather than run. | [javac graph producer](https://github.com/samchon/scip-java/pull/1), [scip-java](https://github.com/scip-code/scip-java) | `scip-java`, `java` | — | `SAMCHON_GRAPH_JAVAC_GRAPH`, `SAMCHON_GRAPH_JAVA_TOOLCHAIN` | Absolute `SAMCHON_GRAPH_JAVAC_GRAPH`, then a project-local or PATH `scip-java` whose `index --help` publishes `--graph-output`. | A Maven or Gradle project whose Java compile tasks the producer can attach its javac plugin to, and the JDK those tasks run. | `linux`, `macos`, `windows` | -| `scip-kotlinc` | Install `scip-java` 0.13.1, the `scip` decoder and a compatible JDK; Kotlin experiments pin a compatible source build. | [scip-java 0.13.1 release](https://github.com/scip-code/scip-java/releases/tag/v0.13.1), [SCIP releases](https://github.com/sourcegraph/scip/releases) | `scip-java`, `scip`, `java` | — | `SAMCHON_GRAPH_SCIP_JAVA`, `SAMCHON_GRAPH_SCIP`, `SAMCHON_GRAPH_JAVA_TOOLCHAIN` | Project-local producer/decoder/JDK precede PATH; absolute environment overrides select each tool. | Maven or Gradle project metadata, dependency/classpath state and Kotlin compiler inputs. | `linux`, `macos`, `windows` | -| `scip-dotnet` | `dotnet tool install --global scip-dotnet`; install the `scip` decoder and matching .NET SDK. | [scip-dotnet on NuGet](https://www.nuget.org/packages/scip-dotnet), [SCIP releases](https://github.com/sourcegraph/scip/releases) | `scip-dotnet`, `scip`, `dotnet` | — | `SAMCHON_GRAPH_SCIP_DOTNET`, `SAMCHON_GRAPH_SCIP`, `SAMCHON_GRAPH_DOTNET_TOOLCHAIN` | Project-local producer/decoder/toolchain precede PATH; absolute environment overrides select each tool. | Solution/project/TFM/NuGet/MSBuild inputs and a resolvable SDK. | `linux`, `macos`, `windows` | +| `samchon-rust-analyzer-hir` | Build the `samchon/rust-analyzer` graph-snapshot fork at commit `378f220482c298775910f0fc46e8fda1bc516ecc`; point `SAMCHON_GRAPH_RUST_ANALYZER_HIR` at that binary or install it as `samchon-rust-analyzer`. | [pinned HIR producer commit](https://github.com/samchon/rust-analyzer/commit/378f220482c298775910f0fc46e8fda1bc516ecc), [initial native HIR producer PR](https://github.com/samchon/rust-analyzer/pull/1), [node ownership repair PR](https://github.com/samchon/rust-analyzer/pull/2), [rust-analyzer build instructions](https://rust-analyzer.github.io/book/contributing/) | `samchon-rust-analyzer`, `rust-analyzer` | — | `SAMCHON_GRAPH_RUST_ANALYZER_HIR` | Absolute `SAMCHON_GRAPH_RUST_ANALYZER_HIR`, then project/PATH `samchon-rust-analyzer`, then a project/PATH `rust-analyzer` only when its version reports the pinned producer commit. | Cargo metadata/config, lock/toolchain inputs, target/features/cfg and build-script/proc-macro universe. | `linux`, `macos`, `windows` | +| `clangd-snapshot` | Build the `samchon/llvm-project` graph-snapshot fork at commit `d6371c37445998d24776692a27e086bb24f9916a`; point `SAMCHON_GRAPH_CLANGD_SNAPSHOT` at `clangd` or install it as `samchon-clangd`, and provide a compilation database. | [pinned Clang producer commit](https://github.com/samchon/llvm-project/commit/d6371c37445998d24776692a27e086bb24f9916a), [bounded resident-view PR](https://github.com/samchon/llvm-project/pull/2), [initial native Clang producer PR](https://github.com/samchon/llvm-project/pull/1), [LLVM build instructions](https://llvm.org/docs/CMake.html) | `samchon-clangd`, `clangd` | `compile_commands.json`, `build/compile_commands.json` | `SAMCHON_GRAPH_CLANGD_SNAPSHOT` | Absolute `SAMCHON_GRAPH_CLANGD_SNAPSHOT`, then project/PATH `samchon-clangd`, then a project/PATH `clangd` only when its version reports the pinned producer commit. | A valid compilation database plus every source, header, generated input, command, target and working-directory identity used by its translation units. | `linux`, `macos`, `windows` | +| `javac-graph` | Build the pinned `scip-java` graph producer at `fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a` and the pinned JDT workspace producer at `0d55a6c13d14e0d0466eeb021920349b3d0c6d35` with JDK 21. A javac build without `--graph-output` is declined before compilation. | [pinned javac graph producer](https://github.com/samchon/scip-java/commit/fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a), [initial javac graph implementation](https://github.com/samchon/scip-java/pull/1), [producer acceptance pull request](https://github.com/samchon/scip-java/pull/2), [pinned JDT workspace producer](https://github.com/samchon/eclipse.jdt.ls/commit/0d55a6c13d14e0d0466eeb021920349b3d0c6d35), [JDT workspace producer pull request](https://github.com/samchon/eclipse.jdt.ls/pull/1), [scip-java](https://github.com/scip-code/scip-java) | `scip-java`, `java` | — | `SAMCHON_GRAPH_JAVAC_GRAPH`, `SAMCHON_GRAPH_JAVA_TOOLCHAIN` | The build lane resolves absolute `SAMCHON_GRAPH_JAVAC_GRAPH`, then a project-local or PATH `scip-java` whose `index --help` publishes `--graph-output`; on decline, the resident lane resolves absolute `SAMCHON_GRAPH_JDT_WORKSPACE` or `samchon-jdtls`. | A Maven or Gradle project whose Java compile tasks can host the javac plugin, or a JDT-importable resident workspace; the pinned experiment builds both producers with JDK 21. | `linux`, `macos`, `windows` | +| `kotlinc-graph` | Build the pinned Kotlin 2.3.20 graph producer at `3a1565d0647d89a28880fa40ecbef0966a1a328c` with JDK 21. The indexed project supplies its Gradle wrapper when present; otherwise the resident producer uses pinned Gradle 9.4.1. | [pinned Kotlin graph producer](https://github.com/samchon/scip-java/commit/3a1565d0647d89a28880fa40ecbef0966a1a328c), [Kotlin graph producer pull request](https://github.com/scip-code/scip-java/pull/1006), [Kotlin Gradle compiler options](https://kotlinlang.org/docs/gradle-compiler-options.html) | `scip-java`, `java` | — | `SAMCHON_GRAPH_KOTLINC_GRAPH`, `SAMCHON_GRAPH_JAVA_TOOLCHAIN` | Absolute `SAMCHON_GRAPH_KOTLINC_GRAPH`, then project-local or PATH `scip-java`; the launcher must publish both `--kotlin-graph-output` and the resident `kotlin-graph-server` protocol before selection. | A Kotlin/JVM Gradle project using Kotlin 2.3.20, a compatible JDK, resolvable dependencies and the complete build/classpath inputs for every compiled target. | `linux`, `macos`, `windows` | +| `scalac-graph` | Build the shipped BSP producer with `mvn --batch-mode --file sidecars/scala/pom.xml verify`, expose its server jar as `samchon-scala-graph`, and configure the matching Scala 2 or Scala 3 typed plugin together with SemanticDB in each indexed target. | [shipped Scala graph source](https://github.com/samchon/compiler-knowledge-graph/tree/master/sidecars/scala), [pinned Scala 2/3 BSP fixture](https://github.com/samchon/graph-benchmark-scala), [Build Server Protocol](https://build-server-protocol.github.io/docs/specification), [SemanticDB specification](https://scalameta.org/docs/semanticdb/specification.html) | `samchon-scala-graph`, `java` | — | `SAMCHON_GRAPH_SCALA_GRAPH`, `SAMCHON_GRAPH_JAVA_TOOLCHAIN` | Absolute `SAMCHON_GRAPH_SCALA_GRAPH`, then project-local or PATH `samchon-scala-graph`; selection also requires a usable `.bsp/*.json` connection and the resident `graph-server` capability. | BSP targets whose scalac options load the matching Scala 2/3 typed plugin and emit SemanticDB in the same Zinc-controlled compile. | `linux`, `macos`, `windows` | +| `swift-indexstore` | Build the shipped SwiftPM sidecar with Swift 6.0 or newer, expose `samchon-swift-graph` on PATH, or point `SAMCHON_GRAPH_SWIFT_GRAPH` at the binary. The package pins the Swift 6.1 IndexStoreDB release commit that remains source-compatible with Swift 6.0 and fixes Clang 19's 64-bit role declaration. | [shipped Swift graph source](https://github.com/samchon/compiler-knowledge-graph/tree/master/sidecars/swift), [pinned IndexStoreDB commit](https://github.com/swiftlang/indexstore-db/commit/54212fce1aecb199070808bdb265e7f17e396015), [IndexStoreDB explicit output units](https://github.com/swiftlang/indexstore-db/blob/54212fce1aecb199070808bdb265e7f17e396015/Sources/IndexStoreDB/IndexStoreDB.swift) | `samchon-swift-graph`, `swift` | — | `SAMCHON_GRAPH_SWIFT_GRAPH`, `SAMCHON_GRAPH_SWIFT_TOOLCHAIN` | Absolute `SAMCHON_GRAPH_SWIFT_GRAPH`, then project-local or PATH `samchon-swift-graph`; selection requires Package.swift, macOS or Linux, a matching libIndexStore, project support and the resident sidecar protocol. | A SwiftPM package, Swift 6.0-compatible toolchain and its libIndexStore on macOS or Linux. | `linux`, `macos` | +| `roslyn-workspace` | Install a matching .NET SDK; the package ships the pinned Roslyn service source, or `SAMCHON_GRAPH_ROSLYN_WORKSPACE` may select a prebuilt service. | [Roslyn workspace model](https://learn.microsoft.com/en-us/dotnet/csharp/roslyn-sdk/work-with-workspace), [Microsoft.CodeAnalysis.Workspaces.MSBuild](https://www.nuget.org/packages/Microsoft.CodeAnalysis.Workspaces.MSBuild) | `samchon-roslyn`, `dotnet` | — | `SAMCHON_GRAPH_ROSLYN_WORKSPACE`, `SAMCHON_GRAPH_DOTNET_TOOLCHAIN` | A prebuilt service override precedes the shipped source; the selected dotnet toolchain builds that source when no service binary resolves. | One unambiguous .sln, .slnx or .csproj selection, restored assets and a matching SDK/MSBuild instance. | `linux`, `macos`, `windows` | | `scip-python` | `npm install -g @sourcegraph/scip-python@0.6.6`; install the `scip` decoder and select Python. | [scip-python 0.6.6 on npm](https://www.npmjs.com/package/@sourcegraph/scip-python/v/0.6.6), [SCIP releases](https://github.com/sourcegraph/scip/releases) | `scip-python`, `scip`, `python3`, `python`, `py` | — | `SAMCHON_GRAPH_SCIP_PYTHON`, `SAMCHON_GRAPH_SCIP`, `SAMCHON_GRAPH_PYTHON_TOOLCHAIN` | Project-local producer/decoder/interpreter precede PATH; Python aliases are tried in order and absolute overrides select each tool. | Python project/config/environment/import/stub inputs. | `linux`, `macos`, `windows` | | `scip-ruby` | Install the pinned `scip-ruby` 0.4.7 release binary, the `scip` decoder and matching Ruby/Bundler. | [scip-ruby 0.4.7 release](https://github.com/sourcegraph/scip-ruby/releases/tag/scip-ruby-v0.4.7), [SCIP releases](https://github.com/sourcegraph/scip/releases) | `scip-ruby`, `scip`, `ruby` | — | `SAMCHON_GRAPH_SCIP_RUBY`, `SAMCHON_GRAPH_SCIP`, `SAMCHON_GRAPH_RUBY_TOOLCHAIN` | Project-local producer/decoder/Ruby precede PATH; absolute environment overrides select each tool. | Gem/Bundler/Sorbet/RBI configuration inputs. | `linux`, `macos`, `windows-when-installed` | | `scip-dart` | `dart pub global activate scip_dart 1.6.2`; install the `scip` decoder and Dart SDK. | [scip_dart 1.6.2](https://pub.dev/packages/scip_dart/versions/1.6.2), [SCIP releases](https://github.com/sourcegraph/scip/releases) | `scip_dart`, `scip`, `dart` | — | `SAMCHON_GRAPH_SCIP_DART`, `SAMCHON_GRAPH_SCIP`, `SAMCHON_GRAPH_DART_TOOLCHAIN` | Project-local producer/decoder/Dart precede PATH; absolute environment overrides select each tool. | pubspec/lock, analysis options and resolved package configuration. | `linux`, `macos`, `windows` | @@ -134,8 +140,8 @@ These are exact same-run cold end-to-end strict/strict-disabled pairs from [`tes | `redis` | `scip-clang` (prior fallback evidence; `clangd-snapshot` not yet measured) | 22,794.688 ms | 262,905.796 ms | | `leveldb` | `scip-clang` (prior fallback evidence; `clangd-snapshot` not yet measured) | 8,352.928 ms | 26,451.952 ms | | `gson` | `scip-java` (prior fallback evidence; `javac-graph` not yet measured) | 88,653.499 ms | 231,398.489 ms | -| `koin` | `scip-java` (prior fallback evidence; `scip-kotlinc` not yet measured) | 211,263.800 ms | 967,711.761 ms | -| `serilog` | `scip-dotnet` | 20,498.324 ms | 25,085.071 ms | +| `koin` | `scip-java` (prior fallback evidence; `kotlinc-graph` not yet measured) | 211,263.800 ms | 967,711.761 ms | +| `serilog` | `scip-dotnet` (prior fallback evidence; `roslyn-workspace` not yet measured) | 20,498.324 ms | 25,085.071 ms | | `flask` | `scip-python` | 10,628.897 ms | 748.454 ms | | `sinatra` | `scip-ruby` | did not finish before 1,800 s | did not finish before 1,800 s | | `darthttp` | `scip-dart` | did not finish before 1,800 s | did not finish before 1,800 s | @@ -150,11 +156,13 @@ A strict result's provenance name must equal the provider below. If it is absent | `typescript` | `ttscgraph` | A project on `ttsc@<=0.23.0` gets a legacy complete dump, which this route refuses rather than adapts, so those projects fall back honestly to `ttscserver`. | A missing target-project ttsc binary, legacy full-dump producer, incompatible request cap, malformed transaction or unsupported schema declines the strict provider. | `ttscserver`, then `@samchon/graph-sitter`. | | `go` | `samchon-graph-go` | Current changed-input export is a full rebuild and does not retain a resident `go/packages` checker session. | A missing Go 1.25+ toolchain, missing pinned scip-go corroborator or invalid workspace/module load declines the strict provider. | `gopls`, then `@samchon/graph-sitter`. | | `lua` | `samchon-graph-lua` | The current exporter calls `vm.getRefs` per declaration and proves references only; #83 replaces it with an occurrence-oriented resident traversal. | A missing LuaLS binary/exporter, invalid workspace result or bounded request declines the strict provider. | Generic LuaLS, then `@samchon/graph-sitter`. | -| `rust` | `samchon-rust-analyzer-hir` | The producer is currently available from the draft fork PR rather than a rust-analyzer release, and Rust has no `renders` relationship family. | A missing pinned producer, incompatible commit/schema, malformed transaction, invalid checkpoint, unsupported bounded option or failed Cargo workspace load declines this route. | Stock `rust-analyzer-scip`, then generic rust-analyzer, then `@samchon/graph-sitter`. | +| `rust` | `samchon-rust-analyzer-hir` | The pinned follow-up is available from the fork commit pending human-submitted review rather than a rust-analyzer release, and Rust has no `renders` relationship family. | A missing pinned producer, incompatible commit/schema, malformed transaction, invalid checkpoint, unsupported bounded option or failed Cargo workspace load declines this route. | Stock `rust-analyzer-scip`, then generic rust-analyzer, then `@samchon/graph-sitter`. | | `c`, `cpp` | `clangd-snapshot` | Calls, instantiation, exports, implements and dispatch are explicitly partial; C/C++ have no decorates, renders or tests family, and the producer is currently a draft fork rather than an LLVM release. | A missing pinned producer, missing or invalid compilation database, incompatible schema/commit, incomplete indexing batch, source/configuration movement or compiler error declines this route. | `scip-clang`, then stock `clangd`, then `@samchon/graph-sitter`. | -| `java` | `javac-graph` | javac's diagnostics stay with the build that emitted them, so this route carries none and says so through its capabilities. The JDT resident workspace lane remains separate work. | A missing producer or JDK, a launcher without `--graph-output`, a producer that does not commit atomic generations, an incomplete coverage matrix, a foreign project root or a bounded request declines the strict provider. | `scip-java`, then `jdtls`, then `@samchon/graph-sitter`. | -| `kotlin` | `scip-kotlinc` | The released Gradle path disables configuration cache and runs `clean scipCompileAll`; it is navigation fallback for #76, not compiler-owned calls or accesses. The producer does not expose the kotlinc revision the indexed build selected, so the compiler row stays empty rather than naming the JVM that launched it. | A missing producer/decoder/JDK, unsupported Maven or Gradle project, or invalid dependency/build configuration declines the strict provider. | `kotlin-language-server`, then `@samchon/graph-sitter`. | -| `csharp` | `scip-dotnet` | The artifact proves declarations but no graph edge family and can log MSBuildWorkspace failures after publishing. | A missing producer/decoder/.NET SDK, absent solution/project input or invalid MSBuild load declines the strict provider. | `csharp-ls`, then `@samchon/graph-sitter`. | +| `java` | `javac-graph` | javac diagnostics stay with the build that emitted them and are not carried by this route. JDT carries resident diagnostics and unsaved-buffer digests but currently proves containment only; standard Gradle main/test/subproject sources share javac task identities, while custom source-set tasks retain JDT-local scope. Its producer remains a pinned fork draft. | A missing producer or JDK, a launcher without `--graph-output`, incompatible schema or coverage, a foreign root, a bounded request, an incomplete JDT generation or moved inputs behind a reused generation declines the affected strict lane. | Pinned `jdt-workspace`, then `scip-java`, then generic `jdtls`, then `@samchon/graph-sitter`. | +| `kotlin` | `kotlinc-graph` | The compiler-owned route is intentionally limited to Gradle Kotlin/JVM on Kotlin 2.3.20. Maven, Android, multiplatform, JS, Native, newer or older compiler minors and bounded indexing options are declined instead of being mislabeled as compiler-owned facts. | A missing compatible producer/JDK/Gradle build, absent resident capability, unsupported target kind/compiler minor, incomplete or conflicting shard generation, source/configuration drift, compiler diagnostics that stop publication or malformed artifact retains the prior strict generation and declines this route. | `scip-kotlinc`, then `kotlin-language-server`, then `@samchon/graph-sitter`. | +| `scala` | `scalac-graph` | Targets without both the matching typed plugin and SemanticDB decline. `renders` and `tests` need framework-specific enrichers and are unsupported; Metals and static extraction remain the ordinary fallbacks. | A missing BSP connection, producer, Java runtime, compatible compiler/plugin pair or SemanticDB output, or a malformed/stale target generation declines without replacing the prior graph. | `metals`, then `@samchon/graph-sitter`; current `scip-java` is never presented as Scala-capable. | +| `swift` | `swift-indexstore` | This is the standalone SwiftPM fallback, not the preferred in-process SourceKit-LSP hook. Xcode projects and Windows decline; dynamic dispatch remains unresolved, and `renders` is unsupported. | A missing Package.swift, sidecar, matching Swift/libIndexStore toolchain, explicit output unit, supported platform, successful build or valid frozen generation retains the prior graph and declines the strict route. | `sourcekit-lsp`, then `@samchon/graph-sitter`. | +| `csharp` | `roslyn-workspace` | Dynamic dispatch remains unresolved with compiler-proven candidates; renders is unsupported and other families retain explicit partial coverage. | A missing service/toolchain, ambiguous project selection, absent restore assets, compiler errors or invalid MSBuild load retains the prior strict generation and rejects refresh. | `scip-dotnet`, then `csharp-ls`, then `@samchon/graph-sitter`. | | `python` | `scip-python` | The bundled historical Pyright core proves references only and can recover from malformed pyproject configuration with defaults. | A missing producer/decoder/Python interpreter, absent project input or unusable Python environment declines the strict provider. | `pyright-langserver`, then `@samchon/graph-sitter`. | | `ruby` | `scip-ruby` | The current artifact proves no graph edge family; it does not expose structural coverage, Sorbet sigils or typed unresolved sites. | A missing producer/decoder/Ruby runtime, unusable Bundler environment or invalid project configuration declines the strict provider. | `ruby-lsp`, then `@samchon/graph-sitter`. | | `dart` | `scip-dart` | The current artifact proves no graph edge family and is not resident Analysis Server state. | A missing producer/decoder/Dart SDK, absent package configuration or failed pub resolution declines the strict provider. | Dart Analysis Server, then `@samchon/graph-sitter`. | @@ -166,8 +174,6 @@ These languages are indexed through their ordinary server and static fallback to | Language | Ordinary server | Why no strict provider | Route | | --- | --- | --- | --- | -| `scala` | `metals` | No registered strict provider; scip-java no longer supports Scala. | [tracked route](https://github.com/samchon/compiler-knowledge-graph/issues/77) | -| `swift` | `sourcekit-lsp` | No packaged IndexStoreDB/SourceKit-LSP snapshot producer is registered. | [tracked route](https://github.com/samchon/compiler-knowledge-graph/issues/78) | | `zig` | `zls` | No analyzer or compiler Sema snapshot producer is registered. | [tracked route](https://github.com/samchon/compiler-knowledge-graph/issues/79) | diff --git a/docs/provider-support.json b/docs/provider-support.json index 840e3480..976242af 100644 --- a/docs/provider-support.json +++ b/docs/provider-support.json @@ -101,9 +101,11 @@ "facts": ["contains", "exports", "imports", "calls", "accesses", "instantiates", "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", "tests", "references"], "commands": ["samchon-rust-analyzer", "rust-analyzer"], "environmentOverrides": ["SAMCHON_GRAPH_RUST_ANALYZER_HIR"], - "install": "Build the `samchon/rust-analyzer` graph-snapshot fork at commit `2850ecba80311bebd4cdaa9fedc5321533b5b1e7`; point `SAMCHON_GRAPH_RUST_ANALYZER_HIR` at that binary or install it as `samchon-rust-analyzer`.", + "install": "Build the `samchon/rust-analyzer` graph-snapshot fork at commit `378f220482c298775910f0fc46e8fda1bc516ecc`; point `SAMCHON_GRAPH_RUST_ANALYZER_HIR` at that binary or install it as `samchon-rust-analyzer`.", "installSources": [ - {"label": "native HIR graph producer PR", "url": "https://github.com/samchon/rust-analyzer/pull/1"}, + {"label": "pinned HIR producer commit", "url": "https://github.com/samchon/rust-analyzer/commit/378f220482c298775910f0fc46e8fda1bc516ecc"}, + {"label": "initial native HIR producer PR", "url": "https://github.com/samchon/rust-analyzer/pull/1"}, + {"label": "node ownership repair PR", "url": "https://github.com/samchon/rust-analyzer/pull/2"}, {"label": "rust-analyzer build instructions", "url": "https://rust-analyzer.github.io/book/contributing/"} ], "resolution": "Absolute `SAMCHON_GRAPH_RUST_ANALYZER_HIR`, then project/PATH `samchon-rust-analyzer`, then a project/PATH `rust-analyzer` only when its version reports the pinned producer commit.", @@ -113,7 +115,7 @@ "nativeAnalysis": "The pinned rust-analyzer fork owns one resident HIR database and exports declarations, semantic relationships, diagnostics, coverage and unresolved boundaries from that exact analysis revision.", "exportMerge": "Content-addressed source shards cross a versioned LSP transaction; the client verifies producer identity, universe, complete manifests, shard digests and graph invariants before atomic publication.", "reuseResident": "No-op requests reuse the resident snapshot; interface changes invalidate dependent shards, while a complete consumer checkpoint restores the same generation after process restart.", - "limitations": "The producer is currently available from the draft fork PR rather than a rust-analyzer release, and Rust has no `renders` relationship family.", + "limitations": "The pinned follow-up is available from the fork commit pending human-submitted review rather than a rust-analyzer release, and Rust has no `renders` relationship family.", "decline": "A missing pinned producer, incompatible commit/schema, malformed transaction, invalid checkpoint, unsupported bounded option or failed Cargo workspace load declines this route.", "fallback": "Stock `rust-analyzer-scip`, then generic rust-analyzer, then `@samchon/graph-sitter`.", "experimentLanguages": ["rust"], @@ -133,9 +135,11 @@ "commands": ["samchon-clangd", "clangd"], "projectCommandSources": ["compile_commands.json", "build/compile_commands.json"], "environmentOverrides": ["SAMCHON_GRAPH_CLANGD_SNAPSHOT"], - "install": "Build the `samchon/llvm-project` graph-snapshot fork at commit `e33d8f51552a523b5696691738f1ef95f8e3a730`; point `SAMCHON_GRAPH_CLANGD_SNAPSHOT` at `clangd` or install it as `samchon-clangd`, and provide a compilation database.", + "install": "Build the `samchon/llvm-project` graph-snapshot fork at commit `d6371c37445998d24776692a27e086bb24f9916a`; point `SAMCHON_GRAPH_CLANGD_SNAPSHOT` at `clangd` or install it as `samchon-clangd`, and provide a compilation database.", "installSources": [ - {"label": "native Clang graph producer PR", "url": "https://github.com/samchon/llvm-project/pull/1"}, + {"label": "pinned Clang producer commit", "url": "https://github.com/samchon/llvm-project/commit/d6371c37445998d24776692a27e086bb24f9916a"}, + {"label": "bounded resident-view PR", "url": "https://github.com/samchon/llvm-project/pull/2"}, + {"label": "initial native Clang producer PR", "url": "https://github.com/samchon/llvm-project/pull/1"}, {"label": "LLVM build instructions", "url": "https://llvm.org/docs/CMake.html"} ], "resolution": "Absolute `SAMCHON_GRAPH_CLANGD_SNAPSHOT`, then project/PATH `samchon-clangd`, then a project/PATH `clangd` only when its version reports the pinned producer commit.", @@ -167,21 +171,25 @@ "facts": ["contains", "exports", "imports", "calls", "accesses", "instantiates", "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", "tests", "references"], "commands": ["scip-java", "java"], "environmentOverrides": ["SAMCHON_GRAPH_JAVAC_GRAPH", "SAMCHON_GRAPH_JAVA_TOOLCHAIN"], - "install": "Install a `scip-java` build whose `index` command accepts `--graph-output`, plus a JDK 17+ toolchain. A released build without that option is declined to the SCIP lane rather than run.", + "install": "Build the pinned `scip-java` graph producer at `fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a` and the pinned JDT workspace producer at `0d55a6c13d14e0d0466eeb021920349b3d0c6d35` with JDK 21. A javac build without `--graph-output` is declined before compilation.", "installSources": [ - {"label": "javac graph producer", "url": "https://github.com/samchon/scip-java/pull/1"}, + {"label": "pinned javac graph producer", "url": "https://github.com/samchon/scip-java/commit/fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a"}, + {"label": "initial javac graph implementation", "url": "https://github.com/samchon/scip-java/pull/1"}, + {"label": "producer acceptance pull request", "url": "https://github.com/samchon/scip-java/pull/2"}, + {"label": "pinned JDT workspace producer", "url": "https://github.com/samchon/eclipse.jdt.ls/commit/0d55a6c13d14e0d0466eeb021920349b3d0c6d35"}, + {"label": "JDT workspace producer pull request", "url": "https://github.com/samchon/eclipse.jdt.ls/pull/1"}, {"label": "scip-java", "url": "https://github.com/scip-code/scip-java"} ], - "resolution": "Absolute `SAMCHON_GRAPH_JAVAC_GRAPH`, then a project-local or PATH `scip-java` whose `index --help` publishes `--graph-output`.", - "requirements": "A Maven or Gradle project whose Java compile tasks the producer can attach its javac plugin to, and the JDK those tasks run.", + "resolution": "The build lane resolves absolute `SAMCHON_GRAPH_JAVAC_GRAPH`, then a project-local or PATH `scip-java` whose `index --help` publishes `--graph-output`; on decline, the resident lane resolves absolute `SAMCHON_GRAPH_JDT_WORKSPACE` or `samchon-jdtls`.", + "requirements": "A Maven or Gradle project whose Java compile tasks can host the javac plugin, or a JDT-importable resident workspace; the pinned experiment builds both producers with JDK 21.", "platforms": ["linux", "macos", "windows"], - "mode": "unchanged-input reuse; committed per-target generations with shard-level deltas", - "nativeAnalysis": "A javac plugin walks the attributed tree the project's own compile task already produced, once per compilation unit.", - "exportMerge": "Committed per-target generations become one validated protocol transaction; every edge endpoint, coverage row and target universe is proved before publication.", - "reuseResident": "The build tool's incremental compilation decides what is rewritten; unchanged shards are carried forward instead of resent, and a moved universe or target set publishes a whole generation.", - "limitations": "javac's diagnostics stay with the build that emitted them, so this route carries none and says so through its capabilities. The JDT resident workspace lane remains separate work.", - "decline": "A missing producer or JDK, a launcher without `--graph-output`, a producer that does not commit atomic generations, an incomplete coverage matrix, a foreign project root or a bounded request declines the strict provider.", - "fallback": "`scip-java`, then `jdtls`, then `@samchon/graph-sitter`.", + "mode": "javac reuses unchanged inputs and commits per-target shard deltas; JDT retains one resident whole-workspace generation and reports initial, unchanged, incremental, reload or error", + "nativeAnalysis": "The javac plugin walks each attributed compilation unit once inside the project's compile task; the JDT command reconciles each resident compilation unit once and emits one bulk snapshot.", + "exportMerge": "javac target generations and the JDT workspace generation each become one validated protocol transaction; JDT is a fallback semantic owner, never an enrichment pass after javac.", + "reuseResident": "Build-tool incremental compilation carries unchanged javac shards forward. JDT keeps the workspace process resident, freezes source buffers under one read boundary and retains the prior strict generation on errors.", + "limitations": "javac diagnostics stay with the build that emitted them and are not carried by this route. JDT carries resident diagnostics and unsaved-buffer digests but currently proves containment only; standard Gradle main/test/subproject sources share javac task identities, while custom source-set tasks retain JDT-local scope. Its producer remains a pinned fork draft.", + "decline": "A missing producer or JDK, a launcher without `--graph-output`, incompatible schema or coverage, a foreign root, a bounded request, an incomplete JDT generation or moved inputs behind a reused generation declines the affected strict lane.", + "fallback": "Pinned `jdt-workspace`, then `scip-java`, then generic `jdtls`, then `@samchon/graph-sitter`.", "experimentLanguages": ["java"], "experimentTool": "scip-java-javac-graph", "experimentCapabilities": ["coverage", "diskDigests", "incremental", "sourceDigests", "universe", "unresolved"], @@ -191,64 +199,131 @@ "childIssues": ["https://github.com/samchon/compiler-knowledge-graph/issues/74"] }, { - "provider": "scip-kotlinc", + "provider": "kotlinc-graph", "languages": ["kotlin"], "status": "registered", - "authority": "semantic-index", - "facts": ["contains", "references"], - "commands": ["scip-java", "scip", "java"], - "environmentOverrides": ["SAMCHON_GRAPH_SCIP_JAVA", "SAMCHON_GRAPH_SCIP", "SAMCHON_GRAPH_JAVA_TOOLCHAIN"], - "install": "Install `scip-java` 0.13.1, the `scip` decoder and a compatible JDK; Kotlin experiments pin a compatible source build.", + "authority": "compiler", + "facts": ["contains", "exports", "imports", "calls", "accesses", "instantiates", "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", "tests", "references"], + "commands": ["scip-java", "java"], + "environmentOverrides": ["SAMCHON_GRAPH_KOTLINC_GRAPH", "SAMCHON_GRAPH_JAVA_TOOLCHAIN"], + "install": "Build the pinned Kotlin 2.3.20 graph producer at `3a1565d0647d89a28880fa40ecbef0966a1a328c` with JDK 21. The indexed project supplies its Gradle wrapper when present; otherwise the resident producer uses pinned Gradle 9.4.1.", "installSources": [ - {"label": "scip-java 0.13.1 release", "url": "https://github.com/scip-code/scip-java/releases/tag/v0.13.1"}, - {"label": "SCIP releases", "url": "https://github.com/sourcegraph/scip/releases"} + {"label": "pinned Kotlin graph producer", "url": "https://github.com/samchon/scip-java/commit/3a1565d0647d89a28880fa40ecbef0966a1a328c"}, + {"label": "Kotlin graph producer pull request", "url": "https://github.com/scip-code/scip-java/pull/1006"}, + {"label": "Kotlin Gradle compiler options", "url": "https://kotlinlang.org/docs/gradle-compiler-options.html"} ], - "resolution": "Project-local producer/decoder/JDK precede PATH; absolute environment overrides select each tool.", - "requirements": "Maven or Gradle project metadata, dependency/classpath state and Kotlin compiler inputs.", + "resolution": "Absolute `SAMCHON_GRAPH_KOTLINC_GRAPH`, then project-local or PATH `scip-java`; the launcher must publish both `--kotlin-graph-output` and the resident `kotlin-graph-server` protocol before selection.", + "requirements": "A Kotlin/JVM Gradle project using Kotlin 2.3.20, a compatible JDK, resolvable dependencies and the complete build/classpath inputs for every compiled target.", "platforms": ["linux", "macos", "windows"], - "mode": "unchanged-snapshot-reuse; full-rebuild-on-change", - "nativeAnalysis": "scip-java drives the selected Maven or Gradle build and its kotlinc producer as one batch.", - "exportMerge": "The complete decoded artifact is merged as a contains/references graph before atomic publication.", - "reuseResident": "Unchanged inputs reuse the validated snapshot; no kotlinc or build session remains resident.", - "limitations": "The released Gradle path disables configuration cache and runs `clean scipCompileAll`; it is navigation fallback for #76, not compiler-owned calls or accesses. The producer does not expose the kotlinc revision the indexed build selected, so the compiler row stays empty rather than naming the JVM that launched it.", - "decline": "A missing producer/decoder/JDK, unsupported Maven or Gradle project, or invalid dependency/build configuration declines the strict provider.", - "fallback": "`kotlin-language-server`, then `@samchon/graph-sitter`.", + "mode": "resident Gradle connection; per-target content-addressed shard generations; atomic incremental replacement", + "nativeAnalysis": "The K2 plugin traverses FIR and IR once inside each ordinary Kotlin Gradle compile task and records compiler-resolved declarations, calls, accesses, types, dispatch, annotations, tests and diagnostics.", + "exportMerge": "The consumer verifies producer/compiler identity, target universes, coverage, unresolved sites, source and disk digests, edge endpoints and complete target manifests before one atomic graph publication.", + "reuseResident": "Exact no-op requests reuse the validated in-memory snapshot without invoking Gradle. Changed source or build inputs reuse one persistent Gradle Tooling connection, KGP daemon, configuration, classpath and incremental caches while carrying unchanged compiler shards forward.", + "limitations": "The compiler-owned route is intentionally limited to Gradle Kotlin/JVM on Kotlin 2.3.20. Maven, Android, multiplatform, JS, Native, newer or older compiler minors and bounded indexing options are declined instead of being mislabeled as compiler-owned facts.", + "decline": "A missing compatible producer/JDK/Gradle build, absent resident capability, unsupported target kind/compiler minor, incomplete or conflicting shard generation, source/configuration drift, compiler diagnostics that stop publication or malformed artifact retains the prior strict generation and declines this route.", + "fallback": "`scip-kotlinc`, then `kotlin-language-server`, then `@samchon/graph-sitter`.", "experimentLanguages": ["kotlin"], - "experimentTool": "scip-java", - "experimentCapabilities": ["universe", "diskDigests"], + "experimentTool": "scip-kotlinc-k2-graph", + "experimentCapabilities": ["coverage", "diagnostics", "diskDigests", "incremental", "sourceDigests", "universe", "unresolved"], "benchmarkProvider": "scip-java", "benchmarks": [{"project": "koin", "strictMs": 211263.800455, "fallbackMs": 967711.761431}], - "upstream": "https://github.com/scip-code/scip-java", + "upstream": "https://github.com/samchon/scip-java", "childIssues": ["https://github.com/samchon/compiler-knowledge-graph/issues/76"] }, { - "provider": "scip-dotnet", + "provider": "scalac-graph", + "languages": ["scala"], + "status": "registered", + "authority": "compiler", + "facts": ["contains", "exports", "imports", "calls", "accesses", "instantiates", "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", "references"], + "commands": ["samchon-scala-graph", "java"], + "environmentOverrides": ["SAMCHON_GRAPH_SCALA_GRAPH", "SAMCHON_GRAPH_JAVA_TOOLCHAIN"], + "install": "Build the shipped BSP producer with `mvn --batch-mode --file sidecars/scala/pom.xml verify`, expose its server jar as `samchon-scala-graph`, and configure the matching Scala 2 or Scala 3 typed plugin together with SemanticDB in each indexed target.", + "installSources": [ + {"label": "shipped Scala graph source", "url": "https://github.com/samchon/compiler-knowledge-graph/tree/master/sidecars/scala"}, + {"label": "pinned Scala 2/3 BSP fixture", "url": "https://github.com/samchon/graph-benchmark-scala"}, + {"label": "Build Server Protocol", "url": "https://build-server-protocol.github.io/docs/specification"}, + {"label": "SemanticDB specification", "url": "https://scalameta.org/docs/semanticdb/specification.html"} + ], + "resolution": "Absolute `SAMCHON_GRAPH_SCALA_GRAPH`, then project-local or PATH `samchon-scala-graph`; selection also requires a usable `.bsp/*.json` connection and the resident `graph-server` capability.", + "requirements": "BSP targets whose scalac options load the matching Scala 2/3 typed plugin and emit SemanticDB in the same Zinc-controlled compile.", + "platforms": ["linux", "macos", "windows"], + "mode": "resident BSP connection; per-target SemanticDB and typed-plugin shards; atomic Zinc incremental replacement", + "nativeAnalysis": "The repository's BSP server asks Zinc to compile; separate Scala 2 and Scala 3 post-typer plugins emit resolved roles during that same compile while SemanticDB supplies declaration and diagnostic cross-checks.", + "exportMerge": "The consumer verifies BSP target universes, compiler/plugin pairing, SemanticDB URI, md5 and build target, complete coverage, source digests and target manifests before one atomic publication.", + "reuseResident": "Exact no-op requests reuse the validated snapshot. Changed inputs use the same BSP server and Zinc analysis, replace emitted shards and carry validated unchanged shards forward without `clean`.", + "limitations": "Targets without both the matching typed plugin and SemanticDB decline. `renders` and `tests` need framework-specific enrichers and are unsupported; Metals and static extraction remain the ordinary fallbacks.", + "decline": "A missing BSP connection, producer, Java runtime, compatible compiler/plugin pair or SemanticDB output, or a malformed/stale target generation declines without replacing the prior graph.", + "fallback": "`metals`, then `@samchon/graph-sitter`; current `scip-java` is never presented as Scala-capable.", + "experimentLanguages": ["scala"], + "experimentTool": "samchon-scala-graph", + "experimentCapabilities": ["coverage", "diagnostics", "diskDigests", "incremental", "sourceDigests", "universe", "unresolved", "bsp", "semanticdb", "typedPlugins", "zinc"], + "benchmarks": [], + "benchmarkPending": "The new strict provider has no cell in the immutable pre-provider benchmark artifact; its native BSP and strict measurements are collected by the issue #77 acceptance experiment before publication.", + "upstream": "https://build-server-protocol.github.io/docs/specification", + "childIssues": ["https://github.com/samchon/compiler-knowledge-graph/issues/77"] + }, + { + "provider": "swift-indexstore", + "languages": ["swift"], + "status": "registered", + "authority": "compiler", + "facts": ["contains", "exports", "imports", "calls", "accesses", "instantiates", "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", "tests", "references"], + "commands": ["samchon-swift-graph", "swift"], + "environmentOverrides": ["SAMCHON_GRAPH_SWIFT_GRAPH", "SAMCHON_GRAPH_SWIFT_TOOLCHAIN"], + "install": "Build the shipped SwiftPM sidecar with Swift 6.0 or newer, expose `samchon-swift-graph` on PATH, or point `SAMCHON_GRAPH_SWIFT_GRAPH` at the binary. The package pins the Swift 6.1 IndexStoreDB release commit that remains source-compatible with Swift 6.0 and fixes Clang 19's 64-bit role declaration.", + "installSources": [ + {"label": "shipped Swift graph source", "url": "https://github.com/samchon/compiler-knowledge-graph/tree/master/sidecars/swift"}, + {"label": "pinned IndexStoreDB commit", "url": "https://github.com/swiftlang/indexstore-db/commit/54212fce1aecb199070808bdb265e7f17e396015"}, + {"label": "IndexStoreDB explicit output units", "url": "https://github.com/swiftlang/indexstore-db/blob/54212fce1aecb199070808bdb265e7f17e396015/Sources/IndexStoreDB/IndexStoreDB.swift"} + ], + "resolution": "Absolute `SAMCHON_GRAPH_SWIFT_GRAPH`, then project-local or PATH `samchon-swift-graph`; selection requires Package.swift, macOS or Linux, a matching libIndexStore, project support and the resident sidecar protocol.", + "requirements": "A SwiftPM package, Swift 6.0-compatible toolchain and its libIndexStore on macOS or Linux.", + "platforms": ["linux", "macos"], + "mode": "resident sidecar process; native SwiftPM incremental builds; explicit-output-unit IndexStoreDB generations", + "nativeAnalysis": "The package's ordinary `swift build --enable-index-store --build-tests -Xswiftc -index-include-locals` produces compiler records; the sidecar takes the exact source and object paths from SwiftPM's current build description, opens a fresh explicit-output-unit IndexStoreDB view and reads compiler USRs, roles and relations.", + "exportMerge": "The consumer verifies the exact sorted output-unit paths and digests, module/triple/configuration universe, one-pass source enrichment, source digests, coverage and unresolved boundaries before atomic publication.", + "reuseResident": "Exact no-op requests reuse the validated in-memory snapshot. Changed requests reuse SwiftPM build products but reconstruct the frozen IndexStoreDB view; this does not claim SourceKit-LSP cache or scheduler ownership.", + "limitations": "This is the standalone SwiftPM fallback, not the preferred in-process SourceKit-LSP hook. Xcode projects and Windows decline; dynamic dispatch remains unresolved, and `renders` is unsupported.", + "decline": "A missing Package.swift, sidecar, matching Swift/libIndexStore toolchain, explicit output unit, supported platform, successful build or valid frozen generation retains the prior graph and declines the strict route.", + "fallback": "`sourcekit-lsp`, then `@samchon/graph-sitter`.", + "experimentLanguages": ["swift"], + "experimentTool": "samchon-swift-graph", + "experimentCapabilities": ["coverage", "diagnostics", "diskDigests", "incremental", "sourceDigests", "universe", "unresolved", "explicitOutputUnits", "indexStoreDB", "sourceEnrichment", "swiftpm"], + "benchmarks": [], + "benchmarkPending": "The new standalone provider has no cell in the immutable pre-provider benchmark artifact; issue #78's SwiftPM lifecycle compares its cold generation with the same native build.", + "upstream": "https://github.com/swiftlang/indexstore-db", + "childIssues": ["https://github.com/samchon/compiler-knowledge-graph/issues/78"] + }, + { + "provider": "roslyn-workspace", "languages": ["csharp"], "status": "registered", - "authority": "semantic-index", - "facts": [], - "commands": ["scip-dotnet", "scip", "dotnet"], - "environmentOverrides": ["SAMCHON_GRAPH_SCIP_DOTNET", "SAMCHON_GRAPH_SCIP", "SAMCHON_GRAPH_DOTNET_TOOLCHAIN"], - "install": "`dotnet tool install --global scip-dotnet`; install the `scip` decoder and matching .NET SDK.", + "authority": "compiler", + "facts": ["contains", "exports", "imports", "calls", "accesses", "instantiates", "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", "tests", "references"], + "commands": ["samchon-roslyn", "dotnet"], + "environmentOverrides": ["SAMCHON_GRAPH_ROSLYN_WORKSPACE", "SAMCHON_GRAPH_DOTNET_TOOLCHAIN"], + "install": "Install a matching .NET SDK; the package ships the pinned Roslyn service source, or `SAMCHON_GRAPH_ROSLYN_WORKSPACE` may select a prebuilt service.", "installSources": [ - {"label": "scip-dotnet on NuGet", "url": "https://www.nuget.org/packages/scip-dotnet"}, - {"label": "SCIP releases", "url": "https://github.com/sourcegraph/scip/releases"} + {"label": "Roslyn workspace model", "url": "https://learn.microsoft.com/en-us/dotnet/csharp/roslyn-sdk/work-with-workspace"}, + {"label": "Microsoft.CodeAnalysis.Workspaces.MSBuild", "url": "https://www.nuget.org/packages/Microsoft.CodeAnalysis.Workspaces.MSBuild"} ], - "resolution": "Project-local producer/decoder/toolchain precede PATH; absolute environment overrides select each tool.", - "requirements": "Solution/project/TFM/NuGet/MSBuild inputs and a resolvable SDK.", + "resolution": "A prebuilt service override precedes the shipped source; the selected dotnet toolchain builds that source when no service binary resolves.", + "requirements": "One unambiguous .sln, .slnx or .csproj selection, restored assets and a matching SDK/MSBuild instance.", "platforms": ["linux", "macos", "windows"], - "mode": "unchanged-snapshot-reuse; full-rebuild-on-change", - "nativeAnalysis": "scip-dotnet loads and analyzes the selected solution through one batch producer run.", - "exportMerge": "The complete decoded artifact publishes declarations but no currently defensible edge family.", - "reuseResident": "Unchanged inputs reuse the validated snapshot; no Roslyn workspace remains resident.", - "limitations": "The artifact proves declarations but no graph edge family and can log MSBuildWorkspace failures after publishing.", - "decline": "A missing producer/decoder/.NET SDK, absent solution/project input or invalid MSBuild load declines the strict provider.", - "fallback": "`csharp-ls`, then `@samchon/graph-sitter`.", + "mode": "resident-immutable-solution; shard-delta; atomic-reload", + "nativeAnalysis": "One resident MSBuildWorkspace captures immutable Solution generations and Roslyn symbols, semantic models and operations.", + "exportMerge": "Document, generated-source and project metadata shards carry compiler-resolved declarations, relationships, diagnostics, coverage and source digests.", + "reuseResident": "No-op refreshes reuse the exact committed generation; source edits update the immutable Solution and build-input changes reload it atomically.", + "limitations": "Dynamic dispatch remains unresolved with compiler-proven candidates; renders is unsupported and other families retain explicit partial coverage.", + "decline": "A missing service/toolchain, ambiguous project selection, absent restore assets, compiler errors or invalid MSBuild load retains the prior strict generation and rejects refresh.", + "fallback": "`scip-dotnet`, then `csharp-ls`, then `@samchon/graph-sitter`.", "experimentLanguages": ["csharp"], - "experimentTool": "scip-dotnet", - "experimentCapabilities": ["universe", "diskDigests"], + "experimentTool": "samchon-roslyn", + "experimentCapabilities": ["universe", "diskDigests", "incremental", "immutableSolution", "sourceGeneratedDocuments"], + "benchmarkProvider": "scip-dotnet", "benchmarks": [{"project": "serilog", "strictMs": 20498.323945, "fallbackMs": 25085.071148}], - "upstream": "https://github.com/sourcegraph/scip-dotnet", + "upstream": "https://github.com/dotnet/roslyn", "childIssues": ["https://github.com/samchon/compiler-knowledge-graph/issues/75"] }, { @@ -373,18 +448,6 @@ } ], "ordinaryOnly": [ - { - "language": "scala", - "server": "metals", - "issue": "https://github.com/samchon/compiler-knowledge-graph/issues/77", - "reason": "No registered strict provider; scip-java no longer supports Scala." - }, - { - "language": "swift", - "server": "sourcekit-lsp", - "issue": "https://github.com/samchon/compiler-knowledge-graph/issues/78", - "reason": "No packaged IndexStoreDB/SourceKit-LSP snapshot producer is registered." - }, { "language": "zig", "server": "zls", diff --git a/packages/graph/build/copy-sidecars.mjs b/packages/graph/build/copy-sidecars.mjs index e3db1e6b..80bc88bb 100644 --- a/packages/graph/build/copy-sidecars.mjs +++ b/packages/graph/build/copy-sidecars.mjs @@ -5,10 +5,10 @@ import { fileURLToPath } from "node:url"; /** * Copy the sidecar sources this package ships into the package itself. * - * The Go sidecar is source a user compiles into `samchon-graph-go`; the Gradle - * Java source reads the opted-in Tooling API model; and the Lua exporter is a - * script the provider hands to lua-language-server at run time. All three must - * exist in an installed package rather than only in this repository. + * The Go, C#, Scala, and Swift sidecars are source a user compiles into their + * native producer; the Gradle Java source reads the opted-in Tooling API model; and + * the Lua exporter is a script the provider hands to lua-language-server. All + * must exist in an installed package rather than only in this repository. * * Named per file rather than copied wholesale. A directory copy would ship * whatever happened to be sitting there — a probe, a scratch file, a build @@ -21,6 +21,14 @@ const packageRoot = path.resolve( const repositoryRoot = path.resolve(packageRoot, "..", ".."); const SIDECARS = { + csharp: [ + "GraphExtractor.cs", + "GraphProtocol.cs", + "Program.cs", + "Samchon.Graph.CSharp.csproj", + "WorkspaceGraphService.cs", + "packages.lock.json", + ], gradle: ["RepositoryContext.java"], go: [ "analyze.go", @@ -36,6 +44,44 @@ const SIDECARS = { // `probe.lua` is deliberately absent: it is the research instrument that // established what the engine exposes, not something a user runs. lua: ["export.lua"], + scala: [ + "README.md", + "pom.xml", + "common/pom.xml", + "common/src/main/java/org/samchon/graph/scala/model/Evidence.java", + "common/src/main/java/org/samchon/graph/scala/model/GraphEdge.java", + "common/src/main/java/org/samchon/graph/scala/model/GraphNode.java", + "common/src/main/java/org/samchon/graph/scala/model/TypedShard.java", + "common/src/main/java/org/samchon/graph/scala/model/UnresolvedSite.java", + "common/src/main/java/org/samchon/graph/scala/plugin/GraphShardWriter.java", + "common/src/main/java/org/samchon/graph/scala/plugin/PluginOptions.java", + "scala2-plugin/pom.xml", + "scala2-plugin/src/main/resources/scalac-plugin.xml", + "scala2-plugin/src/main/scala/org/samchon/graph/scala2/Scala2GraphPlugin.scala", + "scala3-plugin/pom.xml", + "scala3-plugin/src/main/resources/plugin.properties", + "scala3-plugin/src/main/scala/org/samchon/graph/scala3/Scala3GraphPlugin.scala", + "server/pom.xml", + "server/src/main/java/org/samchon/graph/scala/server/model/DiagnosticFact.java", + "server/src/main/java/org/samchon/graph/scala/server/model/SemanticShard.java", + "server/src/main/java/org/samchon/graph/scala/server/model/SnapshotArtifact.java", + "server/src/main/java/org/samchon/graph/scala/server/model/TargetSnapshot.java", + "server/src/main/scala/org/samchon/graph/scala/server/AtomicJson.scala", + "server/src/main/scala/org/samchon/graph/scala/server/BspSession.scala", + "server/src/main/scala/org/samchon/graph/scala/server/Main.scala", + "server/src/main/scala/org/samchon/graph/scala/server/SemanticDbReader.scala", + "server/src/main/scala/org/samchon/graph/scala/server/SnapshotProducer.scala", + ], + swift: [ + "README.md", + "Package.resolved", + "Package.swift", + "Sources/SamchonSwiftGraph/GraphModel.swift", + "Sources/SamchonSwiftGraph/main.swift", + "Sources/SamchonSwiftGraph/SHA256.swift", + "Sources/SamchonSwiftGraph/SourceEnrichment.swift", + "Sources/SamchonSwiftGraph/SwiftGraphProducer.swift", + ], }; for (const [sidecar, files] of Object.entries(SIDECARS)) { @@ -48,6 +94,8 @@ for (const [sidecar, files] of Object.entries(SIDECARS)) { if (!fs.existsSync(from)) { throw new Error(`sidecar source is missing: ${from}`); } - fs.copyFileSync(from, path.join(target, file)); + const to = path.join(target, file); + fs.mkdirSync(path.dirname(to), { recursive: true }); + fs.copyFileSync(from, to); } } diff --git a/packages/graph/build/provider-support.mjs b/packages/graph/build/provider-support.mjs index 0ed75457..703f928f 100644 --- a/packages/graph/build/provider-support.mjs +++ b/packages/graph/build/provider-support.mjs @@ -313,9 +313,14 @@ function validateManifest( } invariant( - Array.isArray(documented.benchmarks) && - documented.benchmarks.length > 0, - `${documented.provider} must name benchmark evidence`, + Array.isArray(documented.benchmarks), + `${documented.provider} benchmark evidence must be a list`, + ); + invariant( + documented.benchmarks.length > 0 || + (typeof documented.benchmarkPending === "string" && + documented.benchmarkPending.trim() !== ""), + `${documented.provider} must name benchmark evidence or its pending boundary`, ); if (documented.benchmarkProvider !== undefined) { invariant( diff --git a/packages/graph/src/SamchonGraphApplication.ts b/packages/graph/src/SamchonGraphApplication.ts index 3de9e884..e4e862b2 100644 --- a/packages/graph/src/SamchonGraphApplication.ts +++ b/packages/graph/src/SamchonGraphApplication.ts @@ -13,6 +13,7 @@ import { runTour } from "./operations/runTour"; import { runTrace } from "./operations/runTrace"; import { SamchonGraphMemory } from "./SamchonGraphMemory"; import { SamchonRepositoryContextMemory } from "./repository"; +import { topologyPhaseTrace } from "./repository/topologyPhaseTrace"; import { ISamchonGraphApplication, ISamchonGraphEscape } from "./structures"; /** @@ -137,6 +138,8 @@ export class SamchonGraphApplication implements ISamchonGraphApplication { ); } const topology = await this.topology(); + const tracing = process.env.SAMCHON_GRAPH_TOPOLOGY_TRACE === "1"; + const joinStarted = tracing ? performance.now() : 0; const confirmed = await this.load(); const compatible = graph.project === topology.dump.project && @@ -176,15 +179,20 @@ export class SamchonGraphApplication implements ISamchonGraphApplication { ? "This code graph carries no input generation to fence against: a graph file served without revalidation withholds one, and dumps written before cross-plane fencing never had one." : "The code generation moved while topology was loading.", }; - const result = topology.inspect( - props.request, - join, - new Set( - graph.nodes - .filter((node) => node.kind === "file") - .map((node) => node.file), - ), + const codeFiles = new Set( + graph.nodes + .filter((node) => node.kind === "file") + .map((node) => node.file), ); + const result = topology.inspect(props.request, join, codeFiles); + if (tracing) { + topologyPhaseTrace("repository-context", "join", joinStarted, { + codeFiles: codeFiles.size, + nodes: result.nodes.length, + edges: result.edges.length, + compatible, + }); + } return { audit: "Repository topology is returned from declared or owning-tool models; file joins are included only when the code generation stayed stable across the topology load.", diff --git a/packages/graph/src/indexer/createResidentGraphSource.ts b/packages/graph/src/indexer/createResidentGraphSource.ts index e5c6c6ed..72219896 100644 --- a/packages/graph/src/indexer/createResidentGraphSource.ts +++ b/packages/graph/src/indexer/createResidentGraphSource.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import fs from "node:fs"; import path from "node:path"; import { ISamchonGraphDiagnostic, @@ -16,6 +17,7 @@ import { dumpProvenanceOf } from "../provider/dumpProvenanceOf"; import { fallbackCoverage } from "../provider/fallbackCoverage"; import { graphCoverageOf } from "../provider/graphCoverageOf"; import { graphUnresolvedOf } from "../provider/graphUnresolvedOf"; +import { graphSnapshotDigests } from "../provider/graphSnapshotDigests"; import { IGraphProvider } from "../provider/IGraphProvider"; import { GRAPH_PROVIDERS } from "../provider/GRAPH_PROVIDERS"; import { IBulkGraphSession } from "../provider/IBulkGraphSession"; @@ -71,6 +73,7 @@ interface IResidentState { inputManifest: Map; inputGeneration: string; buildInputs: string[]; + sourceFiles: string[]; providerTopology: string; /** @@ -85,6 +88,11 @@ interface IResidentState { providerTopologyRows: readonly providerTopology.IRow[]; /** An available strict candidate fell back while this state was built. */ providerFallback: boolean; + + /** Last fact proof accepted from each resident bulk session. */ + bulkFactDigests: Map; + bulkWarnings: Map; + bulkSourceFiles: Map; } interface IResidentDependencies { @@ -182,8 +190,25 @@ export function createResidentGraphSource( dependencies.providers ?? [], served, ); + const inputGeneration = + result.inputGeneration ?? + projectInputGeneration({ + sourceFiles: selected.files, + buildInputFiles: buildInputs.map((input) => + path.resolve(root, input), + ), + manifest: inputManifest, + consumedSources: texts, + providerSources: providerSourcesOf(providerSnapshots), + provenance: + result.dump.provenance ?? + providerSnapshots.map(dumpProvenanceOf), + }); return { - dump: result.dump, + dump: + result.dump.generation === undefined + ? { ...result.dump, generation: { input: inputGeneration } } + : result.dump, sessions, generations: bulkGenerationsOf(sessions), staticLanguages: staticLanguagesOf(result.dump, sessions), @@ -196,21 +221,9 @@ export function createResidentGraphSource( modes: result.modes ?? new Map(), providers: result.providers ?? new Map(), inputManifest, - inputGeneration: - result.inputGeneration ?? - projectInputGeneration({ - sourceFiles: selected.files, - buildInputFiles: buildInputs.map((input) => - path.resolve(root, input), - ), - manifest: inputManifest, - consumedSources: texts, - providerSources: providerSourcesOf(providerSnapshots), - provenance: - result.dump.provenance ?? - providerSnapshots.map(dumpProvenanceOf), - }), + inputGeneration, buildInputs, + sourceFiles: [...selected.files], providerTopology: providerTopology.serialize(availableTopology), providerTopologyRows: availableTopology, providerFallback: availableTopology.some((row) => @@ -220,6 +233,9 @@ export function createResidentGraphSource( row.provider, ), ), + bulkFactDigests: bulkFactDigestsOf(sessions), + bulkWarnings: bulkWarningsOf(sessions), + bulkSourceFiles: bulkSourceFilesOf(sessions), }; } catch (error) { // Once the build hands its sessions to this source, every later failure @@ -465,6 +481,117 @@ export function createResidentGraphSource( current.source = sourceReaderOf(root, sources, current.sessions); current.inputManifest = committedInputs; current.inputGeneration = inputGeneration; + current.sourceFiles = [...committedSelection.files]; + current.bulkFactDigests = bulkFactDigestsOf(current.sessions); + current.bulkWarnings = bulkWarningsOf(current.sessions); + current.bulkSourceFiles = bulkSourceFilesOf(current.sessions); + } + + function commitFactEquivalentBulkRefresh( + current: IResidentState, + prefetched: ReadonlyMap, + signal: AbortSignal, + ): boolean { + // A single fixed bulk owner can prove that a new compiler generation has + // exactly the prior graph facts. Under that proof, unchanged universe, + // warnings, and source membership make the normal graph merge, topology + // discovery, and whole-checkout hash walk redundant. We still hash every + // provider-reported tracked disk source and recompute the project input + // generation before publishing the new manifest envelope. Every tracked + // provider source is read at that fence, including sources whose digest did + // not move, because an unrelated file can change after the provider took + // its immutable snapshot. Mixed, discovery-driven, or structurally changed + // projects return false and use the full coordinator transaction below. + if (!entirelyBulkOwned(options.languages, current.sessions)) return false; + const sessions = [ + ...new Set( + [...current.sessions.values()].filter(isBulkGraphSession), + ), + ]; + if (sessions.length !== 1) return false; + const session = sessions[0]!; + const refresh = [...prefetched] + .filter(([language]) => current.sessions.get(language) === session) + .map(([, candidate]) => candidate)[0]; + if (refresh?.changed !== true) return false; + const snapshot = refresh.snapshot; + const protocol = snapshot.protocol; + const priorProvenance = current.dump.provenance; + if ( + protocol === undefined || + current.bulkFactDigests.get(session) !== protocol.factDigest || + priorProvenance?.length !== 1 || + priorProvenance[0]!.provider !== snapshot.provenance.provider || + priorProvenance[0]!.universe !== snapshot.provenance.universe || + // A stored fact digest means this session had a current snapshot when + // all three bookkeeping maps were captured, so these rows exist together. + !sameStringArray(current.bulkWarnings.get(session)!, snapshot.warnings) || + !sameStringArray( + current.bulkSourceFiles.get(session)!, + [...snapshot.sources.keys()].sort(compareText), + ) + ) { + return false; + } + + assertOpen(); + if (session.current !== snapshot) { + throw new StaleCandidateError( + `@samchon/graph: the ${snapshot.provenance.provider} provider replaced its fact-equivalent snapshot while this refresh was preparing`, + ); + } + const committedInputs = new Map(current.inputManifest); + for (const [file, digest] of snapshot.sources) { + if (!committedInputs.has(file)) continue; + if (digest.diskDigest === "" || fileDigest(file) !== digest.diskDigest) { + throw new StaleCandidateError( + `@samchon/graph: ${file} moved after the provider prepared its fact-equivalent generation`, + ); + } + committedInputs.set(file, digest.diskDigest); + } + const providerSources = providerSourcesOf([snapshot]); + const movement = movedProviderSource( + providerSources, + committedInputs, + committedInputs, + ); + if (movement !== undefined) { + throw new StaleCandidateError( + `@samchon/graph: ${movement}, so no fact-equivalent slice may be published`, + ); + } + if (signal.aborted) throw closedError(); + + const provenance = [ + { + ...priorProvenance[0]!, + manifest: graphSnapshotDigests.manifestOf(snapshot), + }, + ]; + const inputGeneration = projectInputGeneration({ + sourceFiles: current.sourceFiles, + buildInputFiles: current.buildInputs.map((input) => + path.resolve(root, input), + ), + manifest: committedInputs, + providerSources, + provenance, + }); + current.dump = { + ...current.dump, + generation: { input: inputGeneration }, + provenance, + }; + current.generations = bulkGenerationsOf(current.sessions); + current.modes = bulkModesOf(prefetched); + current.source = sourceReaderOf(root, new Map(), current.sessions); + current.inputManifest = committedInputs; + current.inputGeneration = inputGeneration; + current.bulkFactDigests = bulkFactDigestsOf(current.sessions); + current.bulkWarnings = bulkWarningsOf(current.sessions); + current.bulkSourceFiles = bulkSourceFilesOf(current.sessions); + return true; } async function replaceLanguages( @@ -505,6 +632,55 @@ export function createResidentGraphSource( signal: AbortSignal, ): Promise { for (let attempt = 1; ; attempt++) { + let phaseStarted = performance.now(); + const prefetched = await refreshBulkSessions(current.sessions, signal); + traceResident("prefetch", phaseStarted); + const bulkChanged = [...prefetched].some( + ([language, refresh]) => + refresh.changed || + current.generations.get(language) !== refresh.generation, + ); + const entirelyBulk = entirelyBulkOwned( + options.languages, + current.sessions, + ); + const topologyOwned = + entirelyBulk && + [...new Set(current.sessions.values())].every( + (session) => + "kind" in session && + session.kind === "bulk" && + session.ownsProviderTopology === true, + ); + if ( + bulkChanged && + commitFactEquivalentBulkRefresh(current, prefetched, signal) + ) { + traceResident("fact-equivalent-commit", phaseStarted); + return; + } + if ( + !bulkChanged && + entirelyBulk + ) { + // An explicitly selected, wholly compiler-owned project has one + // authority for source membership and freshness: its resident + // provider. Re-walking and hashing the checkout before and after an + // unchanged provider generation asks the filesystem the same question + // twice, costs seconds on large native corpora, and can only observe + // bytes the compiler did not resolve. `IRefresh` also requires an + // unchanged answer to reuse its prior immutable snapshot verbatim, so + // its payload and slice contract were already validated when that + // generation was committed. Mixed and discovery-driven projects + // continue through the coordinator fence below. + current.modes = bulkModesOf(prefetched); + return; + } + const bulkSliceLanguagesChanged = !sameBulkSliceLanguages( + current.sessions, + prefetched, + current.providers, + ); const selected = selectGraphSources(root, options); const liveBuildInputs = residentBuildInputs( selected.languages, @@ -519,39 +695,53 @@ export function createResidentGraphSource( // project does not use is another candidate to probe, and a // half-installed toolchain fails intermittently for the same reason it is // not serving. - const liveRows = providerTopology.reestablish( - providerTopology.available( - root, - selected.presentLanguages, - options, - process.env, - dependencies.providers ?? [], - new Set( - [...current.providers.values()].map((provider) => provider.name), + // A live bulk session already owns its compiler process and publishes + // producer/toolchain identity with each generation. Re-running command + // discovery and help/version probes after that same session reports a + // changed generation cannot alter which process produced it; it only + // adds process-launch latency before merging the candidate. Mixed and + // generic topologies still need the fresh availability comparison. + if (!topologyOwned) { + phaseStarted = performance.now(); + const liveRows = providerTopology.reestablish( + providerTopology.available( + root, + selected.presentLanguages, + options, + process.env, + dependencies.providers ?? [], + new Set( + [...current.providers.values()].map( + (provider) => provider.name, + ), + ), ), - ), - current.providerTopologyRows, - ); - const liveTopology = providerTopology.serialize(liveRows); - if (liveTopology !== current.providerTopology) { - await replaceLanguages(current, signal); - return; + current.providerTopologyRows, + ); + traceResident("topology", phaseStarted); + const liveTopology = providerTopology.serialize(liveRows); + if (liveTopology !== current.providerTopology) { + await replaceLanguages(current, signal); + return; + } + // A stable private tool identity can move while its public version row + // remains equal. Keep the fresh evidence even when serialized topology + // did not move, or the next transient failure would be matched against + // a tool path the provider no longer uses. + current.providerTopologyRows = liveRows; } - // A stable private tool identity can move while its public version row - // remains equal. Keep the fresh evidence even when serialized topology - // did not move, or the next transient failure would be matched against a - // tool path the provider no longer uses. - current.providerTopologyRows = liveRows; if (!sameStringArray(current.buildInputs, liveBuildInputs)) { await replaceLanguages(current, signal); return; } + phaseStarted = performance.now(); const inputManifest = projectInputManifest( root, options, liveBuildInputs, selected.files, ); + traceResident("input-manifest", phaseStarted); if ( current.providerFallback && !sameProjectInputManifest(current.inputManifest, inputManifest) @@ -563,16 +753,6 @@ export function createResidentGraphSource( await replaceLanguages(current, signal); return; } - const prefetched = await refreshBulkSessions(current.sessions, signal); - const bulkChanged = [...prefetched].some( - ([language, refresh]) => - current.generations.get(language) !== refresh.generation, - ); - const bulkSliceLanguagesChanged = !sameBulkSliceLanguages( - current.sessions, - prefetched, - current.providers, - ); const providerMovement = movedProviderSource( providerSourcesOf( [...new Set(prefetched.values())].map( @@ -592,7 +772,9 @@ export function createResidentGraphSource( current.modes = bulkModesOf(prefetched); return; } + phaseStarted = performance.now(); const discovered = discoverLanguages(root, options); + traceResident("language-discovery", phaseStarted); if ( !sameLanguages(current.languages, discovered) || bulkSliceLanguagesChanged @@ -744,6 +926,62 @@ function bulkModesOf( return modes; } +function traceResident(phase: string, started: number): void { + if (process.env["SAMCHON_GRAPH_ROSLYN_TRACE"] !== "1") return; + process.stderr.write( + `${JSON.stringify({ + phase: `roslyn-resident-${phase}`, + elapsedMs: Math.round(performance.now() - started), + })}\n`, + ); +} + +function bulkFactDigestsOf( + sessions: ReadonlyMap, +): Map { + const digests = new Map(); + for (const session of sessions.values()) { + if (!isBulkGraphSession(session)) continue; + const digest = session.current?.protocol?.factDigest; + if (digest !== undefined) digests.set(session, digest); + } + return digests; +} + +function bulkWarningsOf( + sessions: ReadonlyMap, +): Map { + const warnings = new Map(); + for (const session of sessions.values()) { + if (!isBulkGraphSession(session) || session.current === undefined) continue; + warnings.set(session, session.current.warnings); + } + return warnings; +} + +function bulkSourceFilesOf( + sessions: ReadonlyMap, +): Map { + const sources = new Map(); + for (const session of sessions.values()) { + if (!isBulkGraphSession(session) || session.current === undefined) continue; + sources.set(session, [...session.current.sources.keys()].sort(compareText)); + } + return sources; +} + +function fileDigest(file: string): string { + try { + return createHash("sha256").update(fs.readFileSync(file)).digest("hex"); + } catch { + return ""; + } +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : 1; +} + function providerSourcesOf( snapshots: readonly IBulkGraphSession.ISnapshot[], ): Map { @@ -956,6 +1194,25 @@ function bulkLanguagesOf( return languages; } +/** Whether fixed language selection delegates every requested lane to bulk. */ +function entirelyBulkOwned( + requested: readonly GraphLanguage[] | undefined, + sessions: ReadonlyMap, +): boolean { + if ( + requested === undefined || + requested.length === 0 || + requested.length !== sessions.size + ) { + return false; + } + for (const language of requested) { + const session = sessions.get(language); + if (session === undefined || !isBulkGraphSession(session)) return false; + } + return true; +} + /** * What every source file on disk contains right now, as a content hash per file. * diff --git a/packages/graph/src/provider/BatchGraphSession.ts b/packages/graph/src/provider/BatchGraphSession.ts index 03fce27d..60e779dc 100644 --- a/packages/graph/src/provider/BatchGraphSession.ts +++ b/packages/graph/src/provider/BatchGraphSession.ts @@ -210,11 +210,16 @@ export class BatchGraphSession implements IBulkGraphSession { } let producerFailure: Error | undefined; try { - await this.run( - this.options.command, - this.options.indexArgs(artifact), - signal, - ); + const produce = this.options.produce; + if (produce !== undefined) { + await produce({ artifact, signal }); + } else { + await this.run( + this.options.command, + this.options.indexArgs(artifact), + signal, + ); + } } catch (error) { // `run` crosses the only unknown-rejection boundary through `enqueue`, // which normalizes it before this promise can reject. @@ -427,16 +432,12 @@ export class BatchGraphSession implements IBulkGraphSession { abortedProcessError( this.options.provider, command.command, - // Bounded before it is labelled, not after. A producer killed - // after an hour is the one most likely to have filled the whole - // 64 KiB stderr buffer with progress, and the last of that says - // where it had got to — but slicing the finished sentence would - // cut off the `(no stderr; last of stdout)` attribution whenever - // the fallback text is long, leaving a reader unable to tell a - // diagnosis from a scraped stdout tail. The exit-code path keeps - // its full text: a producer that chose to stop usually said why - // once, and this one did not choose to stop at all. - failureDetail(boundedTail(stderr), stdout), + // Keep the same bounded, attributed tails used for an ordinary + // non-zero exit. A producer killed after an hour is especially + // likely to have filled the capture buffers with progress, but + // the final lines still say where it got to. Bounding before + // labelling also keeps the stream attribution intact. + failureDetail(stderr, stdout), ), ); return; @@ -572,13 +573,25 @@ function relocateArtifact(produced: string, artifact: string): void { } export namespace BatchGraphSession { - export interface IOptions { + export type IOptions = ICommonOptions & + ( + | { + indexArgs: (artifact: string) => string[]; + produce?: undefined; + } + | { + indexArgs?: undefined; + /** Resident producer that writes the isolated artifact directly. */ + produce: (props: IProduceProps) => Promise; + } + ); + + interface ICommonOptions { root: string; languages: readonly GraphLanguage[]; provider: string; command: IGraphProvider.ICommand; artifactName: string; - indexArgs: (artifact: string) => string[]; /** * Existing directory that owns this session's unique generation children. @@ -623,6 +636,11 @@ export namespace BatchGraphSession { args: readonly string[], ) => Promise; } + + export interface IProduceProps { + artifact: string; + signal: AbortSignal | undefined; + } } interface ISpawned { @@ -710,34 +728,42 @@ function combineSignals( /** * What the tool said about its own failure, wherever it said it. * - * stderr first, because that is where a well-behaved tool puts diagnostics. But - * a build wrapper is not one tool — `scip-java` runs the project's real Gradle - * or Maven build, and Gradle reports failures on stdout. The benchmark's java - * lane failed with `exited with code 1` and nothing else for exactly that - * reason: the whole explanation had been captured and then dropped because it - * arrived on the wrong stream. + * A build wrapper is not one tool. The JVM can announce `JAVA_TOOL_OPTIONS` on + * stderr while Maven prints the actual failure on stdout, so preferring either + * non-empty stream drops evidence the other one alone owns. Keep both tails + * attributed when both spoke. * * The tail rather than the head, since a build prints its failure last. Bounded * because stdout is the artifact channel for some providers, and an index is not * something to paste into an error message. */ function failureDetail(stderr: string, stdout: string): string { - const detail = stderr.trim(); - if (detail !== "") return `: ${detail}`; - const fallback = stdout.trim(); - if (fallback === "") return ""; - const tail = fallback.slice(-FAILURE_DETAIL_LIMIT); - return `: (no stderr; last of stdout) ${ - tail.length < fallback.length ? `…${tail}` : tail - }`; + const error = stderr.trim(); + const output = stdout.trim(); + if (error === "" && output === "") return ""; + if (error === "") { + return `: (no stderr; last of stdout) ${boundedTail(output)}`; + } + if (output === "") return `: ${boundedTail(error)}`; + const stderrLimit = Math.floor(FAILURE_DETAIL_LIMIT / 2); + return ( + `: stderr tail: ${boundedTail(error, stderrLimit)}` + + `; stdout tail: ${boundedTail( + output, + FAILURE_DETAIL_LIMIT - stderrLimit, + )}` + ); } /** Enough for a build tool's failure summary, short of an artifact. */ const FAILURE_DETAIL_LIMIT = 2000; /** The end of a producer's output, which is where its last progress is. */ -function boundedTail(text: string): string { +function boundedTail( + text: string, + limit: number = FAILURE_DETAIL_LIMIT, +): string { const trimmed = text.trim(); - if (trimmed.length <= FAILURE_DETAIL_LIMIT) return trimmed; - return `…${trimmed.slice(-FAILURE_DETAIL_LIMIT)}`; + if (trimmed.length <= limit) return trimmed; + return `…${trimmed.slice(-limit)}`; } diff --git a/packages/graph/src/provider/GRAPH_PROVIDERS.ts b/packages/graph/src/provider/GRAPH_PROVIDERS.ts index 27a66fce..9ccc1296 100644 --- a/packages/graph/src/provider/GRAPH_PROVIDERS.ts +++ b/packages/graph/src/provider/GRAPH_PROVIDERS.ts @@ -1,11 +1,15 @@ import { IGraphProvider } from "./IGraphProvider"; +import { csharpGraphProvider } from "./csharp/csharpGraphProvider"; import { cppGraphProvider } from "./cpp/cppGraphProvider"; import { goGraphProvider } from "./go/goGraphProvider"; import { javaGraphProvider } from "./java/javaGraphProvider"; +import { kotlinGraphProvider } from "./kotlin/kotlinGraphProvider"; import { luaGraphProvider } from "./lua/luaGraphProvider"; import { rustGraphProvider } from "./rust/rustGraphProvider"; +import { scalaGraphProvider } from "./scala/scalaGraphProvider"; import { standardScipProviders } from "./scip/standardScipProviders"; import { standardSidecarProviders } from "./sidecar/standardSidecarProviders"; +import { swiftGraphProvider } from "./swift/swiftGraphProvider"; import { ttscGraphProvider } from "./ttscgraph/ttscGraphProvider"; /** @@ -30,11 +34,18 @@ export const GRAPH_PROVIDERS: readonly IGraphProvider[] = [ rustGraphProvider, cppGraphProvider, javaGraphProvider, - // Both entries are owned by a strict route as its fallback tier, so the - // registry must not also list them as owners: one language cannot have two. + kotlinGraphProvider, + scalaGraphProvider, + swiftGraphProvider, + // Clang, Java, Kotlin and C# SCIP entries are owned by strict routes as fallback + // tiers, so the registry must not also list them as language owners. + csharpGraphProvider, ...standardScipProviders.filter( (provider) => - provider.name !== "scip-clang" && provider.name !== "scip-java", + provider.name !== "scip-clang" && + provider.name !== "scip-java" && + provider.name !== "scip-kotlinc" && + provider.name !== "scip-dotnet", ), ...standardSidecarProviders, ]; diff --git a/packages/graph/src/provider/GraphSnapshotProtocol.ts b/packages/graph/src/provider/GraphSnapshotProtocol.ts index 2b4b41ec..938a5a25 100644 --- a/packages/graph/src/provider/GraphSnapshotProtocol.ts +++ b/packages/graph/src/provider/GraphSnapshotProtocol.ts @@ -408,6 +408,14 @@ export namespace GraphSnapshotProtocol { * so a caller may only adopt when it drops every reference it has as this * returns. On a 469-shard generation the copy is a second whole graph, * held at the moment the caller is still holding the first. + * + * `reuseValidatedFacts` is an explicit resident-provider optimization. It + * may reuse the prior generation's frozen fact arrays only after every + * changed shard proves the same fact digest, the whole-generation fact + * digest agrees, and the universe, targets, warnings, and source identity + * set are unchanged. Source byte digests and the protocol envelope still + * advance. Any missing proof takes the ordinary assembly and validation + * path; a contradictory proof rejects the transaction. */ public apply( frames: readonly Frame[], @@ -416,6 +424,7 @@ export namespace GraphSnapshotProtocol { warnings?: readonly string[]; validate?: (snapshot: IBulkGraphSession.ISnapshot) => void; adopt?: boolean; + reuseValidatedFacts?: boolean; } = {}, ): IBulkGraphSession.ISnapshot { throwIfAborted(options.signal); @@ -478,6 +487,11 @@ export namespace GraphSnapshotProtocol { begin.baseGeneration === undefined ? new Map() : new Map(this.committed); + let factsEquivalent = + begin.baseGeneration !== undefined && + this.snapshot !== undefined && + begin.universe === this.snapshot.provenance.universe && + sameList(begin.targets, this.snapshot.protocol!.targets); const touched = new Set(); const invalidated = new Set(); for (const frame of frames.slice(2, -1)) { @@ -496,12 +510,24 @@ export namespace GraphSnapshotProtocol { `graph snapshot protocol: shard digest mismatch: ${frame.shard.key}`, ); } - if (this.committed.get(frame.shard.key)?.digest !== digest) { + const prior = this.committed.get(frame.shard.key); + let shardFacts = prior?.factDigest; + if (prior?.digest !== digest) { invalidated.add(frame.shard.key); + if (factsEquivalent) { + if (prior === undefined) factsEquivalent = false; + else { + const priorFacts = + prior.factDigest ?? shardFactDigest(prior.shard); + shardFacts = shardFactDigest(frame.shard); + factsEquivalent = priorFacts === shardFacts; + } + } } next.set(frame.shard.key, { digest, shard: options.adopt === true ? frame.shard : clone(frame.shard), + ...(shardFacts === undefined ? {} : { factDigest: shardFacts }), }); } else if (frame.type === "deleteShard") { assertString(frame.key, "deleteShard.key"); @@ -517,6 +543,7 @@ export namespace GraphSnapshotProtocol { ); } invalidated.add(frame.key); + factsEquivalent = false; } else { throw new Error( `graph snapshot protocol: unexpected ${frame.type} inside transaction`, @@ -557,13 +584,64 @@ export namespace GraphSnapshotProtocol { "graph snapshot protocol: commit shard manifest mismatch", ); } + const warnings = options.warnings ?? []; + const sources = factsEquivalent ? sourcesOf(next) : undefined; + const reuseFacts = + options.reuseValidatedFacts === true && + factsEquivalent && + this.snapshot !== undefined && + commit.factDigest === this.snapshot.protocol!.factDigest && + sources !== undefined && + sameMapKeys(sources, this.snapshot.sources) && + sameList(warnings, this.snapshot.warnings); + if ( + options.reuseValidatedFacts === true && + factsEquivalent && + this.snapshot !== undefined && + commit.factDigest !== this.snapshot.protocol!.factDigest + ) { + throw new Error("graph snapshot protocol: commit fact digest mismatch"); + } + if (reuseFacts) { + // The opt-in proof above establishes that the already-frozen arrays + // are precisely the facts this commit names. Revalidating and copying + // those arrays would traverse the whole graph for a body-only edit; + // the source manifest and protocol envelope are the only new data. + if ( + manifestDigest( + [...sources].map(([file, source]) => ({ file, ...source })), + ) !== begin.manifest + ) { + throw new Error( + "graph snapshot protocol: input manifest digest mismatch", + ); + } + const assembled = assembleFactEquivalent( + this.snapshot!, + begin, + commit, + expectedManifest, + sources, + ); + throwIfAborted(options.signal); + freezeDeep(assembled, "the graph snapshot protocol generation"); + proven(assembled, commit.factDigest); + throwIfAborted(options.signal); + this.committed = next; + this.published = new Map( + [...next].map(([key, entry]) => [key, entry.shard]), + ); + this.identity = clone(hello); + this.snapshot = assembled; + return assembled; + } const assembled = assemble( hello, begin, commit, expectedManifest, next, - options.warnings ?? [], + warnings, ); assertAssembledFacts(assembled, hello); if ( @@ -606,6 +684,88 @@ export namespace GraphSnapshotProtocol { interface ICommittedShard { digest: string; shard: IShard; + factDigest?: string; + } + + function shardFactDigest(shard: IShard): string { + return digest({ + coverage: shard.coverage, + diagnostics: shard.diagnostics, + edges: shard.edges, + nodes: shard.nodes, + unresolved: shard.unresolved, + }); + } + + function sourcesOf( + shards: ReadonlyMap, + ): Map { + const sources = new Map(); + for (const { shard } of shards.values()) { + for (const source of shard.sources) { + const value = { + checkerDigest: source.checkerDigest, + diskDigest: source.diskDigest, + }; + const prior = sources.get(source.file); + if ( + prior !== undefined && + (prior.checkerDigest !== value.checkerDigest || + prior.diskDigest !== value.diskDigest) + ) { + throw new Error( + `graph snapshot protocol: shards disagree about source ${source.file}`, + ); + } + sources.set(source.file, value); + } + } + return sources; + } + + function sameMapKeys( + left: ReadonlyMap, + right: ReadonlyMap, + ): boolean { + return ( + left.size === right.size && [...left.keys()].every((key) => right.has(key)) + ); + } + + function assembleFactEquivalent( + prior: IBulkGraphSession.ISnapshot, + begin: IBegin, + commit: ICommit, + manifest: IBulkGraphSession.IShard[], + sources: Map, + ): IBulkGraphSession.ISnapshot { + return { + languages: prior.languages, + nodes: prior.nodes, + edges: prior.edges, + diagnostics: prior.diagnostics, + sources: sealedMap( + sources, + "the graph snapshot protocol source manifest", + ), + provenance: prior.provenance, + coverage: prior.coverage, + unresolved: prior.unresolved, + protocol: { + version: VERSION, + sequence: begin.sequence, + generation: begin.generation, + // Fact equivalence is possible only for a delta against the currently + // committed generation, so both base fields were proved above. + baseSequence: begin.baseSequence!, + baseGeneration: begin.baseGeneration!, + manifest: begin.manifest, + targets: [...begin.targets], + shards: manifest.map((entry) => ({ ...entry })), + factDigest: commit.factDigest, + }, + warnings: prior.warnings, + }; } /** diff --git a/packages/graph/src/provider/IBulkGraphSession.ts b/packages/graph/src/provider/IBulkGraphSession.ts index 86b2e7c8..73edbf5e 100644 --- a/packages/graph/src/provider/IBulkGraphSession.ts +++ b/packages/graph/src/provider/IBulkGraphSession.ts @@ -22,6 +22,17 @@ import { export interface IBulkGraphSession { readonly kind: "bulk"; + /** + * Whether this live session owns the selected producer command until close. + * + * Most batch sessions launch their recorded command anew and therefore need + * the coordinator to re-check provider topology after a changed generation. + * A genuinely resident producer can instead bind its executable and publish + * exact producer/compiler identity on every generation; command discovery + * beside that process is neither its authority nor a reason to replace it. + */ + readonly ownsProviderTopology?: boolean; + /** * Every language this session publishes as one atomic slice. Never empty. * diff --git a/packages/graph/src/provider/compiler/CompilerGraphSession.ts b/packages/graph/src/provider/compiler/CompilerGraphSession.ts new file mode 100644 index 00000000..5b22f4b8 --- /dev/null +++ b/packages/graph/src/provider/compiler/CompilerGraphSession.ts @@ -0,0 +1,177 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { GraphLanguage } from "../../typings"; +import { BatchGraphSession } from "../BatchGraphSession"; +import { IBulkGraphSession } from "../IBulkGraphSession"; +import { IGraphProvider } from "../IGraphProvider"; +import { CompilerGraphSnapshotAdapter } from "./CompilerGraphSnapshotAdapter"; +import { ResidentGraphProducerClient } from "./ResidentGraphProducerClient"; + +const DEFAULT_MAX_ARTIFACT_BYTES = 256 * 1024 * 1024; + +/** + * A strict compiler route's session: one resident build connection and one + * graph generation per changed input universe. + * + * The producer is attached to the project's real compile tasks, so a refresh + * is a normal build. That is the point of the route rather than a limitation + * of it: the build tool's own incremental state + * decides which sources are recompiled, and the shards it does not rewrite are + * the ones this session does not resend. + * + * The lifecycle around that build is {@link BatchGraphSession}'s, shared with + * every SCIP and sidecar route: an isolated generation directory, a bounded + * cancellable child, an input fingerprint that decides whether the build has + * to run at all, and publication only after the complete candidate loaded. + */ +export class CompilerGraphSession implements IBulkGraphSession { + public readonly kind = "bulk" as const; + public readonly ownsProviderTopology = true; + public readonly languages: readonly GraphLanguage[]; + public readonly root: string; + + private readonly provider: string; + private readonly maxArtifactBytes: number; + private readonly adapter: CompilerGraphSnapshotAdapter; + private readonly validate: (snapshot: IBulkGraphSession.ISnapshot) => void; + private readonly producer: ResidentGraphProducerClient; + private readonly batch: BatchGraphSession; + + public constructor(options: CompilerGraphSession.IOptions) { + const maxArtifactBytes = + options.maxArtifactBytes ?? DEFAULT_MAX_ARTIFACT_BYTES; + if (!Number.isSafeInteger(maxArtifactBytes) || maxArtifactBytes < 1) { + throw new TypeError( + `${options.provider}: maxArtifactBytes must be a positive safe integer`, + ); + } + this.provider = options.provider; + this.maxArtifactBytes = maxArtifactBytes; + this.validate = options.validate; + this.adapter = options.adapter; + this.producer = new ResidentGraphProducerClient({ + root: options.root, + provider: options.provider, + command: options.command, + serverCommand: options.serverCommand, + label: options.label, + }); + let configuration: + | ReturnType> + | undefined; + this.batch = new BatchGraphSession({ + root: options.root, + languages: options.languages, + provider: options.provider, + command: options.command, + artifactName: options.artifactName, + inputs: options.inputs, + // This session owns the producer process and its build connection. Their + // versions cannot change underneath that live process, while probing + // both launchers on every no-op costs more than the resident latency + // budget. Establish the rows once; a restarted producer still states + // its own exact identity in the next artifact and the adapter reloads if + // that identity differs. + configuration: () => + (configuration ??= options.configuration()), + produce: ({ artifact, signal }) => + this.producer.produce(artifact, signal), + load: (props) => this.load(props), + }); + this.languages = this.batch.languages; + this.root = this.batch.root; + } + + public get generation(): number { + return this.batch.generation; + } + + public get current(): IBulkGraphSession.ISnapshot | undefined { + return this.batch.current; + } + + public async refresh( + options: { signal?: AbortSignal } = {}, + ): Promise { + const result = await this.batch.refresh(options); + // `BatchGraphSession` knows whether it had to run the build; only the + // producer knows what the build then did. A generation that reused shards + // from the one before it is an incremental compile, and reporting it as a + // rebuild would tell a reader the whole workspace was recompiled when the + // build tool recompiled one file. + return result.changed + ? { ...result, mode: this.adapter.lastMode } + : result; + } + + public close(): Promise { + return Promise.all([this.batch.close(), this.producer.close()]).then( + () => undefined, + ); + } + + private load( + props: BatchGraphSession.ILoadProps, + ): Promise { + const size = fs.statSync(props.artifact).size; + if (size > this.maxArtifactBytes) { + throw new Error( + `${this.provider}: the graph artifact exceeded the ${String(this.maxArtifactBytes)} byte limit`, + ); + } + return Promise.resolve( + this.adapter.apply(this.decode(fs.readFileSync(props.artifact, "utf8")), { + signal: props.signal, + validate: this.validate, + }), + ); + } + + /** + * Read the artifact, saying whose it is and what was in it when it will not. + * + * A raw `JSON.parse` throws `Unexpected token` with no provider name and no + * trace of the bytes, which is the least useful sentence available about a + * build that printed a message where its graph should be—and builds can + * print a great deal. + */ + private decode(text: string): unknown { + try { + return JSON.parse(text) as unknown; + } catch (error) { + const head = text.trimStart().slice(0, 400); + const message = (error as SyntaxError).message; + throw new Error( + `${this.provider}: the graph artifact is not JSON: ${message}${ + head === "" ? " (the file is empty)" : `: ${head}` + }`, + ); + } + } +} + +export namespace CompilerGraphSession { + export interface IOptions { + root: string; + languages: readonly GraphLanguage[]; + provider: string; + command: IGraphProvider.ICommand; + adapter: CompilerGraphSnapshotAdapter; + serverCommand: string; + label: string; + artifactName: string; + inputs: () => string[]; + configuration: NonNullable; + /** + * The contract gate the registry entry owns. + * + * Required rather than optional: a generation that reached a consumer + * without being held to the provider's declared languages, authority and + * fact families is exactly what the gate exists to prevent, and an + * optional gate is one a future call site can forget. + */ + validate: (snapshot: IBulkGraphSession.ISnapshot) => void; + maxArtifactBytes?: number; + } +} diff --git a/packages/graph/src/provider/compiler/CompilerGraphSnapshotAdapter.ts b/packages/graph/src/provider/compiler/CompilerGraphSnapshotAdapter.ts new file mode 100644 index 00000000..dd1e24d0 --- /dev/null +++ b/packages/graph/src/provider/compiler/CompilerGraphSnapshotAdapter.ts @@ -0,0 +1,1164 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; + +import { + ISamchonGraphCoverage, + ISamchonGraphDiagnostic, + ISamchonGraphEdge, + ISamchonGraphEvidence, + ISamchonGraphNode, + ISamchonGraphUnresolved, + SamchonGraphNodeModifier, +} from "../../structures"; +import { + GRAPH_EDGE_KINDS, + GraphEdgeKind, + GraphLanguage, + GraphNodeKind, +} from "../../typings"; +import { GraphSnapshotProtocol } from "../GraphSnapshotProtocol"; +import { IBulkGraphSession } from "../IBulkGraphSession"; +import { semanticGraphNodeId } from "../semanticIdentity"; +import { ICompilerGraphSnapshot } from "./ICompilerGraphSnapshot"; + +const SHA256 = /^[0-9a-f]{64}$/u; +const NODE_KINDS = new Set([ + "file", + "package", + "namespace", + "module", + "function", + "class", + "interface", + "type", + "enum", + "variable", + "method", + "property", + "parameter", + "field", + "constructor", +]); +const MODIFIERS = new Set([ + "export", + "default", + "declare", + "abstract", + "static", + "readonly", + "async", + "const", + "public", + "private", + "protected", + "internal", + "optional", +]); +const COVERAGE_STATES = new Set([ + "complete", + "partial", + "unsupported", +]); +const UNRESOLVED_REASONS = new Set([ + "dynamic", + "reflection", + "macro-or-generated", + "conditional-build", + "external-boundary", + "analysis-error", + "excluded-input", + "identity-unstable", + "provider-gap", +]); + +/** + * What this route proves about itself, published so a consumer degrades + * against a claim instead of guessing from an empty list. + * + * Diagnostics are explicit because the producer exports FIR diagnostics with + * each source shard rather than asking the consumer to scrape build output. + */ +const CAPABILITIES = [ + "coverage", + "diagnostics", + "diskDigests", + "incremental", + "sourceDigests", + "universe", + "unresolved", +]; + +/** + * Turn one committed compiler artifact into a validated Graph Snapshot + * Protocol generation. + * + * The producer already commits per target: each target carries its own + * content-addressed generation and the universe it compiled against, and an + * incremental build rewrites only the sources its compiler recompiled. So this + * adapter's job is not to invent a transaction but to prove the one it was + * handed—that every edge endpoint exists, that the coverage matrix is + * complete for every target, that no two targets have been folded into one + * universe—and then to express it as a delta against the generation this + * session last published. + */ +export class CompilerGraphSnapshotAdapter { + public readonly store: GraphSnapshotProtocol.Store; + + /** + * What the producer did to earn the last published generation. + * + * Read rather than inferred: a delta exists only when this adapter proved + * the producer's identity, universe and target set had not moved and then + * carried shards forward, which is exactly an incremental compile. + */ + public lastMode: IBulkGraphSession.Mode = "initial"; + + /** Shard key to content digest for the last generation this store kept. */ + private committed = new Map(); + /** Canonical producer identity of the last generation, for delta fencing. */ + private identity: string | undefined; + private sequence = 0; + + public constructor( + private readonly root: string, + private readonly contract: CompilerGraphSnapshotAdapter.IContract, + ) { + this.store = new GraphSnapshotProtocol.Store(root); + } + + public get current(): IBulkGraphSession.ISnapshot | undefined { + return this.store.current; + } + + /** Validate one artifact and publish it, or leave the prior generation. */ + public apply( + value: unknown, + options: { + signal?: AbortSignal | undefined; + warnings?: readonly string[] | undefined; + validate?: ((snapshot: IBulkGraphSession.ISnapshot) => void) | undefined; + } = {}, + ): IBulkGraphSession.ISnapshot { + const raw = assertSnapshot(value, this.root, this.contract); + const rawTargets = [...raw.targets].sort((left, right) => + compareText(left.name, right.name), + ); + const universe = universeOf(raw); + // Keys cannot collide by construction: a target name is unique across the + // artifact, a source is unique within its target, and the two kinds of key + // carry different prefixes. Both uniqueness facts are refusals in + // `assertSnapshot` rather than assumptions made here. + const shards = new Map(); + for (const target of rawTargets) { + for (const shard of targetShards( + this.root, + target, + universe, + this.contract, + )) { + shards.set(shard.key, shard); + } + } + + const digests = new Map(); + for (const [key, shard] of shards) { + digests.set(key, GraphSnapshotProtocol.shardDigest(shard)); + } + const ordered = [...shards.keys()].sort(compareText); + const manifest = ordered.map((key) => ({ + key, + digest: digests.get(key)!, + })); + const hello = helloOf(raw, this.contract); + const identity = canonical(hello); + const targets = rawTargets.map((target) => target.name); + const prior = this.store.current; + + // A delta is only meaningful against the same producer, the same build + // universe and the same target set. When any of those moved, every shard + // the previous generation held is invalid whether or not its bytes are + // identical—a source that compiles to the same facts against a different + // classpath is a different fact, because what it proves about the rest of + // the program has changed. That is a reload, and stating it as one is + // cheaper than sending a delta that invalidates everything anyway. + const canDelta = + prior !== undefined && + this.identity === identity && + prior.provenance.universe === universe && + sameList(prior.protocol!.targets, targets); + + this.sequence += 1; + const begin: GraphSnapshotProtocol.IBegin = { + type: "begin", + sequence: this.sequence, + generation: generationOf(raw), + ...(canDelta + ? { + baseSequence: prior.protocol!.sequence, + baseGeneration: prior.protocol!.generation, + } + : {}), + universe, + manifest: GraphSnapshotProtocol.manifestDigest( + ordered.flatMap((key) => shards.get(key)!.sources), + ), + targets, + }; + + const frames: GraphSnapshotProtocol.Frame[] = [hello, begin]; + // Only what moved. A build that recompiled one source + // rewrites one shard, and re-sending the rest would make every refresh + // cost a whole workspace—the thing this route exists to stop paying. + for (const key of ordered) { + const digest = digests.get(key)!; + if (canDelta && this.committed.get(key) === digest) continue; + frames.push({ type: "upsertShard", digest, shard: shards.get(key)! }); + } + if (canDelta) { + for (const key of [...this.committed.keys()].sort(compareText)) { + if (!shards.has(key)) frames.push({ type: "deleteShard", key }); + } + } + frames.push({ + type: "commit", + sequence: begin.sequence, + generation: begin.generation, + shards: manifest, + // The store recomputes this from the generation it reconstructs and + // refuses a commit that disagrees, so both sides have to walk the same + // manifest in the same order. Deriving it from anything else would make + // the check compare two different things and pass anyway. + factDigest: GraphSnapshotProtocol.factDigest( + assembled(hello, begin, manifest, shards), + ), + }); + + const snapshot = this.store.apply(frames, options); + this.lastMode = + prior === undefined ? "initial" : canDelta ? "incremental" : "reload"; + this.committed = digests; + this.identity = identity; + return snapshot; + } +} + +export namespace CompilerGraphSnapshotAdapter { + export interface IContract { + label: string; + language: GraphLanguage; + provider: string; + producer: string; + facts: readonly GraphEdgeKind[]; + /** Language-specific producer guarantees already validated by this adapter. */ + capabilities?: readonly string[]; + diagnosticCode: string; + shardKeyPrefix: string; + schemaVersion: number; + protocolVersion: number; + identitySalt?: (target: ICompilerGraphSnapshot.ITarget) => string; + validateSnapshot?: (snapshot: ICompilerGraphSnapshot) => void; + validateTarget?: ( + target: ICompilerGraphSnapshot.ITarget, + ) => void; + validateShard?: ( + shard: ICompilerGraphSnapshot.IShard, + target: ICompilerGraphSnapshot.ITarget, + root: string, + ) => void; + } +} + +/** Reconstruct exactly what the store will assemble, without publishing it. */ +function assembled( + hello: GraphSnapshotProtocol.IHello, + begin: GraphSnapshotProtocol.IBegin, + manifest: readonly IBulkGraphSession.IShard[], + shards: ReadonlyMap, +): Parameters[0] { + const nodes: ISamchonGraphNode[] = []; + const edges: ISamchonGraphEdge[] = []; + const coverage: ISamchonGraphCoverage[] = []; + const unresolved: ISamchonGraphUnresolved[] = []; + // Folded, not concatenated, because the store folds when it assembles. A + // generation whose shards name one fact twice -- two units emitting the same + // relation from a symbol they share -- would be digested here over both + // copies and there over one, and the commit refused for a disagreement this + // function invented. + const ordered = manifest.map((entry) => shards.get(entry.key)!); + const folded = GraphSnapshotProtocol.fold(ordered); + nodes.push(...folded.nodes); + edges.push(...folded.edges); + unresolved.push(...folded.unresolved); + for (const shard of ordered) coverage.push(...shard.coverage); + return { + languages: [...hello.languages], + nodes, + edges, + diagnostics: folded.diagnostics, + coverage, + unresolved, + provenance: { + provider: hello.provider, + authority: hello.authority, + facts: [...hello.supportedFacts], + schemaVersion: hello.producerSchemaVersion, + tool: hello.producer, + toolVersion: hello.producerVersion, + compilerVersion: hello.compilerVersion, + protocolVersion: hello.protocolVersion, + universe: begin.universe, + capabilities: [...hello.capabilities], + }, + }; +} + +function helloOf( + raw: ICompilerGraphSnapshot, + contract: CompilerGraphSnapshotAdapter.IContract, +): GraphSnapshotProtocol.IHello { + return { + type: "hello", + protocolVersion: 1, + schemaVersion: 1, + producerSchemaVersion: raw.schemaVersion, + provider: contract.provider, + producer: raw.producer.name, + producerVersion: raw.producer.version, + compilerVersion: compilerVersionOf(raw), + languages: [contract.language], + authority: "compiler", + supportedFacts: [...contract.facts], + capabilities: [...CAPABILITIES, ...(contract.capabilities ?? [])], + }; +} + +/** + * The compiler versions that ran the plugin, as a set rather than a pick. + * + * Every shard states the compiler version that produced it, and one build can + * legitimately run multiple toolchains across its targets. The + * one answer this must not give is the first shard's reading presented as the + * build's, so distinct versions are published together and a reader can see + * that the generation crossed a compiler boundary. + */ +function compilerVersionOf(raw: ICompilerGraphSnapshot): string { + const versions = new Set(); + for (const target of raw.targets) { + for (const shard of target.shards) versions.add(shard.compilerVersion); + } + return [...versions].sort(compareText).join("; "); +} + +/** + * One identity for a generation the producer committed per target. + * + * A build with two targets has two committed generations and no single one of + * them identifies the pair, so this composes them. Through the canonical + * encoding rather than by concatenation: it quotes and escapes every string + * and keeps the array structure, so a target named `a` with generation `bc` + * and one named `ab` with generation `c` cannot collapse onto each other the + * way a joined string would. + */ +function generationOf(raw: ICompilerGraphSnapshot): string { + return digest( + [...raw.targets] + .sort((left, right) => compareText(left.name, right.name)) + .map((target) => [target.name, target.generation, target.universe]), + ); +} + +/** The same composition over the universes alone. */ +function universeOf(raw: ICompilerGraphSnapshot): string { + return digest( + [...raw.targets] + .sort((left, right) => compareText(left.name, right.name)) + .map((target) => [target.name, target.universe]), + ); +} + +/** Every shard one committed target contributes, source shards then metadata. */ +function targetShards( + root: string, + target: ICompilerGraphSnapshot.ITarget, + universe: string, + contract: CompilerGraphSnapshotAdapter.IContract, +): GraphSnapshotProtocol.IShard[] { + const declared = declaredNodes(root, target, contract); + const externals = externalNodes(target, declared, contract); + const shards: GraphSnapshotProtocol.IShard[] = []; + const located = new Set(); + for (const shard of target.shards) { + const adapted = sourceShard( + root, + target, + shard, + declared, + externals, + universe, + contract, + ); + for (const site of adapted.unresolved) located.add(site.family); + shards.push(adapted); + } + shards.push(metadataShard(target, externals, universe, located, contract)); + return shards; +} + +/** + * Every symbol this target declares, by the producer's canonical symbol. + * + * Built across the whole target before any shard is adapted, because a call in + * one compilation unit names a method declared in another and both endpoints + * have to resolve to the same identity. A symbol declared twice inside one + * target is a producer defect whether or not the two records agree: a compiler + * attributes one declaration per symbol, and publishing both would put the + * same node in two shards of one generation. + * + * Across targets it is ordinary. One source compiled into a main and a test + * source set is two declarations in two universes, which is exactly what + * target-scoped identity keeps apart. + */ +function declaredNodes( + root: string, + target: ICompilerGraphSnapshot.ITarget, + contract: CompilerGraphSnapshotAdapter.IContract, +): Map { + const declared = new Map(); + for (const shard of target.shards) { + for (const node of shard.nodes) { + if (declared.has(node.symbol)) { + throw new Error( + `${contract.label}: symbol ${node.symbol} is declared twice in target ${target.name}`, + ); + } + declared.set(node.symbol, adaptNode(root, target, node, contract)); + } + } + return declared; +} + +function sourceShard( + root: string, + target: ICompilerGraphSnapshot.ITarget, + shard: ICompilerGraphSnapshot.IShard, + declared: ReadonlyMap, + externals: ReadonlyMap, + universe: string, + contract: CompilerGraphSnapshotAdapter.IContract, +): GraphSnapshotProtocol.IShard { + const file = graphFile(root, shard.source); + const nodes = shard.nodes + .map((node) => declared.get(node.symbol)!) + .sort((left, right) => compareText(left.id, right.id)); + const edges: ISamchonGraphEdge[] = []; + const seen = new Set(); + for (const edge of shard.edges) { + const from = endpoint(edge.from, file, declared, externals); + const to = endpoint(edge.to, file, declared, externals); + // NUL-separated, the way every other edge key in this package is: a + // project-relative path is a legal endpoint here, and POSIX allows any + // byte but NUL in one. + const key = `${edge.kind}\0${from}\0${to}`; + // The producer keys its own edges by evidence as well as by endpoints, so + // one relationship written at two call sites arrives twice. The graph's + // triple is unique and keeps the first source-order evidence, which the + // producer's canonical ordering makes a deterministic choice rather than + // whichever record happened to be visited first. + if (seen.has(key)) continue; + seen.add(key); + edges.push({ + from, + to, + kind: edge.kind as GraphEdgeKind, + evidence: adaptEvidence(root, edge.evidence), + }); + } + return { + key: shardKey(contract, target.name, shard.source), + target: target.name, + languages: [contract.language], + nodes, + edges, + diagnostics: shard.diagnostics.map( + (diagnostic): ISamchonGraphDiagnostic => ({ + file: graphFile(root, diagnostic.evidence.file), + line: diagnostic.evidence.startLine, + column: diagnostic.evidence.startColumn, + code: contract.diagnosticCode, + message: diagnostic.message, + severity: diagnostic.severity as ISamchonGraphDiagnostic["severity"], + }), + ), + coverage: [], + unresolved: shard.unresolved.map((site) => ({ + provider: contract.provider, + language: contract.language, + target: target.name, + // The generation's universe rather than the target's own. A snapshot + // fences every fact against one build identity, and a multi-target + // generation composes its targets' universes into that one—so the + // target coordinate beside it is what says which of them the site was + // observed in. + universe, + family: site.family as GraphEdgeKind, + evidence: adaptEvidence(root, site.evidence), + reason: site.reason as ISamchonGraphUnresolved["reason"], + ...(site.candidates.length === 0 + ? {} + : { + candidates: site.candidates.map( + (candidate) => declared.get(candidate)?.id ?? candidate, + ), + }), + })), + sources: [ + { + file: sourcePath(root, shard.source), + checkerDigest: shard.checkerDigest, + diskDigest: shard.diskDigest, + }, + ], + }; +} + +/** + * The target's coverage matrix and every endpoint outside its own compilation. + * + * Both belong to the target rather than to any one of its sources. Coverage + * has to appear exactly once per target and family or the assembled generation + * carries duplicate rows; an external symbol is named by however many sources + * reference it and must still be one node. + */ +function metadataShard( + target: ICompilerGraphSnapshot.ITarget, + externals: ReadonlyMap, + universe: string, + located: ReadonlySet, + contract: CompilerGraphSnapshotAdapter.IContract, +): GraphSnapshotProtocol.IShard { + const nodes = [...externals.values()].sort((left, right) => + compareText(left.id, right.id), + ); + const coordinate = `bundled:///${contract.language}/target/${digest([target.name])}`; + return { + key: `${contract.shardKeyPrefix}-target:${digest([target.name])}:${target.universe}`, + target: target.name, + languages: [contract.language], + nodes, + edges: [], + diagnostics: [], + coverage: GRAPH_EDGE_KINDS.map((family) => ({ + provider: contract.provider, + language: contract.language, + target: target.name, + family, + state: target.coverage[family] as ISamchonGraphCoverage["state"], + })), + // A partial family with nowhere to point is still a partial family, and it + // is the one shape a reader cannot act on: "some sites are unproven" with + // no list reads exactly like "every site is proven". The producer declares + // several families partial as a property of the exporter rather than of + // any one call site—a `contains` edge it cannot emit for an anonymous + // declaration has no location to name—so the gap is published at the + // target's own coordinate instead of being left implicit. + unresolved: GRAPH_EDGE_KINDS.filter( + (family) => + target.coverage[family] === "partial" && !located.has(family), + ).map((family) => ({ + provider: contract.provider, + language: contract.language, + target: target.name, + universe, + family, + evidence: { + file: coordinate, + startLine: 1, + startCol: 1, + endLine: 1, + endCol: 1, + }, + reason: "provider-gap" as const, + })), + sources: [ + { + file: coordinate, + checkerDigest: target.universe, + diskDigest: "", + }, + ], + }; +} + +/** + * Every endpoint the target reaches but does not declare, as one node each. + * + * Collected across the whole target before any shard is built, because the + * producer describes an endpoint at the site that reached it and the sites do + * not all know the same amount. A compiler-attributed reference carries the + * element kind and both names; one it could not carries three nulls for the + * same symbol. Resolving that per site would give one symbol two identities + * depending on which shard was adapted first, so the description is settled + * once, here, and the shards only look it up. + * + * A site that names nothing defers to one that does. Two sites that both name + * it and disagree are a producer contradiction: one symbol is not two + * declarations, and picking either would publish a name the compiler never + * gave it. + */ +function externalNodes( + target: ICompilerGraphSnapshot.ITarget, + declared: ReadonlyMap, + contract: CompilerGraphSnapshotAdapter.IContract, +): Map { + const described = new Map(); + for (const shard of target.shards) { + for (const edge of shard.edges) { + for (const symbol of [edge.from, edge.to]) { + if (symbol === shard.source || declared.has(symbol)) continue; + const naming = symbol === edge.to ? namingOf(edge) : { name: symbol }; + const prior = described.get(symbol); + if (prior === undefined || anonymous(prior, symbol)) { + described.set(symbol, naming); + continue; + } + if (anonymous(naming, symbol)) continue; + if ( + prior.name !== naming.name || + prior.qualifiedName !== naming.qualifiedName + ) { + throw new Error( + `${contract.label}: external symbol ${symbol} is named two ways in target ${target.name}`, + ); + } + } + } + } + const externals = new Map(); + for (const [symbol, naming] of described) { + const display = naming.qualifiedName ?? naming.name; + externals.set(symbol, { + id: semanticGraphNodeId( + { + version: 2, + language: contract.language, + symbol, + role: "external_symbol", + native: { + key: identityKey(contract, target, symbol), + stability: "semantic", + }, + scope: { target: target.name }, + stability: "persistent", + }, + display, + ), + kind: "external_symbol", + language: contract.language, + name: naming.name, + ...(naming.qualifiedName === undefined + ? {} + : { qualifiedName: naming.qualifiedName }), + file: "", + external: true, + }); + } + return externals; +} + +interface INaming { + name: string; + qualifiedName?: string; +} + +/** What one edge says about the endpoint it points at, if anything. */ +function namingOf(edge: ICompilerGraphSnapshot.IEdge): INaming { + const name = + edge.targetName === null || edge.targetName === "" + ? edge.to + : declaredName(edge.targetName); + const qualifiedName = + edge.targetQualifiedName === null || edge.targetQualifiedName === "" + ? undefined + : declaredName(edge.targetQualifiedName); + return qualifiedName === undefined ? { name } : { name, qualifiedName }; +} + +/** + * Producer names are already separated from structural signatures. + */ +function declaredName(display: string): string { + return display; +} + +/** Whether a description says nothing the symbol did not already say. */ +function anonymous(naming: INaming, symbol: string): boolean { + return naming.name === symbol && naming.qualifiedName === undefined; +} + +/** + * Resolve one producer endpoint to a graph identity. + * + * Three shapes reach here. The compilation unit own path is what `contains` + * and `exports` hang off, and it stays a file coordinate rather than becoming + * a synthesized node: the same file compiled into two targets would otherwise + * need two file nodes with one id. A symbol the target declares resolves to + * that declaration. Anything else was settled by {@link externalNodes}. + */ +function endpoint( + symbol: string, + file: string, + declared: ReadonlyMap, + externals: ReadonlyMap, +): string { + if (symbol === file) return file; + const node = declared.get(symbol); + // Every endpoint that is neither the source coordinate nor a declaration was + // collected as an external node from these same edges, so the lookup cannot + // miss without the two walks having disagreed about what an endpoint is. + return node === undefined ? externals.get(symbol)!.id : node.id; +} + +function adaptNode( + root: string, + target: ICompilerGraphSnapshot.ITarget, + node: ICompilerGraphSnapshot.INode, + contract: CompilerGraphSnapshotAdapter.IContract, +): ISamchonGraphNode { + const name = declaredName(node.name); + const qualified = + node.qualifiedName === "" ? undefined : declaredName(node.qualifiedName); + const qualifiedName = qualified; + const display = qualifiedName ?? name; + const symbol = node.symbol; + const signature = node.signature === "" ? undefined : node.signature; + return { + id: semanticGraphNodeId( + { + version: 2, + language: contract.language, + symbol, + role: node.kind as GraphNodeKind, + native: { + key: identityKey(contract, target, symbol), + stability: "semantic", + }, + scope: { target: target.name }, + stability: "persistent", + }, + display, + ), + kind: node.kind as GraphNodeKind, + language: contract.language, + name, + ...(qualifiedName === undefined ? {} : { qualifiedName }), + file: graphFile(root, node.file), + external: false, + ...(node.exported ? { exported: true } : {}), + ...(node.modifiers.length === 0 + ? {} + : { modifiers: [...node.modifiers] as SamchonGraphNodeModifier[] }), + ...(signature === undefined ? {} : { signature }), + evidence: adaptEvidence(root, node.evidence), + }; +} + +function adaptEvidence( + root: string, + evidence: ICompilerGraphSnapshot.IEvidence, +): ISamchonGraphEvidence { + return { + file: graphFile(root, evidence.file), + startLine: evidence.startLine, + startCol: evidence.startColumn, + endLine: evidence.endLine, + endCol: evidence.endColumn, + }; +} + +function assertSnapshot( + value: unknown, + root: string, + contract: CompilerGraphSnapshotAdapter.IContract, +): ICompilerGraphSnapshot { + if (!isRecord(value)) { + throw new Error(`${contract.label}: the graph artifact is not an object`); + } + const raw = value as unknown as ICompilerGraphSnapshot; + if (raw.schemaVersion !== contract.schemaVersion) { + throw new Error( + `${contract.label}: unsupported artifact schema ${String(raw.schemaVersion)}; this adapter reads ${String(contract.schemaVersion)}`, + ); + } + if (!isRecord(raw.producer)) { + throw new Error(`${contract.label}: the artifact names no producer`); + } + if (raw.producer.name !== contract.producer) { + throw new Error( + `${contract.label}: foreign producer ${String(raw.producer.name)}`, + ); + } + if (raw.producer.protocolVersion !== contract.protocolVersion) { + throw new Error( + `${contract.label}: unsupported producer protocol ${String(raw.producer.protocolVersion)}`, + ); + } + if (typeof raw.producer.version !== "string" || raw.producer.version === "") { + throw new Error(`${contract.label}: the producer states no version`); + } + const capabilities = raw.producer.capabilities; + if ( + !isRecord(capabilities) || + typeof capabilities.atomicGenerations !== "boolean" || + typeof capabilities.incremental !== "boolean" || + typeof capabilities.diagnostics !== "boolean" + ) { + throw new Error(`${contract.label}: the producer states no capability block`); + } + // Atomic generations are what the whole transaction rests on. A producer + // that says it cannot commit one has published shards this route has no way + // to fence, and reading them anyway would put a half-written build behind a + // content-addressed generation identity. + if (!capabilities.atomicGenerations) { + throw new Error( + `${contract.label}: the producer does not commit atomic generations`, + ); + } + if (!capabilities.incremental) { + throw new Error( + `${contract.label}: the producer does not preserve incremental generations`, + ); + } + if (!capabilities.diagnostics) { + throw new Error( + `${contract.label}: the producer does not publish compiler diagnostics`, + ); + } + contract.validateSnapshot?.(raw); + if (typeof raw.projectRoot !== "string" || raw.projectRoot === "") { + throw new Error(`${contract.label}: the artifact names no project root`); + } + if (!samePath(raw.projectRoot, root)) { + throw new Error( + `${contract.label}: the artifact was produced for ${raw.projectRoot}, not ${root}`, + ); + } + if (!Array.isArray(raw.targets) || raw.targets.length === 0) { + throw new Error(`${contract.label}: the artifact committed no target`); + } + const names = new Set(); + const sources = new Set(); + for (const target of raw.targets) { + assertTarget(target, names, sources, root, contract); + } + return raw; +} + +function assertTarget( + target: ICompilerGraphSnapshot.ITarget, + names: Set, + sources: Set, + root: string, + contract: CompilerGraphSnapshotAdapter.IContract, +): void { + if ( + !isRecord(target) || + typeof target.name !== "string" || + target.name === "" || + !SHA256.test(target.generation) || + !SHA256.test(target.universe) || + !isRecord(target.coverage) || + !Array.isArray(target.shards) || + target.shards.length === 0 + ) { + throw new Error(`${contract.label}: malformed committed target`); + } + contract.validateTarget?.(target); + if (names.has(target.name)) { + throw new Error(`${contract.label}: duplicate committed target ${target.name}`); + } + names.add(target.name); + // Every family, every time. A matrix missing a row cannot be read as either + // "complete" or "unsupported", and the difference between those two is the + // whole reason a consumer is allowed to treat an absent edge as absence. + const families = Object.keys(target.coverage); + if ( + families.length !== GRAPH_EDGE_KINDS.length || + GRAPH_EDGE_KINDS.some( + (family) => + !COVERAGE_STATES.has( + target.coverage[family] as ISamchonGraphCoverage["state"], + ), + ) + ) { + throw new Error( + `${contract.label}: target ${target.name} has an incomplete coverage matrix`, + ); + } + // A family this route is not registered to prove cannot be claimed complete + // by the producer either; the two statements would contradict each other in + // the same generation. + for (const family of GRAPH_EDGE_KINDS) { + if ( + !contract.facts.includes(family) && + target.coverage[family] !== "unsupported" + ) { + throw new Error( + `${contract.label}: target ${target.name} claims ${family}, which this route does not prove`, + ); + } + } + for (const shard of target.shards) { + assertShard(shard, target, sources, root, contract); + } +} + +function assertShard( + shard: ICompilerGraphSnapshot.IShard, + target: ICompilerGraphSnapshot.ITarget, + sources: Set, + root: string, + contract: CompilerGraphSnapshotAdapter.IContract, +): void { + if ( + !isRecord(shard) || + shard.schemaVersion !== contract.schemaVersion || + shard.language !== contract.language || + typeof shard.source !== "string" || + shard.source === "" || + // Canonical, project-relative and POSIX. Two walks over these edges compare + // an endpoint against this string: one against the producer's spelling of + // it, one against the normalized path a graph node carries. `./a/File.ext` + // and `a/File.ext` are the same file and different strings, so a producer + // that spells it the first way makes those walks disagree about what an + // endpoint is—and the disagreement surfaces as a dereference of the + // lookup that did not find it rather than as a sentence naming the defect. + // Requiring one spelling is what makes the two walks provably the same. + !isCanonicalRelativeSource(shard.source) || + !SHA256.test(shard.checkerDigest) || + // A disk digest is required, not optional. This route claims the + // `diskDigests` capability, and the coordinator will not publish a + // generation whose sources it cannot hash for itself—so an empty one is + // a snapshot that parses, validates, and is then refused three refreshes + // later by a fence that cannot say it was the producer's doing. Every + // compiler unit has an on-disk identity, including a generated one; + // a producer that could not read one has not proved what it compiled. + !SHA256.test(shard.diskDigest) || + shard.target !== target.name || + typeof shard.compilerVersion !== "string" || + shard.compilerVersion === "" || + !Array.isArray(shard.nodes) || + !Array.isArray(shard.edges) || + !Array.isArray(shard.unresolved) || + !Array.isArray(shard.diagnostics) + ) { + throw new Error( + `${contract.label}: malformed shard in target ${target.name}`, + ); + } + contract.validateShard?.(shard, target, root); + const key = `${target.name}\0${shard.source}`; + if (sources.has(key)) { + throw new Error( + `${contract.label}: source ${shard.source} is committed twice in target ${target.name}`, + ); + } + sources.add(key); + const symbols = new Set(); + for (const node of shard.nodes) { + if ( + !isRecord(node) || + typeof node.symbol !== "string" || + node.symbol === "" || + !NODE_KINDS.has(node.kind as GraphNodeKind) || + typeof node.name !== "string" || + node.name === "" || + typeof node.qualifiedName !== "string" || + typeof node.file !== "string" || + node.file !== shard.source || + typeof node.exported !== "boolean" || + !Array.isArray(node.modifiers) || + node.modifiers.some( + (modifier) => !MODIFIERS.has(modifier as SamchonGraphNodeModifier), + ) || + new Set(node.modifiers).size !== node.modifiers.length || + typeof node.signature !== "string" || + typeof node.origin !== "string" || + node.origin === "" + ) { + throw new Error( + `${contract.label}: malformed declaration in ${shard.source}`, + ); + } + if (symbols.has(node.symbol)) { + throw new Error( + `${contract.label}: duplicate declaration ${node.symbol} in ${shard.source}`, + ); + } + symbols.add(node.symbol); + assertEvidence(node.evidence, shard.source, contract); + } + for (const edge of shard.edges) { + if ( + !isRecord(edge) || + typeof edge.from !== "string" || + edge.from === "" || + typeof edge.to !== "string" || + edge.to === "" || + !GRAPH_EDGE_KINDS.includes(edge.kind as GraphEdgeKind) || + !contract.facts.includes(edge.kind as GraphEdgeKind) || + !isNullableString(edge.access) || + !isNullableString(edge.provenance) || + !isNullableString(edge.targetName) || + !isNullableString(edge.targetQualifiedName) || + (edge.targetKind !== null && + !NODE_KINDS.has(edge.targetKind as GraphNodeKind)) + ) { + throw new Error(`${contract.label}: malformed edge in ${shard.source}`); + } + assertEvidence(edge.evidence, shard.source, contract); + } + for (const site of shard.unresolved) { + if ( + !isRecord(site) || + !GRAPH_EDGE_KINDS.includes(site.family as GraphEdgeKind) || + !UNRESOLVED_REASONS.has(site.reason as ISamchonGraphUnresolved["reason"]) || + !Array.isArray(site.candidates) || + site.candidates.some((candidate) => typeof candidate !== "string") || + new Set(site.candidates).size !== site.candidates.length + ) { + throw new Error( + `${contract.label}: malformed unresolved site in ${shard.source}`, + ); + } + assertEvidence(site.evidence, shard.source, contract); + } + for (const diagnostic of shard.diagnostics) { + if ( + !isRecord(diagnostic) || + !["error", "warning", "info", "hint"].includes(diagnostic.severity) || + typeof diagnostic.message !== "string" || + diagnostic.message === "" + ) { + throw new Error( + `${contract.label}: malformed diagnostic in ${shard.source}`, + ); + } + assertEvidence(diagnostic.evidence, shard.source, contract); + } +} + +function assertEvidence( + evidence: ICompilerGraphSnapshot.IEvidence, + source: string, + contract: CompilerGraphSnapshotAdapter.IContract, +): void { + if ( + !isRecord(evidence) || + typeof evidence.file !== "string" || + evidence.file !== source || + !positiveInteger(evidence.startLine) || + !positiveInteger(evidence.startColumn) || + !positiveInteger(evidence.endLine) || + !positiveInteger(evidence.endColumn) + ) { + throw new Error(`${contract.label}: malformed evidence in ${source}`); + } +} + +/** Whether the producer named a source the way both endpoint walks read it. */ +function isCanonicalRelativeSource(source: string): boolean { + return ( + !source.includes("\0") && + !source.includes("\\") && + !path.posix.isAbsolute(source) && + !/^[A-Za-z]:/u.test(source) && + path.posix.normalize(source) === source && + source + .split("/") + .every((part) => part !== "" && part !== "." && part !== "..") + ); +} + +function shardKey( + contract: CompilerGraphSnapshotAdapter.IContract, + target: string, + source: string, +): string { + return `${contract.shardKeyPrefix}-shard:${digest([target, source])}`; +} + +function identityKey( + contract: CompilerGraphSnapshotAdapter.IContract, + target: ICompilerGraphSnapshot.ITarget, + symbol: string, +): string { + const salt = contract.identitySalt?.(target); + return salt === undefined ? symbol : `${salt}\0${symbol}`; +} + +function graphFile(root: string, file: string): string { + return path + .relative(root, path.resolve(root, file)) + .split(path.sep) + .join("/"); +} + +function sourcePath(root: string, file: string): string { + return path.normalize(path.resolve(root, file)); +} + +function positiveInteger(value: number): boolean { + return Number.isSafeInteger(value) && value >= 1; +} + +function isNullableString(value: unknown): boolean { + return value === null || typeof value === "string"; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function samePath(left: string, right: string): boolean { + const normalizedLeft = path.resolve(left); + const normalizedRight = path.resolve(right); + /* c8 ignore next 3 -- only one platform arm runs on a given OS. */ + return process.platform === "win32" + ? normalizedLeft.toLowerCase() === normalizedRight.toLowerCase() + : normalizedLeft === normalizedRight; +} + +function digest(value: unknown): string { + return createHash("sha256").update(canonical(value)).digest("hex"); +} + +function canonical(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) { + return `[${value.map(canonical).join(",")}]`; + } + return `{${Object.keys(value as Record) + .sort(compareText) + .map( + (key) => + `${JSON.stringify(key)}:${canonical((value as Record)[key])}`, + ) + .join(",")}}`; +} + +function compareText(left: string, right: string): number { + // Two-way: every collection sorted through here holds distinct keys, so the + // equal arm is unreachable and covering it by directive would stop the gate + // from enforcing the ordering itself. + return left < right ? -1 : 1; +} + +function sameList( + left: readonly string[], + right: readonly string[], +): boolean { + return ( + left.length === right.length && + left.every((value, index) => value === right[index]) + ); +} diff --git a/packages/graph/src/provider/compiler/ICompilerGraphSnapshot.ts b/packages/graph/src/provider/compiler/ICompilerGraphSnapshot.ts new file mode 100644 index 00000000..8717af84 --- /dev/null +++ b/packages/graph/src/provider/compiler/ICompilerGraphSnapshot.ts @@ -0,0 +1,95 @@ +/** + * Shared wire shape for compiler-owned, target-scoped graph generations. + * + * Language adapters validate any additional producer metadata before this + * common shape is converted to the Graph Snapshot Protocol. + */ +export interface ICompilerGraphSnapshot { + schemaVersion: number; + projectRoot: string; + producer: ICompilerGraphSnapshot.IProducer; + targets: ICompilerGraphSnapshot.ITarget[]; +} + +export namespace ICompilerGraphSnapshot { + export interface IProducer { + name: string; + version: string; + protocolVersion: number; + capabilities: ICapabilities; + } + + export interface ICapabilities { + atomicGenerations: boolean; + incremental: boolean; + diagnostics: boolean; + } + + export interface ITarget { + name: string; + generation: string; + universe: string; + coverage: Record; + shards: IShard[]; + } + + export interface IShard { + schemaVersion: number; + language: string; + source: string; + checkerDigest: string; + diskDigest: string; + target: string; + compilerVersion: string; + nodes: INode[]; + edges: IEdge[]; + unresolved: IUnresolved[]; + diagnostics: IDiagnostic[]; + } + + export interface IEvidence { + file: string; + startLine: number; + startColumn: number; + endLine: number; + endColumn: number; + } + + export interface INode { + symbol: string; + kind: string; + name: string; + qualifiedName: string; + file: string; + exported: boolean; + modifiers: string[]; + signature: string; + origin: string; + evidence: IEvidence; + } + + export interface IEdge { + from: string; + to: string; + kind: string; + access: string | null; + provenance: string | null; + targetKind: string | null; + targetName: string | null; + targetQualifiedName: string | null; + evidence: IEvidence; + } + + export interface IUnresolved { + family: string; + reason: string; + evidence: IEvidence; + candidates: string[]; + } + + export interface IDiagnostic { + severity: string; + message: string; + evidence: IEvidence; + } +} diff --git a/packages/graph/src/provider/compiler/ResidentGraphProducerClient.ts b/packages/graph/src/provider/compiler/ResidentGraphProducerClient.ts new file mode 100644 index 00000000..76f5b33b --- /dev/null +++ b/packages/graph/src/provider/compiler/ResidentGraphProducerClient.ts @@ -0,0 +1,375 @@ +import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; + +import { ownedProcess } from "../../utils/ownedProcess"; +import { spawnableCommand } from "../../utils/spawnableCommand"; +import { IGraphProvider } from "../IGraphProvider"; + +const PROTOCOL_VERSION = 1; +const DEFAULT_REQUEST_TIMEOUT_MS = 300_000; +const DEFAULT_MAX_RESPONSE_BYTES = 64 * 1024; +const MAX_STDERR_CHARS = 64 * 1024; +const MAX_TIMER_MS = 2_147_483_647; + +interface Child { + process: ChildProcessWithoutNullStreams; + response: string; + responseBytes: number; + stderr: string; + exit: Promise; + termination?: Promise; +} + +interface Pending { + child: Child; + resolve: (value: undefined) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; + signal?: AbortSignal; + abort?: () => void; +} + +/** Restartable NDJSON client for a compiler graph producer. */ +export class ResidentGraphProducerClient { + private readonly root: string; + private readonly provider: string; + private readonly command: IGraphProvider.ICommand; + private readonly serverCommand: string; + private readonly label: string; + private readonly requestTimeoutMs: number; + private readonly maxResponseBytes: number; + private child: Child | undefined; + private readonly ownedChildren = new Set(); + private readonly pending = new Map(); + private nextId = 1; + private closed = false; + private closing: Promise | undefined; + + public constructor(options: ResidentGraphProducerClient.IOptions) { + this.root = options.root; + this.provider = options.provider; + this.command = options.command; + this.serverCommand = options.serverCommand; + this.label = options.label; + this.requestTimeoutMs = + options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + this.maxResponseBytes = + options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES; + if ( + !Number.isSafeInteger(this.requestTimeoutMs) || + this.requestTimeoutMs < 1 || + this.requestTimeoutMs > MAX_TIMER_MS + ) { + throw new TypeError( + `${this.provider}: requestTimeoutMs must be an integer between 1 and ${String(MAX_TIMER_MS)}`, + ); + } + if (!Number.isSafeInteger(this.maxResponseBytes) || this.maxResponseBytes < 1) { + throw new TypeError( + `${this.provider}: maxResponseBytes must be a positive safe integer`, + ); + } + } + + public produce( + artifact: string, + signal: AbortSignal | undefined, + ): Promise { + if (this.closed) { + return Promise.reject(new Error(`${this.provider}: session is closed`)); + } + if (signal?.aborted === true) { + return Promise.reject(cancelled(this.provider, this.label)); + } + const child = this.ensureChild(); + child.stderr = ""; + const id = this.nextId++; + return new Promise((resolve, reject) => { + const pending: Pending = { + child, + resolve, + reject, + timer: setTimeout(() => { + this.failChild( + child, + new Error( + `${this.provider}: ${this.label} request timed out after ${String(this.requestTimeoutMs)} ms${stderrSuffix(child)}`, + ), + ); + }, this.requestTimeoutMs), + signal, + }; + pending.timer.unref(); + this.pending.set(id, pending); + if (signal !== undefined) { + pending.abort = () => + this.failChild(child, cancelled(this.provider, this.label)); + signal.addEventListener("abort", pending.abort, { once: true }); + } + if (signal?.aborted === true) { + pending.abort!(); + return; + } + child.process.stdin.write( + `${JSON.stringify({ id, protocolVersion: PROTOCOL_VERSION, output: artifact })}\n`, + ); + }); + } + + public close(): Promise { + if (this.closing !== undefined) return this.closing; + this.closed = true; + const failure = new Error(`${this.provider}: session is closed`); + this.failPending(failure); + this.child = undefined; + this.closing = Promise.all( + [...this.ownedChildren].map((child) => this.terminate(child)), + ).then(() => undefined); + return this.closing; + } + + private ensureChild(): Child { + /* c8 ignore next -- produce rejects a closed client before it can call here. */ + if (this.closed) throw new Error(`${this.provider}: session is closed`); + if ( + this.child !== undefined && + this.child.process.exitCode === null && + this.child.process.signalCode === null + ) { + return this.child; + } + const invocation = spawnableCommand.append( + { ...this.command, args: [...this.command.args] }, + [this.serverCommand, "--cwd", this.root], + ); + const command = ownedProcess.command( + invocation.command, + invocation.args, + invocation.windowsVerbatimArguments, + ); + const process = spawn(command.command, command.args, { + cwd: this.root, + env: globalThis.process.env, + detached: ownedProcess.group(), + shell: false, + stdio: ownedProcess.stdio(command, ["pipe", "pipe", "pipe"]), + windowsHide: true, + windowsVerbatimArguments: command.windowsVerbatimArguments, + }) as ChildProcessWithoutNullStreams; + ownedProcess.start(process, command); + const child: Child = { + process, + response: "", + responseBytes: 0, + stderr: "", + exit: ownedProcess.exit(process), + }; + this.child = child; + this.ownedChildren.add(child); + process.stdout.setEncoding("utf8"); + process.stderr.setEncoding("utf8"); + process.stdout.on("data", (chunk: string) => this.consume(child, chunk)); + process.stderr.on("data", (chunk: string) => { + child.stderr = (child.stderr + chunk).slice(-MAX_STDERR_CHARS); + }); + /* c8 ignore start -- the Windows process-group shim reports a failed + * launch through stderr and exit; POSIX emits this direct child event. */ + process.on("error", (error) => + this.failChild( + child, + new Error(`${this.provider}: ${this.label} server failed: ${error.message}`), + ), + ); + /* c8 ignore stop */ + process.stdin.on("error", (error) => + this.failChild( + child, + new Error( + `${this.provider}: ${this.label} server stdin failed: ${error.message}${stderrSuffix(child)}`, + ), + ), + ); + process.on("exit", (code, signal) => + this.failChild( + child, + new Error( + `${this.provider}: ${this.label} server exited (${String(signal ?? code)})${stderrSuffix(child)}`, + ), + ), + ); + return child; + } + + private consume(child: Child, chunk: string): void { + // Only a buffered event delivered after retirement can name an old child; + // owned termination discards it and has no state effect. + /* c8 ignore next */ + if (this.child !== child) return; + let start = 0; + for (;;) { + const newline = chunk.indexOf("\n", start); + if (newline === -1) { + if (start < chunk.length) this.append(child, chunk.slice(start)); + return; + } + if (!this.append(child, chunk.slice(start, newline))) return; + const line = child.response.trim(); + child.response = ""; + child.responseBytes = 0; + start = newline + 1; + if (line === "") continue; + let value: unknown; + try { + value = JSON.parse(line) as unknown; + } catch (error) { + this.failChild( + child, + new Error( + `${this.provider}: invalid ${this.label} server response: ${asError(error).message}`, + ), + ); + return; + } + let response: IResponse; + try { + response = parseResponse(value, this.provider, this.label); + } catch (error) { + this.failChild(child, asError(error)); + return; + } + const pending = this.pending.get(response.id); + if (pending === undefined) { + this.failChild( + child, + new Error( + `${this.provider}: unexpected ${this.label} response id ${String(response.id)}`, + ), + ); + return; + } + // Request IDs never repeat and a pending request is inserted only with + // the one current child that receives its response. + /* c8 ignore start */ + if (pending.child !== child) { + this.failChild( + child, + new Error( + `${this.provider}: ${this.label} response crossed producer processes`, + ), + ); + return; + } + /* c8 ignore stop */ + if (response.ok) this.settle(response.id, pending); + else { + this.settle( + response.id, + pending, + new Error(`${this.provider}: ${response.error}${stderrSuffix(child)}`), + ); + } + } + } + + private append(child: Child, chunk: string): boolean { + child.responseBytes += Buffer.byteLength(chunk, "utf8"); + if (child.responseBytes > this.maxResponseBytes) { + this.failChild( + child, + new Error( + `${this.provider}: ${this.label} response exceeded the ${String(this.maxResponseBytes)} byte limit`, + ), + ); + return false; + } + child.response += chunk; + return true; + } + + private settle(id: number, pending: Pending, error?: Error): void { + this.pending.delete(id); + clearTimeout(pending.timer); + if (pending.abort !== undefined) { + pending.signal?.removeEventListener("abort", pending.abort); + } + if (error === undefined) pending.resolve(undefined); + else pending.reject(error); + } + + private failChild(child: Child, error: Error): void { + if (this.child !== child) return; + this.child = undefined; + this.failPending(error, child); + void this.terminate(child) + .then(() => this.ownedChildren.delete(child)) + .catch(() => undefined); + } + + private failPending(error: Error, child?: Child): void { + for (const [id, pending] of this.pending) { + if (child !== undefined) { + // One current producer owns every pending request; a replacement starts + // only after this synchronous loop settles it. + /* c8 ignore next */ + if (pending.child !== child) continue; + } + this.settle(id, pending, error); + } + } + + private terminate(child: Child): Promise { + child.termination ??= ownedProcess.terminate( + child.process, + child.exit, + this.provider, + { cooperativeStdin: true }, + ); + return child.termination; + } +} + +function parseResponse(value: unknown, provider: string, label: string): IResponse { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${provider}: ${label} response must be an object`); + } + const row = value as Record; + if (!Number.isSafeInteger(row.id) || row.protocolVersion !== PROTOCOL_VERSION) { + throw new Error(`${provider}: invalid ${label} response identity`); + } + if (row.ok === true) return { id: row.id as number, ok: true }; + if (row.ok === false && typeof row.error === "string" && row.error !== "") { + return { id: row.id as number, ok: false, error: row.error }; + } + throw new Error(`${provider}: invalid ${label} response result`); +} + +function cancelled(provider: string, label: string): Error { + const error = new Error(`${provider}: ${label} request was aborted`); + error.name = "AbortError"; + return error; +} + +function stderrSuffix(child: Child): string { + const text = child.stderr.trim(); + return text === "" ? "" : `\nstderr:\n${text}`; +} + +function asError(error: unknown): Error { + /* c8 ignore next -- JSON.parse and parseResponse throw Error instances. */ + return error instanceof Error ? error : new Error(String(error)); +} + +type IResponse = + | { id: number; ok: true } + | { id: number; ok: false; error: string }; + +export namespace ResidentGraphProducerClient { + export interface IOptions { + root: string; + provider: string; + command: IGraphProvider.ICommand; + serverCommand: string; + label: string; + requestTimeoutMs?: number; + maxResponseBytes?: number; + } +} diff --git a/packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts b/packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts index 7d87e709..92287c2b 100644 --- a/packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts +++ b/packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts @@ -1,3 +1,3 @@ /** Exact samchon/llvm-project producer revision required by this adapter. */ export const CPP_CLANG_PRODUCER_COMMIT = - "e33d8f51552a523b5696691738f1ef95f8e3a730"; + "d6371c37445998d24776692a27e086bb24f9916a"; diff --git a/packages/graph/src/provider/cpp/CppGraphClient.ts b/packages/graph/src/provider/cpp/CppGraphClient.ts index eaca5cda..955cf4a8 100644 --- a/packages/graph/src/provider/cpp/CppGraphClient.ts +++ b/packages/graph/src/provider/cpp/CppGraphClient.ts @@ -9,6 +9,7 @@ import { appendAll } from "../../indexer/appendAll"; import { LspClient } from "../../lsp/LspClient"; import { LspResponseError } from "../../lsp/LspResponseError"; import { GraphLanguage } from "../../typings"; +import { isSubPath } from "../../utils/isSubPath"; import { IBulkGraphSession } from "../IBulkGraphSession"; import { cppGraphHeapTrace } from "./cppGraphHeapTrace"; import { CppGraphReloadRequired } from "./CppGraphReloadRequired"; @@ -18,7 +19,6 @@ import { ICppGraphSnapshot } from "./ICppGraphSnapshot"; const GRAPH_METHOD = "samchon/graphSnapshot"; const SERVER_CANCELLED = -32802; const CONTENT_MODIFIED = -32801; -const DEFAULT_READY_TIMEOUT_MS = 300_000; /** * How much published-body text this consumer keeps parsed at once. * @@ -80,12 +80,16 @@ export class CppGraphClient implements IBulkGraphSession { ) => void; private readonly initializationOptions: unknown; private readonly requestTimeoutMs: number | undefined; - private readonly readyTimeoutMs: number; + private readonly readyTimeoutMs: number | undefined; private readonly pieceBudgetBytes: number; private readonly lifecycleAbort = new AbortController(); private queue: Promise = Promise.resolve(); private initialized: Promise | undefined; - private watchedInputs = new Map(); + private watchedInputs = new Map(); + private readonly dirtyInputs = new Set(); + private readonly inputWatches = new Map(); + private readonly inputParentWatches = new Map(); + private polledInputs = new Set(); private version = 0; private closed = false; private closing: Promise | undefined; @@ -101,7 +105,7 @@ export class CppGraphClient implements IBulkGraphSession { this.validate = options.validate ?? (() => undefined); this.initializationOptions = options.initializationOptions; this.requestTimeoutMs = options.requestTimeoutMs; - this.readyTimeoutMs = options.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS; + this.readyTimeoutMs = options.readyTimeoutMs; this.pieceBudgetBytes = options.pieceBudgetBytes ?? PIECE_BUDGET_BYTES; this.lsp = new LspClient( options.command, @@ -132,9 +136,9 @@ export class CppGraphClient implements IBulkGraphSession { return this.enqueue(async () => { const signal = combineSignals(options.signal, this.lifecycleAbort.signal); await this.initialize(signal); - const moved = this.notifyInputChanges(); + const moved = await this.notifyInputChanges(); const result = await this.applySnapshot(signal, moved); - this.commitSnapshotInputs(result.snapshot); + if (result.changed) this.commitSnapshotInputs(result.snapshot); if (!result.changed) { return { changed: false, @@ -157,6 +161,7 @@ export class CppGraphClient implements IBulkGraphSession { if (this.closing !== undefined) return this.closing; this.closed = true; this.lifecycleAbort.abort(new Error("C/C++ clang graph: session is closed")); + this.closeInputWatches(); this.closing = this.lsp.close(); return this.closing; } @@ -189,9 +194,11 @@ export class CppGraphClient implements IBulkGraphSession { signal, ); this.lsp.notify("initialized", {}); - const inputs = inputDigests(this.root); + const files = inputFiles(this.root); + this.syncInputWatches(files); + const inputs = inputDigests(files); const changes = [...inputs] - .filter(([, digest]) => digest !== null) + .filter(([, input]) => input.digest !== null) .map(([file]) => ({ uri: pathToFileURL(file).href, type: 1 })); if (changes.length !== 0) { this.lsp.notify("workspace/didChangeWatchedFiles", { changes }); @@ -209,13 +216,54 @@ export class CppGraphClient implements IBulkGraphSession { * consumer that watched it move and then published the previous generation * unchanged would be reporting on a checkout it no longer describes. */ - private notifyInputChanges(): boolean { - const current = inputDigests(this.root, this.current); - const files = new Set([...this.watchedInputs.keys(), ...current.keys()]); + private async notifyInputChanges(): Promise { + // Let native filesystem notifications queued by the write that prompted + // this load reach their directory watchers before deciding it is a no-op. + // Project-owned inputs are also polled: direct writes followed immediately + // by load must not depend on an OS event's delivery latency. External SDK + // and dependency trees stay bound by their directory events without + // turning every no-op into a stat walk over tens of thousands of headers. + await inputEventTurn(); + if (this.closed) return false; + const required = inputFiles(this.root); + this.addInputWatches(required); + const files = new Set(this.dirtyInputs); + this.dirtyInputs.clear(); + for (const file of this.polledInputs) files.add(file); + for (const [directory, watch] of this.inputWatches) { + this.ensureInputParentWatch(directory); + // A parent watcher observes the directory entry rather than following + // its inode, so atomic replacement retires the child handle on every + // host without an O(dependency-directory count) no-op stat walk. If the + // parent cannot be watched, retain the identity poll as a correctness + // fallback only for that degraded directory. + if ( + !isSubPath(this.root, directory) && + !this.hasInputParentWatch(directory) && + watch.watcher !== undefined && + watch.identity !== directoryIdentity(directory) + ) { + watch.watcher.close(); + watch.watcher = undefined; + } + if (watch.watcher === undefined) { + for (const file of watch.files) files.add(file); + // Reattach before reading. A write before this point is in the digest; + // a write after it is held by the new watcher for this or the next + // refresh. Opening after the read would leave a lost-update window. + this.openInputWatch(directory, watch); + } + } + for (const file of required) { + if (!this.watchedInputs.has(file)) files.add(file); + } + const current = new Map(this.watchedInputs); const changes: Array<{ uri: string; type: 1 | 2 | 3 }> = []; for (const file of [...files].sort(compareText)) { - const before = this.watchedInputs.get(file); - const after = current.get(file); + const before = this.watchedInputs.get(file)?.digest; + const input = fileDigest(file, this.watchedInputs.get(file)); + const after = input.digest; + current.set(file, input); if (before === after) continue; const type = before === undefined || before === null ? 1 : after === null || after === undefined ? 3 : 2; changes.push({ uri: pathToFileURL(file).href, type }); @@ -228,22 +276,233 @@ export class CppGraphClient implements IBulkGraphSession { } private commitSnapshotInputs(snapshot: IBulkGraphSession.ISnapshot): void { - const committed = inputDigests(this.root, snapshot); + const files = inputFiles(this.root, snapshot); + // Attach watchers before reading the post-snapshot baseline. A file that + // moves during the read is either rejected by fileDigest's stable-read + // fence or arrives as a dirty event checked by the next resident load. + this.syncInputWatches(files); + const committed = inputDigests(files, this.watchedInputs); for (const [file, source] of snapshot.sources) { if (!path.isAbsolute(file)) continue; - committed.set( - file, - source.diskDigest === "" ? null : source.diskDigest, - ); + const digest = source.diskDigest === "" ? null : source.diskDigest; + const scanned = committed.get(file); + committed.set(file, { + digest, + // A producer digest is a baseline for the next refresh only. Reuse the + // scan fingerprint when it proves those are the bytes on disk now; if + // the file moved after the frozen snapshot, force the next refresh to + // read it and compare against the producer's older identity. + fingerprint: + scanned?.digest === digest ? scanned.fingerprint : null, + }); } this.watchedInputs = committed; } + private addInputWatches(files: Iterable): void { + for (const file of files) { + if (isSubPath(this.root, file)) this.polledInputs.add(file); + const directory = path.dirname(file); + const current = this.inputWatches.get(directory); + if (current !== undefined) { + current.files.add(file); + this.ensureInputParentWatch(directory); + continue; + } + const watch: IInputWatch = { + files: new Set([file]), + identity: null, + }; + this.inputWatches.set(directory, watch); + this.ensureInputParentWatch(directory); + this.openInputWatch(directory, watch); + } + } + + private syncInputWatches(files: Iterable): void { + const wanted = new Map>(); + const polled = new Set(); + for (const file of files) { + if (isSubPath(this.root, file)) polled.add(file); + const directory = path.dirname(file); + let entries = wanted.get(directory); + if (entries === undefined) { + entries = new Set(); + wanted.set(directory, entries); + } + entries.add(file); + } + for (const [directory, watch] of this.inputWatches) { + const entries = wanted.get(directory); + if (entries === undefined) { + watch.watcher?.close(); + this.inputWatches.delete(directory); + this.releaseInputParentWatch(directory); + for (const file of watch.files) this.dirtyInputs.delete(file); + continue; + } + for (const file of watch.files) { + if (!entries.has(file)) this.dirtyInputs.delete(file); + } + watch.files = entries; + wanted.delete(directory); + this.ensureInputParentWatch(directory); + if (watch.watcher === undefined) this.openInputWatch(directory, watch); + } + for (const [directory, entries] of wanted) { + const watch: IInputWatch = { files: entries, identity: null }; + this.inputWatches.set(directory, watch); + this.ensureInputParentWatch(directory); + this.openInputWatch(directory, watch); + } + this.polledInputs = polled; + } + + private openInputWatch(directory: string, watch: IInputWatch): void { + const before = directoryIdentity(directory); + try { + const watcher = fs.watch(directory, { persistent: false }, (event) => { + if ( + this.closed || + this.inputWatches.get(directory) !== watch || + watch.watcher !== watcher + ) + return; + for (const file of watch.files) this.dirtyInputs.add(file); + // A rename can be the watched directory itself being atomically + // replaced. Native watchers follow the old inode on Unix, so retire + // this handle and reopen the path after its files have been polled. + if (event === "rename" && watch.watcher === watcher) { + watcher.close(); + watch.watcher = undefined; + } + }); + watcher.on("error", () => { + if ( + this.closed || + this.inputWatches.get(directory) !== watch || + watch.watcher !== watcher + ) + return; + for (const file of watch.files) this.dirtyInputs.add(file); + watcher.close(); + watch.watcher = undefined; + }); + watch.watcher = watcher; + const after = directoryIdentity(directory); + watch.identity = after; + if (before !== after && watch.watcher === watcher) { + // The path moved between proving its identity and attaching the + // handle. Poll its files now and retry the handle at the next sync. + for (const file of watch.files) this.dirtyInputs.add(file); + watcher.close(); + watch.watcher = undefined; + } + } catch { + // A missing build directory and filesystems without watch support stay + // correct by polling the files assigned to this directory on each load. + watch.identity = directoryIdentity(directory); + watch.watcher = undefined; + } + } + + private ensureInputParentWatch(directory: string): void { + if (isSubPath(this.root, directory)) return; + const parent = path.dirname(directory); + let watch = this.inputParentWatches.get(parent); + if (watch === undefined) { + watch = { directories: new Set() }; + this.inputParentWatches.set(parent, watch); + } + watch.directories.add(directory); + if (watch.watcher !== undefined) return; + try { + const watcher = fs.watch( + parent, + { persistent: false }, + (event, filename) => { + if ( + this.closed || + this.inputParentWatches.get(parent) !== watch || + watch.watcher !== watcher + ) + return; + if (event !== "rename") return; + const changed = + filename === null + ? undefined + : path.resolve(parent, filename.toString()).toLowerCase(); + for (const child of watch.directories) { + if (changed !== undefined && child.toLowerCase() !== changed) { + continue; + } + this.retireInputWatch(child); + } + }, + ); + watcher.on("error", () => { + if ( + this.closed || + this.inputParentWatches.get(parent) !== watch || + watch.watcher !== watcher + ) + return; + watcher.close(); + watch.watcher = undefined; + for (const child of watch.directories) { + this.retireInputWatch(child); + } + }); + watch.watcher = watcher; + } catch { + // `notifyInputChanges` identity-polls only the external directories whose + // parent watch could not be opened, and retries this attachment next load. + watch.watcher = undefined; + } + } + + private hasInputParentWatch(directory: string): boolean { + return ( + this.inputParentWatches.get(path.dirname(directory))?.watcher !== undefined + ); + } + + private releaseInputParentWatch(directory: string): void { + if (isSubPath(this.root, directory)) return; + const parent = path.dirname(directory); + const watch = this.inputParentWatches.get(parent); + if (watch === undefined) return; + watch.directories.delete(directory); + if (watch.directories.size !== 0) return; + watch.watcher?.close(); + this.inputParentWatches.delete(parent); + } + + private retireInputWatch(directory: string): void { + const watch = this.inputWatches.get(directory); + if (watch === undefined) return; + for (const file of watch.files) this.dirtyInputs.add(file); + watch.watcher?.close(); + watch.watcher = undefined; + } + + private closeInputWatches(): void { + for (const watch of this.inputWatches.values()) watch.watcher?.close(); + for (const watch of this.inputParentWatches.values()) watch.watcher?.close(); + this.inputWatches.clear(); + this.inputParentWatches.clear(); + this.dirtyInputs.clear(); + this.polledInputs.clear(); + } + private async requestSnapshot( signal: AbortSignal, moved: boolean, ): Promise { - const deadline = performance.now() + this.readyTimeoutMs; + const deadline = + this.readyTimeoutMs === undefined + ? undefined + : performance.now() + this.readyTimeoutMs; let backoff = RETRY_DELAY_MS; let waiting: string | undefined; for (;;) { @@ -285,7 +544,7 @@ export class CppGraphClient implements IBulkGraphSession { "graph snapshot is not ready", ); if (error.code === CONTENT_MODIFIED && !indexing) - this.notifyInputChanges(); + await this.notifyInputChanges(); // Say what is being waited on, once per distinct answer. // // The producer refuses until every translation unit the compilation @@ -304,7 +563,9 @@ export class CppGraphClient implements IBulkGraphSession { `@samchon/graph: c, cpp: waiting for the clang graph producer: ${waiting}\n`, ); } - if (performance.now() >= deadline) { + const remaining = + deadline === undefined ? undefined : deadline - performance.now(); + if (remaining !== undefined && remaining <= 0) { throw new Error( `C/C++ clang graph: producer did not become ready within ${String(this.readyTimeoutMs)} ms: ${error.message}`, ); @@ -315,7 +576,7 @@ export class CppGraphClient implements IBulkGraphSession { // bound quietly widened — the thing this provider keeps having to // correct elsewhere. await delay( - Math.min(backoff, Math.max(0, deadline - performance.now())), + remaining === undefined ? backoff : Math.min(backoff, remaining), signal, ); // Backing off, because polling twenty times a second for a condition @@ -631,6 +892,22 @@ interface ICompileCommand { file?: unknown; } +interface IInputDigest { + digest: string | null; + fingerprint: string | null; +} + +interface IInputWatch { + files: Set; + identity: string | null; + watcher?: fs.FSWatcher; +} + +interface IInputParentWatch { + directories: Set; + watcher?: fs.FSWatcher; +} + function compilationDatabaseFiles(root: string): string[] { for (const candidate of [ path.join(root, "compile_commands.json"), @@ -661,10 +938,10 @@ function compilationDatabaseFiles(root: string): string[] { return []; } -function inputDigests( +function inputFiles( root: string, snapshot?: IBulkGraphSession.ISnapshot, -): Map { +): string[] { const files = new Set([ path.join(root, ".clangd"), path.join(root, "compile_flags.txt"), @@ -675,21 +952,75 @@ function inputDigests( for (const file of snapshot?.sources.keys() ?? []) { if (path.isAbsolute(file)) files.add(file); } + return [...files].sort(compareText); +} + +function inputDigests( + files: Iterable, + previous: ReadonlyMap = new Map(), +): Map { return new Map( [...files] .sort(compareText) - .map((file) => [file, fileDigest(file)] as const), + .map((file) => [file, fileDigest(file, previous.get(file))] as const), ); } -function fileDigest(file: string): string | null { +function fileDigest( + file: string, + previous: IInputDigest | undefined, +): IInputDigest { + // The no-op lifecycle is allowed to be a metadata walk, not a whole-corpus + // byte walk. ctime joins mtime and size because callers can restore mtime; + // inode and device keep a replacement from inheriting the old identity. + // A regular writer cannot restore ctime, on either Unix or NTFS. + for (let attempt = 0; attempt !== 3; ++attempt) { + try { + const before = fileFingerprint(file); + if (before === null) return { digest: null, fingerprint: null }; + if (previous?.fingerprint === before) return previous; + const digest = createHash("sha256") + .update(fs.readFileSync(file)) + .digest("hex"); + const after = fileFingerprint(file); + if (before === after) return { digest, fingerprint: after }; + } catch { + return { digest: null, fingerprint: null }; + } + } + // A file that keeps moving cannot establish a reusable identity. Publishing + // it as unknown makes an older producer baseline visibly move and ensures a + // later settled refresh reads the bytes again. + return { digest: null, fingerprint: null }; +} + +function fileFingerprint(file: string): string | null { + const stat = fs.statSync(file, { bigint: true }); + if (!stat.isFile()) return null; + return [ + stat.dev, + stat.ino, + stat.size, + stat.mtimeNs, + stat.ctimeNs, + ].join(":"); +} + +function directoryIdentity(directory: string): string | null { try { - return createHash("sha256").update(fs.readFileSync(file)).digest("hex"); + const stat = fs.statSync(directory, { bigint: true }); + return [stat.dev, stat.ino, stat.birthtimeNs].join(":"); } catch { return null; } } +function inputEventTurn(): Promise { + return new Promise((resolve) => { + setImmediate(resolve); + }); +} + function serverRequest(method: string, params: unknown): unknown { if (method !== "workspace/configuration") return null; const items = (params as { items?: unknown })?.items; diff --git a/packages/graph/src/provider/cpp/cppGraphProvider.ts b/packages/graph/src/provider/cpp/cppGraphProvider.ts index 401c6fc0..64d4a99f 100644 --- a/packages/graph/src/provider/cpp/cppGraphProvider.ts +++ b/packages/graph/src/provider/cpp/cppGraphProvider.ts @@ -142,12 +142,11 @@ export const cppGraphProvider: IGraphProvider = { [ "--background-index", `-j=${String(workers)}`, - // Only when somebody is reading. A producer that stops answering - // says why in its log and nowhere else, and a run that waited twenty - // minutes for one had nothing but request lines to show for it. The - // switch that passes the log through is the switch that asks for it. + // Only when somebody is reading. Info retains indexing progress, + // refusals, and reply timing without verbose transport logging copying + // a successful graph response into stderr beside the response itself. ...(process.env["SAMCHON_GRAPH_LSP_SERVER_LOG"] === "1" - ? ["--log=verbose"] + ? ["--log=info"] : []), ], ); diff --git a/packages/graph/src/provider/csharp/CSHARP_ROSLYN_FACTS.ts b/packages/graph/src/provider/csharp/CSHARP_ROSLYN_FACTS.ts new file mode 100644 index 00000000..832eb900 --- /dev/null +++ b/packages/graph/src/provider/csharp/CSHARP_ROSLYN_FACTS.ts @@ -0,0 +1,18 @@ +import { GraphEdgeKind } from "../../typings"; + +export const CSHARP_ROSLYN_FACTS = [ + "contains", + "exports", + "imports", + "calls", + "accesses", + "instantiates", + "type_ref", + "extends", + "implements", + "overrides", + "dispatches", + "decorates", + "tests", + "references", +] as const satisfies readonly GraphEdgeKind[]; diff --git a/packages/graph/src/provider/csharp/CSHARP_ROSLYN_PRODUCER.ts b/packages/graph/src/provider/csharp/CSHARP_ROSLYN_PRODUCER.ts new file mode 100644 index 00000000..e5ce4c90 --- /dev/null +++ b/packages/graph/src/provider/csharp/CSHARP_ROSLYN_PRODUCER.ts @@ -0,0 +1,2 @@ +/** Executable name of the shipped Roslyn workspace service. */ +export const CSHARP_ROSLYN_PRODUCER = "samchon-roslyn"; diff --git a/packages/graph/src/provider/csharp/CSHARP_ROSLYN_PROVIDER.ts b/packages/graph/src/provider/csharp/CSHARP_ROSLYN_PROVIDER.ts new file mode 100644 index 00000000..78972da0 --- /dev/null +++ b/packages/graph/src/provider/csharp/CSHARP_ROSLYN_PROVIDER.ts @@ -0,0 +1,2 @@ +/** Stable provider identity of the compiler-owned C# route. */ +export const CSHARP_ROSLYN_PROVIDER = "roslyn-workspace"; diff --git a/packages/graph/src/provider/csharp/CsharpGraphClient.ts b/packages/graph/src/provider/csharp/CsharpGraphClient.ts new file mode 100644 index 00000000..56f0a16f --- /dev/null +++ b/packages/graph/src/provider/csharp/CsharpGraphClient.ts @@ -0,0 +1,388 @@ +import { pathToFileURL } from "node:url"; + +import { LspClient } from "../../lsp/LspClient"; +import { LspResponseError } from "../../lsp/LspResponseError"; +import { GraphLanguage } from "../../typings"; +import { GraphSnapshotProtocol } from "../GraphSnapshotProtocol"; +import { IBulkGraphSession } from "../IBulkGraphSession"; +import { ICsharpGraphSnapshot } from "./ICsharpGraphSnapshot"; + +const COMMAND = "csharp.graph.snapshot"; +const CONTENT_MODIFIED = -32801; +const RETRY_DELAY_MS = 50; + +/** Resident client for one immutable Roslyn Solution generation. */ +export class CsharpGraphClient implements IBulkGraphSession { + public readonly kind = "bulk" as const; + public readonly languages: readonly GraphLanguage[] = ["csharp"]; + public readonly root: string; + + private readonly lsp: LspClient; + private readonly store: GraphSnapshotProtocol.Store; + private readonly validate: (snapshot: IBulkGraphSession.ISnapshot) => void; + private readonly initializationOptions: unknown; + private readonly requestTimeoutMs: number | undefined; + private readonly readyTimeoutMs: number | undefined; + private readonly lifecycleAbort = new AbortController(); + private initialized: Promise | undefined; + private queue: Promise = Promise.resolve(); + private version = 0; + private closed = false; + private closing: Promise | undefined; + + public constructor(options: CsharpGraphClient.IOptions) { + this.root = options.root; + this.store = new GraphSnapshotProtocol.Store(options.root); + this.validate = options.validate; + this.initializationOptions = options.initializationOptions; + this.requestTimeoutMs = options.requestTimeoutMs; + this.readyTimeoutMs = options.readyTimeoutMs; + this.lsp = new LspClient( + options.command, + options.args, + options.requestTimeoutMs, + options.root, + options.maxMessageBytes, + options.windowsVerbatimArguments, + ); + } + + public get generation(): number { + return this.version; + } + + public get current(): IBulkGraphSession.ISnapshot | undefined { + return this.store.current; + } + + public refresh( + options: { signal?: AbortSignal } = {}, + ): Promise { + if (this.closed) { + return Promise.reject(new Error("Roslyn workspace graph: session is closed")); + } + return this.enqueue(async () => { + const signal = combineSignals(options.signal, this.lifecycleAbort.signal); + await this.initialize(signal); + const requestStarted = performance.now(); + const raw = await this.requestSnapshot(signal); + trace("request", performance.now() - requestStarted); + const prior = this.store.current; + assertEnvelope(raw); + if (raw.mode === "unchanged") { + if ( + prior === undefined || + raw.frames.length !== 0 || + raw.sequence !== prior.protocol?.sequence || + raw.generation !== prior.protocol.generation || + raw.universe !== prior.provenance.universe + ) { + throw new Error( + "Roslyn workspace graph: unchanged envelope does not match the committed generation", + ); + } + return { + changed: false, + generation: this.version, + mode: "unchanged", + snapshot: prior, + }; + } + assertTransactionEnvelope(raw, prior); + const applyStarted = performance.now(); + const snapshot = this.store.apply(raw.frames, { + signal, + validate: this.validate, + reuseValidatedFacts: true, + }); + trace("store", performance.now() - applyStarted); + /* c8 ignore start -- the store validates these same frame coordinates + * before committing; this guards against a future store regression. */ + if ( + raw.sequence !== snapshot.protocol?.sequence || + raw.generation !== snapshot.protocol.generation || + raw.universe !== snapshot.provenance.universe + ) { + throw new Error( + "Roslyn workspace graph: response mode disagrees with its validated transaction", + ); + } + /* c8 ignore stop */ + this.version += 1; + return { + changed: true, + generation: this.version, + mode: raw.mode, + snapshot, + }; + }, options.signal); + } + + public close(): Promise { + if (this.closing !== undefined) return this.closing; + this.closed = true; + this.lifecycleAbort.abort( + new Error("Roslyn workspace graph: session is closed"), + ); + this.closing = this.lsp.close(); + return this.closing; + } + + private initialize(signal: AbortSignal): Promise { + this.initialized ??= this.initializeOnce(this.lifecycleAbort.signal); + return signal === this.lifecycleAbort.signal + ? this.initialized + : raceWithAbort(this.initialized, signal); + } + + private async initializeOnce(signal: AbortSignal): Promise { + await this.lsp.request( + "initialize", + { + processId: process.pid, + rootUri: pathToFileURL(this.root).href, + capabilities: { workspace: { configuration: true } }, + ...(this.initializationOptions === undefined + ? {} + : { initializationOptions: this.initializationOptions }), + workspaceFolders: [ + { + uri: pathToFileURL(this.root).href, + name: "samchon-graph-csharp", + }, + ], + }, + this.requestTimeoutMs, + signal, + ); + this.lsp.notify("initialized", {}); + } + + private async requestSnapshot(signal: AbortSignal): Promise { + const deadline = + this.readyTimeoutMs === undefined + ? undefined + : performance.now() + this.readyTimeoutMs; + for (;;) { + try { + return await this.lsp.request( + "workspace/executeCommand", + { + command: COMMAND, + arguments: [ + { + knownGeneration: + this.store.current?.protocol?.generation ?? null, + }, + ], + }, + this.requestTimeoutMs, + signal, + ); + } catch (error) { + if (!(error instanceof LspResponseError) || error.code !== CONTENT_MODIFIED) { + throw error; + } + const remaining = + deadline === undefined ? undefined : deadline - performance.now(); + if (remaining !== undefined && remaining <= 0) { + throw new Error( + `Roslyn workspace graph: producer inputs did not settle within ${String(this.readyTimeoutMs)} ms: ${error.message}`, + ); + } + await delay( + remaining === undefined + ? RETRY_DELAY_MS + : Math.min(RETRY_DELAY_MS, remaining), + signal, + ); + } + } + } + + private enqueue(task: () => Promise, signal?: AbortSignal): Promise { + let resolveResult!: (value: T) => void; + let rejectResult!: (error: Error) => void; + let started = false; + let settled = false; + const result = new Promise((resolve, reject) => { + resolveResult = (value) => { + settled = true; + resolve(value); + }; + rejectResult = (error) => { + settled = true; + reject(error); + }; + }); + const cancelQueued = (): void => { + if (!started) rejectResult(abortError(signal!)); + }; + if (signal?.aborted) { + rejectResult(abortError(signal)); + return result; + } + signal?.addEventListener("abort", cancelQueued, { once: true }); + this.queue = this.queue + .catch(() => undefined) + .then(async () => { + started = true; + signal?.removeEventListener("abort", cancelQueued); + if (settled) return; + try { + resolveResult(await task()); + } catch (error) { + rejectResult(asError(error)); + } + }); + return result; + } +} + +export namespace CsharpGraphClient { + export interface IOptions { + root: string; + command: string; + args: readonly string[]; + initializationOptions?: unknown; + requestTimeoutMs?: number; + readyTimeoutMs?: number; + maxMessageBytes?: number; + windowsVerbatimArguments?: boolean; + validate: (snapshot: IBulkGraphSession.ISnapshot) => void; + } +} + +function assertTransactionEnvelope( + raw: ICsharpGraphSnapshot, + prior: IBulkGraphSession.ISnapshot | undefined, +): void { + const begins = raw.frames.filter( + (frame): frame is GraphSnapshotProtocol.IBegin => frame.type === "begin", + ); + const commits = raw.frames.filter( + (frame): frame is GraphSnapshotProtocol.ICommit => frame.type === "commit", + ); + if ( + begins.length !== 1 || + commits.length !== 1 || + begins[0]!.sequence !== raw.sequence || + begins[0]!.generation !== raw.generation || + begins[0]!.universe !== raw.universe || + commits[0]!.sequence !== raw.sequence || + commits[0]!.generation !== raw.generation + ) { + throw new Error( + "Roslyn workspace graph: envelope disagrees with its frame transaction", + ); + } + const begin = begins[0]!; + const full = + begin.baseSequence === undefined && begin.baseGeneration === undefined; + const exactBase = + prior?.protocol !== undefined && + begin.baseSequence === prior.protocol.sequence && + begin.baseGeneration === prior.protocol.generation; + const expected = + prior === undefined + ? full + ? "initial" + : undefined + : prior.provenance.universe !== raw.universe + ? full + ? "reload" + : undefined + : exactBase + ? "incremental" + : full + ? "rebuild" + : undefined; + if (raw.mode !== expected) { + throw new Error( + "Roslyn workspace graph: response mode disagrees with its frame transaction", + ); + } +} + +function assertEnvelope(value: unknown): asserts value is ICsharpGraphSnapshot { + if ( + typeof value !== "object" || + value === null || + (value as ICsharpGraphSnapshot).protocolVersion !== 1 || + !["initial", "incremental", "rebuild", "reload", "unchanged"].includes( + (value as ICsharpGraphSnapshot).mode, + ) || + !Number.isSafeInteger((value as ICsharpGraphSnapshot).sequence) || + (value as ICsharpGraphSnapshot).sequence < 1 || + !/^[a-f0-9]{64}$/u.test((value as ICsharpGraphSnapshot).generation) || + !/^[a-f0-9]{64}$/u.test((value as ICsharpGraphSnapshot).universe) || + !Array.isArray((value as ICsharpGraphSnapshot).frames) + ) { + throw new Error("Roslyn workspace graph: malformed producer envelope"); + } +} + +function combineSignals( + request: AbortSignal | undefined, + lifecycle: AbortSignal, +): AbortSignal { + return request === undefined + ? lifecycle + : AbortSignal.any([request, lifecycle]); +} + +function raceWithAbort(promise: Promise, signal: AbortSignal): Promise { + /* c8 ignore start -- enqueue rejects an already-aborted caller before this + * helper can receive it; this remains a defensive standalone invariant. */ + if (signal.aborted) return Promise.reject(abortError(signal)); + /* c8 ignore stop */ + return new Promise((resolve, reject) => { + const cancel = (): void => reject(abortError(signal)); + signal.addEventListener("abort", cancel, { once: true }); + void promise + .then((value) => { + signal.removeEventListener("abort", cancel); + resolve(value); + }) + .catch((error: unknown) => { + signal.removeEventListener("abort", cancel); + reject(error); + }); + }); +} + +function delay(milliseconds: number, signal: AbortSignal): Promise { + /* c8 ignore start -- request() owns cancellation until its rejection and + * there is no asynchronous gap before this retry delay installs its owner. */ + if (signal.aborted) return Promise.reject(abortError(signal)); + /* c8 ignore stop */ + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal.removeEventListener("abort", cancel); + resolve(undefined); + }, milliseconds); + timer.unref?.(); + const cancel = (): void => { + clearTimeout(timer); + reject(abortError(signal)); + }; + signal.addEventListener("abort", cancel, { once: true }); + }); +} + +function abortError(signal: AbortSignal): Error { + const error = asError(signal.reason); + error.name = "AbortError"; + return error; +} + +function asError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + +function trace(phase: string, elapsedMs: number): void { + if (process.env["SAMCHON_GRAPH_ROSLYN_TRACE"] !== "1") return; + process.stderr.write( + `${JSON.stringify({ phase: `roslyn-client-${phase}`, elapsedMs: Math.round(elapsedMs) })}\n`, + ); +} diff --git a/packages/graph/src/provider/csharp/ICsharpGraphSnapshot.ts b/packages/graph/src/provider/csharp/ICsharpGraphSnapshot.ts new file mode 100644 index 00000000..c36f0bd6 --- /dev/null +++ b/packages/graph/src/provider/csharp/ICsharpGraphSnapshot.ts @@ -0,0 +1,12 @@ +import { GraphSnapshotProtocol } from "../GraphSnapshotProtocol"; +import { IBulkGraphSession } from "../IBulkGraphSession"; + +/** Envelope returned by the resident Roslyn service. */ +export interface ICsharpGraphSnapshot { + protocolVersion: 1; + mode: IBulkGraphSession.Mode; + sequence: number; + generation: string; + universe: string; + frames: GraphSnapshotProtocol.Frame[]; +} diff --git a/packages/graph/src/provider/csharp/csharpGraphProvider.ts b/packages/graph/src/provider/csharp/csharpGraphProvider.ts new file mode 100644 index 00000000..dc98025c --- /dev/null +++ b/packages/graph/src/provider/csharp/csharpGraphProvider.ts @@ -0,0 +1,161 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { spawnableCommand } from "../../utils/spawnableCommand"; +import { assertGraphSnapshotContract } from "../assertGraphSnapshotContract"; +import { IGraphProvider } from "../IGraphProvider"; +import { resolveProviderCommand } from "../resolveProviderCommand"; +import { standardScipProviders } from "../scip/standardScipProviders"; +import { CSHARP_ROSLYN_FACTS } from "./CSHARP_ROSLYN_FACTS"; +import { CSHARP_ROSLYN_PRODUCER } from "./CSHARP_ROSLYN_PRODUCER"; +import { CSHARP_ROSLYN_PROVIDER } from "./CSHARP_ROSLYN_PROVIDER"; +import { CsharpGraphClient } from "./CsharpGraphClient"; + +const OVERRIDE = "SAMCHON_GRAPH_ROSLYN_WORKSPACE"; +const DOTNET_OVERRIDE = "SAMCHON_GRAPH_DOTNET_TOOLCHAIN"; +const scipDotnet = standardScipProviders.find( + (provider) => provider.name === "scip-dotnet", +); +/* c8 ignore next 4 -- the static standard-provider registry always contains + * the scip-dotnet fallback; startup must still fail closed if it is edited. */ +if (scipDotnet === undefined) { + throw new Error("roslyn-workspace: the scip-dotnet fallback is not registered"); +} + +/** Compiler-owned C# graph over one resident immutable Roslyn Solution. */ +export const csharpGraphProvider: IGraphProvider = { + name: CSHARP_ROSLYN_PROVIDER, + languages: ["csharp"], + authority: "compiler", + facts: CSHARP_ROSLYN_FACTS, + resolution: { + commands: [CSHARP_ROSLYN_PRODUCER, "dotnet"], + environmentOverrides: [OVERRIDE, DOTNET_OVERRIDE], + }, + fallbacks: [scipDotnet], + buildInputs: scipDotnet.buildInputs, + configuration: (_root, env) => [ + "producer-schema=1", + `${OVERRIDE}=${env[OVERRIDE] ?? "unconfigured"}`, + `${DOTNET_OVERRIDE}=${env[DOTNET_OVERRIDE] ?? "unconfigured"}`, + ], + refuse: (options) => { + const refused = [ + options.server === undefined ? undefined : "server", + options.maxFiles === undefined ? undefined : "maxFiles", + options.lspReferenceLimit === undefined + ? undefined + : "lspReferenceLimit", + ].filter((value): value is string => value !== undefined); + return refused.length === 0 + ? undefined + : `csharp: ${CSHARP_ROSLYN_PROVIDER} publishes whole-solution generations and cannot honor ${refused.join(", ")}`; + }, + resolve: (root, env) => resolveCsharpGraphCommand(root, env), + open: (props) => + new CsharpGraphClient({ + root: props.root, + command: props.command.command, + args: props.command.args, + initializationOptions: props.options.initializationOptions, + requestTimeoutMs: props.options.lspTimeoutMs, + readyTimeoutMs: props.options.lspReadyTimeoutMs, + maxMessageBytes: props.options.lspMaxMessageBytes, + windowsVerbatimArguments: props.command.windowsVerbatimArguments, + validate: (snapshot) => + assertGraphSnapshotContract( + snapshot, + csharpGraphProvider, + props.languages, + props.root, + ), + }), +}; + +function resolveCsharpGraphCommand( + root: string, + env: NodeJS.ProcessEnv, +): IGraphProvider.ICommand | undefined { + if (!hasEntryPoint(root)) return undefined; + const installed = resolveProviderCommand(root, env, { + command: CSHARP_ROSLYN_PRODUCER, + override: OVERRIDE, + }); + if (installed !== undefined) return installed; + const project = path.resolve( + __dirname, + "..", + "..", + "..", + "sidecars", + "csharp", + "Samchon.Graph.CSharp.csproj", + ); + /* c8 ignore start -- the package build copies this enumerated source file; + * absence can only mean the installed package itself is corrupt. */ + if (!fs.existsSync(project)) return undefined; + /* c8 ignore stop */ + const dotnetAttempt = resolveProviderCommand.attempt(root, env, { + command: "dotnet", + override: DOTNET_OVERRIDE, + }); + const dotnet = dotnetAttempt.command; + return dotnet === undefined + ? undefined + : spawnableCommand.append( + { ...dotnet, args: [...dotnet.args] }, + [ + "run", + "--project", + project, + "--configuration", + "Release", + "--verbosity", + "quiet", + "--no-launch-profile", + "--", + "--dotnet-host", + dotnetAttempt.executable!, + ], + ); +} + +function hasEntryPoint(root: string): boolean { + const entries = fs.readdirSync(root, { withFileTypes: true }); + if ( + entries.some( + (entry) => + entry.isFile() && + [".sln", ".slnx", ".csproj"].includes( + path.extname(entry.name).toLowerCase(), + ), + ) + ) { + return true; + } + return entries + .filter( + (entry) => + entry.isDirectory() && + ![".git", ".wiki", "bin", "node_modules", "obj"].includes( + entry.name, + ), + ) + .some((entry) => hasProject(path.join(root, entry.name))); +} + +function hasProject(directory: string): boolean { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + if (entry.isFile() && path.extname(entry.name).toLowerCase() === ".csproj") { + return true; + } + if ( + entry.isDirectory() && + ![".git", ".wiki", "bin", "node_modules", "obj"].includes(entry.name) && + hasProject(path.join(directory, entry.name)) + ) { + return true; + } + } + return false; +} diff --git a/packages/graph/src/provider/csharp/index.ts b/packages/graph/src/provider/csharp/index.ts new file mode 100644 index 00000000..60d50c65 --- /dev/null +++ b/packages/graph/src/provider/csharp/index.ts @@ -0,0 +1,6 @@ +export * from "./CSHARP_ROSLYN_FACTS"; +export * from "./CSHARP_ROSLYN_PRODUCER"; +export * from "./CSHARP_ROSLYN_PROVIDER"; +export * from "./CsharpGraphClient"; +export * from "./csharpGraphProvider"; +export * from "./ICsharpGraphSnapshot"; diff --git a/packages/graph/src/provider/index.ts b/packages/graph/src/provider/index.ts index 3ccb5f50..27643560 100644 --- a/packages/graph/src/provider/index.ts +++ b/packages/graph/src/provider/index.ts @@ -7,16 +7,20 @@ export * from "./fallbackCoverage"; export * from "./graphCoverageOf"; export * from "./graphUnresolvedOf"; export * from "./GraphSnapshotProtocol"; +export * from "./csharp"; export * from "./cpp"; export * from "./go"; export * from "./IBulkGraphSession"; export * from "./IGraphProvider"; export * from "./java"; +export * from "./kotlin"; export * from "./lua"; export * from "./providerInputFiles"; export * from "./resolveProviderCommand"; export * from "./rust"; +export * from "./scala"; export * from "./scip"; export * from "./sidecar"; +export * from "./swift"; export * from "./selectGraphProviders"; export * from "./semanticIdentity"; diff --git a/packages/graph/src/provider/java/IJdtGraphSnapshot.ts b/packages/graph/src/provider/java/IJdtGraphSnapshot.ts new file mode 100644 index 00000000..74ac0a5a --- /dev/null +++ b/packages/graph/src/provider/java/IJdtGraphSnapshot.ts @@ -0,0 +1,100 @@ +/** Raw java.graph.snapshot response emitted by the pinned JDT workspace producer. */ +export interface IJdtGraphSnapshot { + schemaVersion: number; + protocolVersion: number; + producer: IJdtGraphSnapshot.IProducer; + capabilities: IJdtGraphSnapshot.ICapabilities; + universe: string; + generation: string; + complete: boolean; + mode: IJdtGraphSnapshot.Mode; + sequence: number; + projects: IJdtGraphSnapshot.IProject[]; + sources: IJdtGraphSnapshot.ISource[]; + nodes: IJdtGraphSnapshot.INode[]; + edges: IJdtGraphSnapshot.IEdge[]; + diagnostics: IJdtGraphSnapshot.IDiagnostic[]; + coverage: Record; + unresolved: unknown[]; +} + +export namespace IJdtGraphSnapshot { + export interface IProducer { + name: string; + version: string; + compilerVersion: string; + } + + export interface ICapabilities { + atomicGenerations: boolean; + resident: boolean; + sourceDigests: boolean; + diskDigests: boolean; + unsavedBuffers: boolean; + diagnostics: boolean; + facts: string[]; + } + + export interface IProject { + name: string; + location: string; + output: string; + compilerVersion: string; + options: Record; + classpath: unknown[]; + } + + export interface ISource { + project: string; + uri: string; + checkerDigest: string; + checkerEncoding: string; + diskDigest: string; + } + + export interface INode { + project: string; + symbol: string; + nativeKey: string; + stability: "persistent" | "structural" | "generation"; + uri: string; + name: string; + qualifiedName: string; + kind: string; + signature: string; + declarationKind: string; + exported: boolean; + modifiers: string[]; + evidence: IEvidence; + } + + export interface IEdge { + from: string; + to: string; + kind: string; + evidence: IEvidence; + } + + export interface IDiagnostic { + uri: string; + severity: "error" | "warning" | "information"; + code: string; + message: string; + evidence: IEvidence; + } + + export interface IEvidence { + uri: string; + startLine: number; + startColumn: number; + endLine: number; + endColumn: number; + } + + export type Mode = "initial" | "reload" | "unchanged" | "incremental" | "error"; + + export const SCHEMA_VERSION = 1; + export const PROTOCOL_VERSION = 1; + export const PRODUCER = "eclipse-jdtls-graph-snapshot"; + export const CHECKER_ENCODING = "jdt-utf16-code-units-v1"; +} diff --git a/packages/graph/src/provider/java/JDT_GRAPH_FACTS.ts b/packages/graph/src/provider/java/JDT_GRAPH_FACTS.ts new file mode 100644 index 00000000..92203520 --- /dev/null +++ b/packages/graph/src/provider/java/JDT_GRAPH_FACTS.ts @@ -0,0 +1,4 @@ +import { GraphEdgeKind } from "../../typings"; + +/** Relationship families the JDT workspace snapshot currently proves. */ +export const JDT_GRAPH_FACTS: readonly GraphEdgeKind[] = ["contains"]; diff --git a/packages/graph/src/provider/java/JDT_GRAPH_PRODUCER_COMMIT.ts b/packages/graph/src/provider/java/JDT_GRAPH_PRODUCER_COMMIT.ts new file mode 100644 index 00000000..bcd154d6 --- /dev/null +++ b/packages/graph/src/provider/java/JDT_GRAPH_PRODUCER_COMMIT.ts @@ -0,0 +1,3 @@ +/** Exact Eclipse JDT.LS fork revision implementing java.graph.snapshot. */ +export const JDT_GRAPH_PRODUCER_COMMIT = + "0d55a6c13d14e0d0466eeb021920349b3d0c6d35"; diff --git a/packages/graph/src/provider/java/JDT_GRAPH_PROVIDER.ts b/packages/graph/src/provider/java/JDT_GRAPH_PROVIDER.ts new file mode 100644 index 00000000..a6b1ff41 --- /dev/null +++ b/packages/graph/src/provider/java/JDT_GRAPH_PROVIDER.ts @@ -0,0 +1,2 @@ +/** Registry identity of the resident JDT workspace graph route. */ +export const JDT_GRAPH_PROVIDER = "jdt-workspace"; diff --git a/packages/graph/src/provider/java/JavaGraphSnapshotAdapter.ts b/packages/graph/src/provider/java/JavaGraphSnapshotAdapter.ts index 9eec7231..812e20af 100644 --- a/packages/graph/src/provider/java/JavaGraphSnapshotAdapter.ts +++ b/packages/graph/src/provider/java/JavaGraphSnapshotAdapter.ts @@ -21,6 +21,7 @@ import { IJavaGraphSnapshot } from "./IJavaGraphSnapshot"; import { JAVA_GRAPH_FACTS } from "./JAVA_GRAPH_FACTS"; import { JAVA_GRAPH_PRODUCER } from "./JAVA_GRAPH_PRODUCER"; import { JAVA_GRAPH_PROVIDER } from "./JAVA_GRAPH_PROVIDER"; +import { javaDeclarationSymbol } from "./javaDeclarationSymbol"; const SHA256 = /^[0-9a-f]{64}$/u; const NODE_KINDS = new Set([ @@ -689,6 +690,13 @@ function adaptNode( node.qualifiedName === "" ? undefined : declaredName(node.qualifiedName); const qualifiedName = qualified; const display = qualifiedName ?? name; + const symbol = javaDeclarationSymbol({ + kind: node.kind as GraphNodeKind, + name, + ...(qualifiedName === undefined ? {} : { qualifiedName }), + ...(node.signature === "" ? {} : { signature: node.signature }), + displayName: node.name, + }); // The parameter list the producer displays is not lost, only moved. A // producer-supplied signature is the better statement of it and wins; where // there is none, the display it came from becomes the signature so a reader @@ -704,9 +712,9 @@ function adaptNode( { version: 2, language: "java", - symbol: node.symbol, + symbol, role: node.kind as GraphNodeKind, - native: { key: node.symbol, stability: "semantic" }, + native: { key: symbol, stability: "semantic" }, scope: { target: target.name }, stability: "persistent", }, diff --git a/packages/graph/src/provider/java/JdtGraphClient.ts b/packages/graph/src/provider/java/JdtGraphClient.ts new file mode 100644 index 00000000..011952a2 --- /dev/null +++ b/packages/graph/src/provider/java/JdtGraphClient.ts @@ -0,0 +1,276 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { LspClient } from "../../lsp/LspClient"; +import { GraphLanguage } from "../../typings"; +import { providerInputFiles } from "../providerInputFiles"; +import { IBulkGraphSession } from "../IBulkGraphSession"; +import { IJdtGraphSnapshot } from "./IJdtGraphSnapshot"; +import { JdtGraphSnapshotAdapter } from "./JdtGraphSnapshotAdapter"; + +const COMMAND = "java.graph.snapshot"; + +/** Resident JDT client that receives one whole workspace snapshot per refresh. */ +export class JdtGraphClient implements IBulkGraphSession { + public readonly kind = "bulk" as const; + public readonly languages: readonly GraphLanguage[] = ["java"]; + public readonly root: string; + + private readonly lsp: LspClient; + private readonly adapter: JdtGraphSnapshotAdapter; + private readonly validate: (snapshot: IBulkGraphSession.ISnapshot) => void; + private readonly initializationOptions: unknown; + private readonly requestTimeoutMs: number | undefined; + private readonly lifecycleAbort = new AbortController(); + private initialized: Promise | undefined; + private watchedInputs = new Map(); + private queue: Promise = Promise.resolve(); + private version = 0; + private closed = false; + private closing: Promise | undefined; + + public constructor(options: JdtGraphClient.IOptions) { + this.root = options.root; + this.adapter = new JdtGraphSnapshotAdapter(options.root); + this.validate = options.validate; + this.initializationOptions = options.initializationOptions; + this.requestTimeoutMs = options.requestTimeoutMs; + this.lsp = new LspClient( + options.command, + options.args, + options.requestTimeoutMs, + options.root, + options.maxMessageBytes, + options.windowsVerbatimArguments, + ); + } + + public get generation(): number { + return this.version; + } + + public get current(): IBulkGraphSession.ISnapshot | undefined { + return this.adapter.current; + } + + public refresh( + options: { signal?: AbortSignal } = {}, + ): Promise { + if (this.closed) { + return Promise.reject(new Error("JDT workspace graph: session is closed")); + } + return this.enqueue(async () => { + const signal = combineSignals(options.signal, this.lifecycleAbort.signal); + await this.initialize(signal); + const inputs = this.notifyInputChanges(); + const raw = await this.lsp.request( + "workspace/executeCommand", + { command: COMMAND, arguments: [] }, + this.requestTimeoutMs, + signal, + ); + const result = this.adapter.apply(raw, { + signal, + validate: this.validate, + }); + if (inputs.moved && !result.changed) { + throw new Error( + "JDT workspace graph: watched Java inputs moved but the producer reused its generation", + ); + } + this.watchedInputs = inputs.current; + if (result.changed) this.version += 1; + return { + changed: result.changed, + generation: this.version, + mode: result.mode, + snapshot: result.snapshot, + }; + }, options.signal); + } + + public close(): Promise { + if (this.closing !== undefined) return this.closing; + this.closed = true; + this.lifecycleAbort.abort( + new Error("JDT workspace graph: session is closed"), + ); + this.closing = this.lsp.close(); + return this.closing; + } + + private initialize(signal: AbortSignal): Promise { + this.initialized ??= this.initializeOnce(this.lifecycleAbort.signal); + return signal === this.lifecycleAbort.signal + ? this.initialized + : raceWithAbort(this.initialized, signal); + } + + private async initializeOnce(signal: AbortSignal): Promise { + await this.lsp.request( + "initialize", + { + processId: process.pid, + rootUri: pathToFileURL(this.root).href, + capabilities: { + window: { workDoneProgress: true }, + workspace: { configuration: true }, + }, + ...(this.initializationOptions === undefined + ? {} + : { initializationOptions: this.initializationOptions }), + workspaceFolders: [ + { + uri: pathToFileURL(this.root).href, + name: "samchon-graph-java", + }, + ], + }, + this.requestTimeoutMs, + signal, + ); + this.lsp.notify("initialized", {}); + } + + private notifyInputChanges(): { + current: Map; + moved: boolean; + } { + const current = javaInputDigests(this.root); + const files = new Set([...this.watchedInputs.keys(), ...current.keys()]); + const changes: Array<{ uri: string; type: 1 | 2 | 3 }> = []; + for (const file of [...files].sort(compareText)) { + const before = this.watchedInputs.get(file); + const after = current.get(file); + if (before === after) continue; + changes.push({ + uri: pathToFileURL(file).href, + type: before === undefined ? 1 : after === undefined ? 3 : 2, + }); + } + if (changes.length !== 0) { + this.lsp.notify("workspace/didChangeWatchedFiles", { changes }); + } + return { current, moved: changes.length !== 0 }; + } + + private enqueue(task: () => Promise, signal?: AbortSignal): Promise { + let resolveResult!: (value: T) => void; + let rejectResult!: (error: Error) => void; + let started = false; + let settled = false; + const result = new Promise((resolve, reject) => { + resolveResult = (value) => { + settled = true; + resolve(value); + }; + rejectResult = (error) => { + settled = true; + reject(error); + }; + }); + const cancelQueued = (): void => { + if (!started) rejectResult(abortError(signal!)); + }; + if (signal?.aborted) { + rejectResult(abortError(signal)); + return result; + } + signal?.addEventListener("abort", cancelQueued, { once: true }); + this.queue = this.queue + .catch(() => undefined) + .then(async () => { + started = true; + signal?.removeEventListener("abort", cancelQueued); + if (settled) return; + try { + resolveResult(await task()); + } catch (error) { + rejectResult(asError(error)); + } + }); + return result; + } +} + +export namespace JdtGraphClient { + export interface IOptions { + root: string; + command: string; + args: readonly string[]; + initializationOptions?: unknown; + requestTimeoutMs?: number; + maxMessageBytes?: number; + windowsVerbatimArguments?: boolean; + validate: (snapshot: IBulkGraphSession.ISnapshot) => void; + } +} + +function javaInputDigests(root: string): Map { + const answer = new Map(); + for (const relative of providerInputFiles(root, ["java"], [])) { + const file = path.resolve(root, relative); + try { + answer.set( + file, + createHash("sha256").update(fs.readFileSync(file)).digest("hex"), + ); + /* c8 ignore start -- a source disappearing between the directory walk and + * this read is a benign filesystem race; the next refresh reports it. */ + } catch { + continue; + } + /* c8 ignore stop */ + } + return answer; +} + +function combineSignals( + request: AbortSignal | undefined, + lifecycle: AbortSignal, +): AbortSignal { + return request === undefined + ? lifecycle + : AbortSignal.any([request, lifecycle]); +} + +function raceWithAbort( + task: Promise, + signal: AbortSignal, +): Promise { + /* c8 ignore start -- enqueue rejects a pre-aborted caller before this + * initialization boundary; this closes only the instruction-boundary race. */ + if (signal.aborted) return Promise.reject(abortError(signal)); + /* c8 ignore stop */ + return new Promise((resolve, reject) => { + const abort = (): void => { + signal.removeEventListener("abort", abort); + reject(abortError(signal)); + }; + signal.addEventListener("abort", abort, { once: true }); + void task + .then((value) => { + signal.removeEventListener("abort", abort); + resolve(value); + }) + .catch((error: unknown) => { + signal.removeEventListener("abort", abort); + reject(error); + }); + }); +} + +function abortError(signal: AbortSignal): Error { + return signal.reason as Error; +} + +function asError(value: unknown): Error { + return value instanceof Error ? value : new Error(String(value)); +} + +function compareText(left: string, right: string): number { + // The set contains distinct paths, so equality is not a reachable arm. + return left < right ? -1 : 1; +} diff --git a/packages/graph/src/provider/java/JdtGraphSnapshotAdapter.ts b/packages/graph/src/provider/java/JdtGraphSnapshotAdapter.ts new file mode 100644 index 00000000..cd78845a --- /dev/null +++ b/packages/graph/src/provider/java/JdtGraphSnapshotAdapter.ts @@ -0,0 +1,545 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { + ISamchonGraphCoverage, + ISamchonGraphDiagnostic, + ISamchonGraphEdge, + ISamchonGraphEvidence, + ISamchonGraphNode, + SamchonGraphNodeModifier, +} from "../../structures"; +import { GRAPH_EDGE_KINDS, GraphNodeKind } from "../../typings"; +import { fileFromUri } from "../../utils/fileFromUri"; +import { isSubPath } from "../../utils/isSubPath"; +import { projectRelative } from "../../utils/projectRelative"; +import { GraphSnapshotProtocol } from "../GraphSnapshotProtocol"; +import { IBulkGraphSession } from "../IBulkGraphSession"; +import { semanticGraphNodeId } from "../semanticIdentity"; +import { IJdtGraphSnapshot } from "./IJdtGraphSnapshot"; +import { JDT_GRAPH_FACTS } from "./JDT_GRAPH_FACTS"; +import { JDT_GRAPH_PROVIDER } from "./JDT_GRAPH_PROVIDER"; +import { javaDeclarationSymbol } from "./javaDeclarationSymbol"; + +const SHA256 = /^[0-9a-f]{64}$/u; +const NODE_KINDS = new Set([ + "file", + "package", + "module", + "function", + "class", + "interface", + "type", + "enum", + "variable", + "method", + "parameter", + "field", + "constructor", +]); +const RAW_MODIFIERS = new Set([ + "public", + "protected", + "private", + "static", + "abstract", + "final", +]); +const CAPABILITIES = [ + "coverage", + "diagnostics", + "diskDigests", + "incremental", + "sourceDigests", + "universe", + "unsavedBuffers", +]; +const TARGET = "jdt-workspace"; + +/** Validates and publishes one frozen JDT workspace generation. */ +export class JdtGraphSnapshotAdapter { + public readonly store: GraphSnapshotProtocol.Store; + + private sequence = 0; + + public constructor(private readonly root: string) { + this.store = new GraphSnapshotProtocol.Store(root); + } + + public get current(): IBulkGraphSession.ISnapshot | undefined { + return this.store.current; + } + + public apply( + value: unknown, + options: { + signal?: AbortSignal; + validate?: (snapshot: IBulkGraphSession.ISnapshot) => void; + } = {}, + ): { + changed: boolean; + mode: IBulkGraphSession.Mode; + snapshot: IBulkGraphSession.ISnapshot; + } { + const raw = assertSnapshot(value, this.root); + if (!raw.complete || raw.mode === "error") { + const errors = raw.diagnostics.filter( + (diagnostic) => diagnostic.severity === "error", + ); + const summary = errors + .slice(0, 3) + .map((diagnostic) => diagnostic.message) + .join("; "); + throw new Error( + `JDT workspace graph: producer retained the prior strict generation after ${String(errors.length)} error(s): ${summary}`, + ); + } + const prior = this.store.current; + if (prior?.protocol?.generation === raw.generation) { + return { changed: false, mode: "unchanged", snapshot: prior }; + } + + const shard = adaptShard(this.root, raw); + const digest = GraphSnapshotProtocol.shardDigest(shard); + const manifest = [{ key: shard.key, digest }]; + const hello = helloOf(raw); + const sequence = this.sequence + 1; + const begin: GraphSnapshotProtocol.IBegin = { + type: "begin", + sequence, + generation: raw.generation, + universe: raw.universe, + manifest: GraphSnapshotProtocol.manifestDigest(shard.sources), + targets: [TARGET], + }; + const body = assembled(hello, begin, shard); + const frames: GraphSnapshotProtocol.Frame[] = [ + hello, + begin, + { type: "upsertShard", digest, shard }, + { + type: "commit", + sequence: begin.sequence, + generation: begin.generation, + shards: manifest, + factDigest: GraphSnapshotProtocol.factDigest(body), + }, + ]; + const snapshot = this.store.apply(frames, { + ...(options.signal === undefined ? {} : { signal: options.signal }), + ...(options.validate === undefined + ? {} + : { validate: options.validate }), + }); + this.sequence = sequence; + return { + changed: true, + mode: + prior === undefined + ? "initial" + : prior.provenance.universe === raw.universe + ? "incremental" + : "reload", + snapshot, + }; + } +} + +function adaptShard( + root: string, + raw: IJdtGraphSnapshot, +): GraphSnapshotProtocol.IShard { + const ids = new Map(); + const nodes = raw.nodes.map((node) => { + const adapted = adaptNode(root, raw, node); + ids.set(node.symbol, adapted.id); + return adapted; + }); + const edges: ISamchonGraphEdge[] = raw.edges.map((edge) => ({ + from: ids.get(edge.from)!, + to: ids.get(edge.to)!, + kind: "contains", + evidence: adaptEvidence(root, edge.evidence), + })); + const diagnostics: ISamchonGraphDiagnostic[] = raw.diagnostics.map( + (diagnostic) => ({ + file: graphFile(root, diagnostic.uri), + line: diagnostic.evidence.startLine, + column: diagnostic.evidence.startColumn, + code: diagnostic.code, + message: diagnostic.message, + severity: + diagnostic.severity === "information" + ? "info" + : diagnostic.severity, + }), + ); + const coverage: ISamchonGraphCoverage[] = GRAPH_EDGE_KINDS.map((family) => ({ + provider: JDT_GRAPH_PROVIDER, + language: "java" as const, + target: TARGET, + family, + state: family === "contains" ? "complete" : "unsupported", + })); + return { + key: `jdt-workspace:${raw.universe}`, + target: TARGET, + languages: ["java"], + nodes, + edges, + diagnostics, + coverage, + unresolved: [], + sources: [ + ...raw.sources.map((source) => ({ + file: sourceFile(root, source.uri), + checkerDigest: source.checkerDigest, + diskDigest: source.diskDigest, + })), + { + file: `bundled:///java/jdt-workspace/${digest(raw.projects)}`, + checkerDigest: raw.universe, + diskDigest: "", + }, + ], + }; +} + +function adaptNode( + root: string, + raw: IJdtGraphSnapshot, + node: IJdtGraphSnapshot.INode, +): ISamchonGraphNode { + const kind = node.kind as GraphNodeKind; + const qualifiedName = node.qualifiedName === "" ? undefined : node.qualifiedName; + const display = qualifiedName ?? node.name; + const generationScoped = node.stability === "generation"; + const symbol = + node.stability === "persistent" + ? javaDeclarationSymbol({ + kind, + name: node.name, + ...(qualifiedName === undefined ? {} : { qualifiedName }), + ...(node.signature === "" ? {} : { signature: node.signature }), + }) + : node.symbol; + const modifiers = node.modifiers.map((modifier) => + modifier === "final" ? "readonly" : modifier, + ) as SamchonGraphNodeModifier[]; + return { + id: semanticGraphNodeId( + { + version: 2, + language: "java", + symbol, + role: kind, + native: { + key: symbol, + stability: generationScoped ? "positional" : "semantic", + }, + scope: { target: targetOf(root, raw, node.project, node.uri) }, + stability: generationScoped ? "generation" : "persistent", + ...(generationScoped ? { generation: raw.generation } : {}), + }, + display, + ), + kind, + language: "java", + name: node.name, + ...(qualifiedName === undefined ? {} : { qualifiedName }), + file: graphFile(root, node.uri), + external: false, + ...(node.exported ? { exported: true } : {}), + ...(node.stability === "structural" ? { closure: true } : {}), + ...(modifiers.length === 0 ? {} : { modifiers }), + ...(node.signature === "" ? {} : { signature: node.signature }), + evidence: adaptEvidence(root, node.evidence), + }; +} + +function adaptEvidence( + root: string, + evidence: IJdtGraphSnapshot.IEvidence, +): ISamchonGraphEvidence { + return { + file: graphFile(root, evidence.uri), + startLine: evidence.startLine, + startCol: evidence.startColumn, + endLine: evidence.endLine, + endCol: evidence.endColumn, + }; +} + +function graphFile(root: string, uri: string): string { + return projectRelative(root, sourceFile(root, uri)); +} + +function sourceFile(root: string, uri: string): string { + const file = path.normalize(fileFromUri(uri)); + if (!isSubPath(root, file)) { + throw new Error(`JDT workspace graph: source escaped the project root: ${uri}`); + } + return file; +} + +function targetOf( + root: string, + raw: IJdtGraphSnapshot, + projectName: string, + sourceUri: string, +): string { + const project = raw.projects.find((candidate) => candidate.name === projectName)!; + const directory = sourceFile(root, project.location); + if (fs.existsSync(path.join(directory, "pom.xml"))) { + const relative = projectRelative(root, directory); + return `maven:${relative === "" ? "." : relative}`; + } + const gradleRoot = gradleRootOf(root, directory); + const task = gradleJavaTask(directory, sourceFile(root, sourceUri)); + if (gradleRoot !== undefined && task !== undefined) { + const relative = projectRelative(gradleRoot, directory); + const projectPath = + relative === "" ? "" : `:${relative.split("/").join(":")}`; + return `${projectPath}:${task}`; + } + return `jdt:${projectName}`; +} + +function gradleRootOf(root: string, project: string): string | undefined { + for (let cursor = project; isSubPath(root, cursor); cursor = path.dirname(cursor)) { + if ( + fs.existsSync(path.join(cursor, "settings.gradle")) || + fs.existsSync(path.join(cursor, "settings.gradle.kts")) + ) { + return cursor; + } + if (cursor === root || path.dirname(cursor) === cursor) break; + } + return fs.existsSync(path.join(project, "build.gradle")) || + fs.existsSync(path.join(project, "build.gradle.kts")) + ? project + : undefined; +} + +function gradleJavaTask( + project: string, + source: string, +): "compileJava" | "compileTestJava" | undefined { + const relative = projectRelative(project, source); + if (relative.startsWith("src/main/java/")) return "compileJava"; + if (relative.startsWith("src/test/java/")) return "compileTestJava"; + return undefined; +} + +function helloOf( + raw: IJdtGraphSnapshot, +): GraphSnapshotProtocol.IHello { + return { + type: "hello", + protocolVersion: 1, + schemaVersion: 1, + producerSchemaVersion: raw.schemaVersion, + provider: JDT_GRAPH_PROVIDER, + producer: raw.producer.name, + producerVersion: raw.producer.version, + compilerVersion: raw.producer.compilerVersion, + languages: ["java"], + authority: "compiler", + supportedFacts: [...JDT_GRAPH_FACTS], + capabilities: [...CAPABILITIES], + }; +} + +function assembled( + hello: GraphSnapshotProtocol.IHello, + begin: GraphSnapshotProtocol.IBegin, + shard: GraphSnapshotProtocol.IShard, +): Parameters[0] { + return { + languages: [...hello.languages], + nodes: [...shard.nodes], + edges: [...shard.edges], + diagnostics: [...shard.diagnostics], + coverage: [...shard.coverage], + unresolved: [...shard.unresolved], + provenance: { + provider: hello.provider, + authority: hello.authority, + facts: [...hello.supportedFacts], + schemaVersion: hello.producerSchemaVersion, + tool: hello.producer, + toolVersion: hello.producerVersion, + compilerVersion: hello.compilerVersion, + protocolVersion: hello.protocolVersion, + universe: begin.universe, + capabilities: [...hello.capabilities], + }, + }; +} + +function assertSnapshot(value: unknown, root: string): IJdtGraphSnapshot { + if (!isRecord(value)) { + throw new Error("JDT workspace graph: snapshot is not an object"); + } + const raw = value as unknown as IJdtGraphSnapshot; + if ( + raw.schemaVersion !== IJdtGraphSnapshot.SCHEMA_VERSION || + raw.protocolVersion !== IJdtGraphSnapshot.PROTOCOL_VERSION || + raw.producer?.name !== IJdtGraphSnapshot.PRODUCER || + !nonempty(raw.producer?.version) || + !nonempty(raw.producer?.compilerVersion) || + !SHA256.test(raw.universe) || + !SHA256.test(raw.generation) || + typeof raw.complete !== "boolean" || + !["initial", "reload", "unchanged", "incremental", "error"].includes( + raw.mode, + ) || + !Number.isSafeInteger(raw.sequence) || + raw.sequence < 0 || + !Array.isArray(raw.projects) || + raw.projects.length === 0 || + !Array.isArray(raw.sources) || + !Array.isArray(raw.nodes) || + !Array.isArray(raw.edges) || + !Array.isArray(raw.diagnostics) || + !Array.isArray(raw.unresolved) || + raw.unresolved.length !== 0 || + !isRecord(raw.coverage) || + raw.coverage["contains"] !== "complete" + ) { + throw new Error("JDT workspace graph: malformed producer snapshot"); + } + assertCapabilities(raw.capabilities); + const projects = new Set(); + for (const project of raw.projects) { + if ( + !isRecord(project) || + !nonempty(project.name) || + projects.has(project.name) || + !nonempty(project.location) || + !nonempty(project.output) || + !nonempty(project.compilerVersion) || + !isRecord(project.options) || + !Array.isArray(project.classpath) + ) { + throw new Error("JDT workspace graph: malformed project universe"); + } + projects.add(project.name); + sourceFile(root, project.location); + } + const sourceUris = new Set(); + for (const source of raw.sources) { + if ( + !isRecord(source) || + !projects.has(source.project) || + !nonempty(source.uri) || + source.checkerEncoding !== IJdtGraphSnapshot.CHECKER_ENCODING || + !SHA256.test(source.checkerDigest) || + (source.diskDigest !== "" && !SHA256.test(source.diskDigest)) + ) { + throw new Error("JDT workspace graph: malformed source manifest"); + } + graphFile(root, source.uri); + sourceUris.add(source.uri); + } + const symbols = new Set(); + for (const node of raw.nodes) { + if ( + !isRecord(node) || + !projects.has(node.project) || + !nonempty(node.symbol) || + symbols.has(node.symbol) || + !nonempty(node.nativeKey) || + !["persistent", "structural", "generation"].includes(node.stability) || + !sourceUris.has(node.uri) || + !nonempty(node.name) || + typeof node.qualifiedName !== "string" || + !NODE_KINDS.has(node.kind as GraphNodeKind) || + typeof node.signature !== "string" || + !nonempty(node.declarationKind) || + typeof node.exported !== "boolean" || + !Array.isArray(node.modifiers) || + node.modifiers.some( + (modifier) => typeof modifier !== "string" || !RAW_MODIFIERS.has(modifier), + ) || + !validEvidence(node.evidence, sourceUris) + ) { + throw new Error("JDT workspace graph: malformed declaration"); + } + symbols.add(node.symbol); + } + for (const edge of raw.edges) { + if ( + !isRecord(edge) || + edge.kind !== "contains" || + !symbols.has(edge.from) || + !symbols.has(edge.to) || + !validEvidence(edge.evidence, sourceUris) + ) { + throw new Error("JDT workspace graph: malformed containment edge"); + } + } + for (const diagnostic of raw.diagnostics) { + if ( + !isRecord(diagnostic) || + !sourceUris.has(diagnostic.uri) || + !["error", "warning", "information"].includes(diagnostic.severity) || + !nonempty(diagnostic.code) || + !nonempty(diagnostic.message) || + !validEvidence(diagnostic.evidence, sourceUris) + ) { + throw new Error("JDT workspace graph: malformed diagnostic"); + } + } + if ( + raw.complete === raw.diagnostics.some( + (diagnostic) => diagnostic.severity === "error", + ) || + (raw.complete && raw.sequence < 1) || + (raw.complete && raw.mode === "error") || + (!raw.complete && raw.mode !== "error") + ) { + throw new Error("JDT workspace graph: contradictory completion state"); + } + return raw; +} + +function assertCapabilities(value: unknown): void { + if ( + !isRecord(value) || + value["atomicGenerations"] !== true || + value["resident"] !== true || + value["sourceDigests"] !== true || + value["diskDigests"] !== true || + value["unsavedBuffers"] !== true || + value["diagnostics"] !== true || + !Array.isArray(value["facts"]) || + value["facts"].length !== 1 || + value["facts"][0] !== "contains" + ) { + throw new Error("JDT workspace graph: incompatible capabilities"); + } +} + +function validEvidence( + value: unknown, + sources: ReadonlySet, +): value is IJdtGraphSnapshot.IEvidence { + if (!isRecord(value) || !sources.has(String(value["uri"]))) return false; + return ["startLine", "startColumn", "endLine", "endColumn"].every( + (key) => Number.isSafeInteger(value[key]) && Number(value[key]) >= 1, + ); +} + +function nonempty(value: unknown): value is string { + return typeof value === "string" && value !== "" && !value.includes("\0"); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function digest(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} diff --git a/packages/graph/src/provider/java/index.ts b/packages/graph/src/provider/java/index.ts index 7be107ba..74007c3a 100644 --- a/packages/graph/src/provider/java/index.ts +++ b/packages/graph/src/provider/java/index.ts @@ -1,7 +1,15 @@ export * from "./IJavaGraphSnapshot"; +export * from "./IJdtGraphSnapshot"; +export * from "./JDT_GRAPH_FACTS"; +export * from "./JDT_GRAPH_PRODUCER_COMMIT"; +export * from "./JDT_GRAPH_PROVIDER"; export * from "./JAVA_GRAPH_FACTS"; export * from "./JAVA_GRAPH_PRODUCER"; export * from "./JAVA_GRAPH_PROVIDER"; export * from "./JavaGraphSession"; export * from "./JavaGraphSnapshotAdapter"; +export * from "./JdtGraphClient"; +export * from "./JdtGraphSnapshotAdapter"; export * from "./javaGraphProvider"; +export * from "./jdtGraphProvider"; +export * from "./javaDeclarationSymbol"; diff --git a/packages/graph/src/provider/java/javaDeclarationSymbol.ts b/packages/graph/src/provider/java/javaDeclarationSymbol.ts new file mode 100644 index 00000000..6e4eb0ba --- /dev/null +++ b/packages/graph/src/provider/java/javaDeclarationSymbol.ts @@ -0,0 +1,35 @@ +import { GraphNodeKind } from "../../typings"; + +/** Canonical Java declaration key shared by javac and JDT producer lanes. */ +export function javaDeclarationSymbol(props: { + kind: GraphNodeKind; + name: string; + qualifiedName?: string; + signature?: string; + displayName?: string; +}): string { + const qualified = callableBase(props.qualifiedName ?? props.name); + const parameters = callableKinds.has(props.kind) + ? parameterList(props.displayName) ?? parameterList(props.signature) ?? "" + : ""; + return `java-declaration-v1|${props.kind}|${qualified}|${parameters.replace(/\s+/gu, "")}`; +} + +function parameterList(value: string | undefined): string | undefined { + if (value === undefined) return undefined; + const open = value.indexOf("("); + if (open < 0) return undefined; + const close = value.indexOf(")", open + 1); + return close < 0 ? undefined : value.slice(open + 1, close); +} + +function callableBase(value: string): string { + const open = value.indexOf("("); + return open < 0 ? value : value.slice(0, open); +} + +const callableKinds = new Set([ + "function", + "method", + "constructor", +]); diff --git a/packages/graph/src/provider/java/javaGraphProvider.ts b/packages/graph/src/provider/java/javaGraphProvider.ts index 30fdf89e..13742157 100644 --- a/packages/graph/src/provider/java/javaGraphProvider.ts +++ b/packages/graph/src/provider/java/javaGraphProvider.ts @@ -10,6 +10,7 @@ import { toolchainVersion } from "../toolchainVersion"; import { JAVA_GRAPH_FACTS } from "./JAVA_GRAPH_FACTS"; import { JAVA_GRAPH_PROVIDER } from "./JAVA_GRAPH_PROVIDER"; import { JavaGraphSession } from "./JavaGraphSession"; +import { jdtGraphProvider } from "./jdtGraphProvider"; const OVERRIDE = "SAMCHON_GRAPH_JAVAC_GRAPH"; const TOOLCHAIN_OVERRIDE = "SAMCHON_GRAPH_JAVA_TOOLCHAIN"; @@ -81,7 +82,7 @@ export const javaGraphProvider: IGraphProvider = { commands: ["scip-java", "java"], environmentOverrides: [OVERRIDE, TOOLCHAIN_OVERRIDE], }, - fallbacks: [javaScipProvider], + fallbacks: [jdtGraphProvider, javaScipProvider], buildInputs: javaScipProvider.buildInputs, configuration: (root, env) => [...javaToolchain(root, env).rows], diff --git a/packages/graph/src/provider/java/jdtGraphProvider.ts b/packages/graph/src/provider/java/jdtGraphProvider.ts new file mode 100644 index 00000000..10cd94dc --- /dev/null +++ b/packages/graph/src/provider/java/jdtGraphProvider.ts @@ -0,0 +1,69 @@ +import { assertGraphSnapshotContract } from "../assertGraphSnapshotContract"; +import { IGraphProvider } from "../IGraphProvider"; +import { resolveProviderCommand } from "../resolveProviderCommand"; +import { standardScipProviders } from "../scip/standardScipProviders"; +import { JdtGraphClient } from "./JdtGraphClient"; +import { JDT_GRAPH_FACTS } from "./JDT_GRAPH_FACTS"; +import { JDT_GRAPH_PRODUCER_COMMIT } from "./JDT_GRAPH_PRODUCER_COMMIT"; +import { JDT_GRAPH_PROVIDER } from "./JDT_GRAPH_PROVIDER"; + +const OVERRIDE = "SAMCHON_GRAPH_JDT_WORKSPACE"; +const javaScipProvider = standardScipProviders.find( + (provider) => provider.name === "scip-java", +); +/* c8 ignore next 4 -- the static standard-provider registry always contains + * the scip-java descriptor; startup must still fail closed if it is edited. */ +if (javaScipProvider === undefined) { + throw new Error("jdt-workspace: the scip-java fallback is not registered"); +} + +/** Resident JDT semantic-owner lane backed by one bulk executeCommand. */ +export const jdtGraphProvider: IGraphProvider = { + name: JDT_GRAPH_PROVIDER, + languages: ["java"], + authority: "compiler", + facts: JDT_GRAPH_FACTS, + resolution: { + commands: ["samchon-jdtls"], + environmentOverrides: [OVERRIDE], + }, + buildInputs: javaScipProvider.buildInputs, + configuration: (_root, env) => [ + `producer-commit=${JDT_GRAPH_PRODUCER_COMMIT}`, + `${OVERRIDE}=${env[OVERRIDE] ?? "unconfigured"}`, + ], + refuse: (options) => { + const refused = [ + options.server === undefined ? undefined : "server", + options.maxFiles === undefined ? undefined : "maxFiles", + options.lspReferenceLimit === undefined + ? undefined + : "lspReferenceLimit", + ].filter((value): value is string => value !== undefined); + return refused.length === 0 + ? undefined + : `java: ${JDT_GRAPH_PROVIDER} publishes one whole resident workspace generation and cannot honor ${refused.join(", ")}`; + }, + resolve: (root, env) => + resolveProviderCommand(root, env, { + command: "samchon-jdtls", + override: OVERRIDE, + }), + open: (props) => + new JdtGraphClient({ + root: props.root, + command: props.command.command, + args: props.command.args, + initializationOptions: props.options.initializationOptions, + requestTimeoutMs: props.options.lspTimeoutMs, + maxMessageBytes: props.options.lspMaxMessageBytes, + windowsVerbatimArguments: props.command.windowsVerbatimArguments, + validate: (snapshot) => + assertGraphSnapshotContract( + snapshot, + jdtGraphProvider, + props.languages, + props.root, + ), + }), +}; diff --git a/packages/graph/src/provider/kotlin/IKotlinGraphSnapshot.ts b/packages/graph/src/provider/kotlin/IKotlinGraphSnapshot.ts new file mode 100644 index 00000000..ecf8ebdd --- /dev/null +++ b/packages/graph/src/provider/kotlin/IKotlinGraphSnapshot.ts @@ -0,0 +1,156 @@ +/** + * The aggregate artifact `scip-java index --kotlin-graph-output` writes. + * + * The producer lives in another repository and ships as a released launcher, + * so this file states what this adapter pins rather than importing a shared + * type. Every field is restated because the pin is the contract: a producer + * that adds a field is compatible, one that changes what a field means is not, + * and only the version numbers below can say which happened. + */ +export interface IKotlinGraphSnapshot { + /** The artifact schema, equal to {@link IKotlinGraphSnapshot.SCHEMA_VERSION}. */ + schemaVersion: number; + + /** Absolute directory the producer indexed. */ + projectRoot: string; + + producer: IKotlinGraphSnapshot.IProducer; + + /** + * One entry per committed build target, sorted by name. + * + * A multi-module Gradle build commits each Kotlin/JVM target separately, and + * they are separate universes: two targets can compile the same source + * against different classpaths and neither reading is wrong. + */ + targets: IKotlinGraphSnapshot.ITarget[]; +} + +export namespace IKotlinGraphSnapshot { + export interface IProducer { + /** Equal to the pinned K2 producer identity; any other build is declined. */ + name: string; + + /** The launcher release that aggregated the generation. */ + version: string; + + /** Equal to {@link IKotlinGraphSnapshot.PROTOCOL_VERSION}. */ + protocolVersion: number; + + capabilities: ICapabilities; + } + + /** + * What the producer states it can do, as booleans rather than a name list. + * + * A missing key is a producer this adapter has not been taught to read, so + * every one is required and no default is assumed. + */ + export interface ICapabilities { + atomicGenerations: boolean; + incremental: boolean; + diagnostics: boolean; + } + + export interface ITarget { + /** Build-target coordinate: a Gradle project, JVM target and compilation. */ + name: string; + + /** SHA-256 the producer committed this target's generation under. */ + generation: string; + + /** SHA-256 of the build universe the target compiled against. */ + universe: string; + + /** One state per relationship family; every family is present. */ + coverage: Record; + + shards: IShard[]; + } + + /** One compilation unit's facts, as the kotlinc plugin wrote them. */ + export interface IShard { + schemaVersion: number; + language: string; + /** Project-relative source path. */ + source: string; + /** SHA-256 of the bytes kotlinc compiled. */ + checkerDigest: string; + /** SHA-256 of the same file on disk. */ + diskDigest: string; + /** The target this unit was compiled for; equal to its owner's name. */ + target: string; + /** The Kotlin compiler's `kotlin.version`. */ + compilerVersion: string; + nodes: INode[]; + edges: IEdge[]; + unresolved: IUnresolved[]; + diagnostics: IDiagnostic[]; + } + + export interface IEvidence { + file: string; + startLine: number; + startColumn: number; + endLine: number; + endColumn: number; + } + + export interface INode { + /** Canonical Kotlin semantic symbol; the endpoint every edge names. */ + symbol: string; + kind: string; + name: string; + /** Owner-qualified name, or `""` for a top-level declaration. */ + qualifiedName: string; + file: string; + exported: boolean; + modifiers: string[]; + /** Structural signature, or `""` when the declaration has none. */ + signature: string; + /** FIR/compiler origin of handwritten, generated, or synthetic declarations. */ + origin: string; + evidence: IEvidence; + } + + export interface IEdge { + from: string; + to: string; + kind: string; + /** `read`/`write` for an access, otherwise null. */ + access: string | null; + /** How the producer settled the endpoint, or null. */ + provenance: string | null; + /** The endpoint's kind when it is outside this compilation unit. */ + targetKind: string | null; + targetName: string | null; + targetQualifiedName: string | null; + evidence: IEvidence; + } + + export interface IUnresolved { + family: string; + reason: string; + evidence: IEvidence; + candidates: string[]; + } + + export interface IDiagnostic { + severity: string; + message: string; + evidence: IEvidence; + } + + /** + * The artifact schema this adapter reads. + * + * Equal to `KotlinGraphShard.SCHEMA_VERSION` and the aggregator's own literal + * in scip-java. Equality is exact: upstream moves this number when a field + * is added, removed or given a new meaning, and validating field types + * cannot detect a semantic change whose JSON shape stayed the same. + */ + export const SCHEMA_VERSION = 1; + + /** The producer protocol version this adapter speaks. */ + export const PROTOCOL_VERSION = 1; +} diff --git a/packages/graph/src/provider/kotlin/KOTLIN_GRAPH_FACTS.ts b/packages/graph/src/provider/kotlin/KOTLIN_GRAPH_FACTS.ts new file mode 100644 index 00000000..7486e4c5 --- /dev/null +++ b/packages/graph/src/provider/kotlin/KOTLIN_GRAPH_FACTS.ts @@ -0,0 +1,5 @@ +import { GRAPH_EDGE_KINDS, GraphEdgeKind } from "../../typings"; + +/** Relationship families the pinned K2 exporter may publish. */ +export const KOTLIN_GRAPH_FACTS: readonly GraphEdgeKind[] = + GRAPH_EDGE_KINDS.filter((kind) => kind !== "renders"); diff --git a/packages/graph/src/provider/kotlin/KOTLIN_GRAPH_PRODUCER.ts b/packages/graph/src/provider/kotlin/KOTLIN_GRAPH_PRODUCER.ts new file mode 100644 index 00000000..d8ecde99 --- /dev/null +++ b/packages/graph/src/provider/kotlin/KOTLIN_GRAPH_PRODUCER.ts @@ -0,0 +1,2 @@ +/** Producer identity written by the pinned K2 graph exporter. */ +export const KOTLIN_GRAPH_PRODUCER = "scip-kotlinc-k2-graph"; diff --git a/packages/graph/src/provider/kotlin/KOTLIN_GRAPH_PROVIDER.ts b/packages/graph/src/provider/kotlin/KOTLIN_GRAPH_PROVIDER.ts new file mode 100644 index 00000000..3f432199 --- /dev/null +++ b/packages/graph/src/provider/kotlin/KOTLIN_GRAPH_PROVIDER.ts @@ -0,0 +1,2 @@ +/** Registry identity of the compiler-owned Kotlin/JVM graph route. */ +export const KOTLIN_GRAPH_PROVIDER = "kotlinc-graph"; diff --git a/packages/graph/src/provider/kotlin/KotlinGraphProducerClient.ts b/packages/graph/src/provider/kotlin/KotlinGraphProducerClient.ts new file mode 100644 index 00000000..84164e6a --- /dev/null +++ b/packages/graph/src/provider/kotlin/KotlinGraphProducerClient.ts @@ -0,0 +1,19 @@ +import { ResidentGraphProducerClient } from "../compiler/ResidentGraphProducerClient"; + +/** Kotlin specialization of the restartable compiler-producer transport. */ +export class KotlinGraphProducerClient extends ResidentGraphProducerClient { + public constructor(options: KotlinGraphProducerClient.IOptions) { + super({ + ...options, + serverCommand: "kotlin-graph-server", + label: "Kotlin graph", + }); + } +} + +export namespace KotlinGraphProducerClient { + export type IOptions = Omit< + ResidentGraphProducerClient.IOptions, + "serverCommand" | "label" + >; +} diff --git a/packages/graph/src/provider/kotlin/KotlinGraphSession.ts b/packages/graph/src/provider/kotlin/KotlinGraphSession.ts new file mode 100644 index 00000000..7e9cd753 --- /dev/null +++ b/packages/graph/src/provider/kotlin/KotlinGraphSession.ts @@ -0,0 +1,22 @@ +import { CompilerGraphSession } from "../compiler/CompilerGraphSession"; +import { KotlinGraphSnapshotAdapter } from "./KotlinGraphSnapshotAdapter"; + +/** Kotlin specialization of the shared resident compiler session. */ +export class KotlinGraphSession extends CompilerGraphSession { + public constructor(options: KotlinGraphSession.IOptions) { + super({ + ...options, + adapter: new KotlinGraphSnapshotAdapter(options.root), + serverCommand: "kotlin-graph-server", + label: "Kotlin graph", + artifactName: "graph.json", + }); + } +} + +export namespace KotlinGraphSession { + export type IOptions = Omit< + CompilerGraphSession.IOptions, + "adapter" | "serverCommand" | "label" | "artifactName" + >; +} diff --git a/packages/graph/src/provider/kotlin/KotlinGraphSnapshotAdapter.ts b/packages/graph/src/provider/kotlin/KotlinGraphSnapshotAdapter.ts new file mode 100644 index 00000000..12bd366b --- /dev/null +++ b/packages/graph/src/provider/kotlin/KotlinGraphSnapshotAdapter.ts @@ -0,0 +1,24 @@ +import { CompilerGraphSnapshotAdapter } from "../compiler/CompilerGraphSnapshotAdapter"; +import { IKotlinGraphSnapshot } from "./IKotlinGraphSnapshot"; +import { KOTLIN_GRAPH_FACTS } from "./KOTLIN_GRAPH_FACTS"; +import { KOTLIN_GRAPH_PRODUCER } from "./KOTLIN_GRAPH_PRODUCER"; +import { KOTLIN_GRAPH_PROVIDER } from "./KOTLIN_GRAPH_PROVIDER"; + +const KOTLIN_CONTRACT: CompilerGraphSnapshotAdapter.IContract = { + label: "Kotlin graph", + language: "kotlin", + provider: KOTLIN_GRAPH_PROVIDER, + producer: KOTLIN_GRAPH_PRODUCER, + facts: KOTLIN_GRAPH_FACTS, + diagnosticCode: "kotlinc", + shardKeyPrefix: "kotlin", + schemaVersion: IKotlinGraphSnapshot.SCHEMA_VERSION, + protocolVersion: IKotlinGraphSnapshot.PROTOCOL_VERSION, +}; + +/** Kotlin specialization of the shared strict snapshot adapter. */ +export class KotlinGraphSnapshotAdapter extends CompilerGraphSnapshotAdapter { + public constructor(root: string) { + super(root, KOTLIN_CONTRACT); + } +} diff --git a/packages/graph/src/provider/kotlin/index.ts b/packages/graph/src/provider/kotlin/index.ts new file mode 100644 index 00000000..e3b04940 --- /dev/null +++ b/packages/graph/src/provider/kotlin/index.ts @@ -0,0 +1,8 @@ +export * from "./IKotlinGraphSnapshot"; +export * from "./KOTLIN_GRAPH_FACTS"; +export * from "./KOTLIN_GRAPH_PRODUCER"; +export * from "./KOTLIN_GRAPH_PROVIDER"; +export * from "./KotlinGraphSession"; +export * from "./KotlinGraphProducerClient"; +export * from "./KotlinGraphSnapshotAdapter"; +export * from "./kotlinGraphProvider"; diff --git a/packages/graph/src/provider/kotlin/kotlinGraphProvider.ts b/packages/graph/src/provider/kotlin/kotlinGraphProvider.ts new file mode 100644 index 00000000..cea1e880 --- /dev/null +++ b/packages/graph/src/provider/kotlin/kotlinGraphProvider.ts @@ -0,0 +1,240 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { spawnableCommand } from "../../utils/spawnableCommand"; +import { assertGraphSnapshotContract } from "../assertGraphSnapshotContract"; +import { IGraphProvider } from "../IGraphProvider"; +import { providerInputFiles } from "../providerInputFiles"; +import { resolveProviderCommand } from "../resolveProviderCommand"; +import { standardScipProviders } from "../scip/standardScipProviders"; +import { toolchainVersion } from "../toolchainVersion"; +import { KOTLIN_GRAPH_FACTS } from "./KOTLIN_GRAPH_FACTS"; +import { KOTLIN_GRAPH_PROVIDER } from "./KOTLIN_GRAPH_PROVIDER"; +import { KotlinGraphSession } from "./KotlinGraphSession"; + +const OVERRIDE = "SAMCHON_GRAPH_KOTLINC_GRAPH"; +const TOOLCHAIN_OVERRIDE = "SAMCHON_GRAPH_JAVA_TOOLCHAIN"; +const GRAPH_OPTION = "--kotlin-graph-output"; +const SERVER_COMMAND = "kotlin-graph-server"; +const SERVER_CAPABILITY = "Serve compiler-owned Kotlin graph generations over NDJSON."; +const kotlinScipProvider = standardScipProviders.find( + (provider) => provider.name === "scip-kotlinc", +); +/* c8 ignore next 4 -- the static standard-provider registry always contains + * the scip-kotlinc descriptor; startup must still fail closed if it is edited. */ +if (kotlinScipProvider === undefined) { + throw new Error("kotlinc-graph: the scip-kotlinc fallback is not registered"); +} +/** + * The build files a registry entry watches, shared with the SCIP lane because + * the two routes read the same project configuration. + * + * These are the registry's `buildInputs`, and they are deliberately *not* the + * session's inputs. A registry entry declares the files outside its own + * language whose change invalidates it; a session fingerprints everything it + * compiled. Handing the first list to the second is what let a Kotlin source + * edit leave the fingerprint unmoved, so the session reused a snapshot taken + * before the edit and the coordinator's fence refused it correctly, and with + * a digest that described bytes no longer on disk. + */ +const kotlinBuildInputs = kotlinScipProvider.buildInputs; +/* c8 ignore start -- every SCIP descriptor derives its build inputs from the + * project; startup must still fail closed if that stops being true. */ +if (typeof kotlinBuildInputs !== "function") { + throw new Error( + "kotlinc-graph: the scip-kotlinc fallback names no build inputs", + ); +} +/* c8 ignore stop */ + +/** + * Every input whose change can move a Kotlin target's committed generation. + * + * Sources and build files together, because either alone is a half-answer: a + * classpath edit in `pom.xml` recompiles sources it never touched, and a + * source edit moves facts no build file mentions. This is the fingerprint that + * decides whether the build has to run at all, so a file missing from it is a + * file whose edit the route will not notice. + */ +const kotlinInputs = (root: string): string[] => [ + ...new Set([ + ...providerInputFiles(root, ["kotlin"], []), + ...kotlinBuildInputs(root), + ]), +]; + +/** + * The compiler-owned Kotlin route: kotlinc writes the graph, not a second reader. + * + * The producer is a plugin on the project's own compile tasks, so the facts + * come from the attributed trees the build already produced and cost one + * traversal of each. That is what separates this entry from the SCIP lane + * behind it, which runs a whole indexing build of its own and still cannot + * prove a call because SCIP carries no distinct call role, while this route + * publishes the call, access, and dispatch roles kotlinc resolved. + * + * It sits ahead of that lane rather than replacing it. A released `scip-java` + * that predates `--kotlin-graph-output` is a perfectly good navigation indexer and + * declining to it is the honest answer; what must not happen is this route + * quietly answering with SCIP facts under a compiler-authority provenance. + */ +export const kotlinGraphProvider: IGraphProvider = { + name: KOTLIN_GRAPH_PROVIDER, + languages: ["kotlin"], + authority: "compiler", + facts: KOTLIN_GRAPH_FACTS, + resolution: { + commands: ["scip-java", "java"], + environmentOverrides: [OVERRIDE, TOOLCHAIN_OVERRIDE], + }, + fallbacks: [kotlinScipProvider], + buildInputs: kotlinScipProvider.buildInputs, + configuration: (root, env) => [...kotlinToolchain(root, env).rows], + configurationDerivation: (root, env) => kotlinToolchain(root, env), + refuse: (options) => { + const refused = [ + options.server === undefined ? undefined : "server", + options.maxFiles === undefined ? undefined : "maxFiles", + options.lspReferenceLimit === undefined + ? undefined + : "lspReferenceLimit", + ].filter((value): value is string => value !== undefined); + return refused.length === 0 + ? undefined + : `kotlin: ${KOTLIN_GRAPH_PROVIDER} publishes whole-target generations from the project's own compile tasks and cannot honor ${refused.join(", ")}`; + }, + resolve: (root, env) => { + if (!hasGradleBuild(root)) return undefined; + const launcher = resolveProviderCommand(root, env, { + command: "scip-java", + override: OVERRIDE, + }); + if (launcher === undefined) return undefined; + // The graph output is a capability of the launcher, not of its name. A + // released build without it answers `index` perfectly well and writes no + // graph at all, so the strict route would run a whole build and then find + // nothing to read. Asking `index --help` costs one process and turns that + // into an ordinary decline before anything is compiled. + return publishesGraphOutput(root, env, launcher) && + publishesResidentServer(root, env, launcher) + ? launcher + : undefined; + }, + open: (props) => + new KotlinGraphSession({ + root: props.root, + languages: props.languages, + provider: KOTLIN_GRAPH_PROVIDER, + command: props.command, + inputs: () => kotlinInputs(props.root), + configuration: () => kotlinToolchain(props.root, process.env), + validate: (snapshot) => + assertGraphSnapshotContract( + snapshot, + kotlinGraphProvider, + props.languages, + props.root, + ), + }), +}; + +/** The build-integrated exporter currently supports Kotlin/JVM Gradle only. */ +function hasGradleBuild(root: string): boolean { + return [ + "settings.gradle", + "settings.gradle.kts", + "build.gradle", + "build.gradle.kts", + "gradlew", + "gradlew.bat", + ].some((file) => fs.existsSync(path.join(root, file))); +} + + +/** + * The JDK that will run the plugin and the launcher that will attach it. + * + * Both are identity, not decoration. A build that switches JDKs recompiles + * against a different Kotlin compiler and resolves different overloads, and a + * launcher upgrade can change the shard schema underneath an unchanged + * project; a universe that ignored either would reuse facts neither produced. + */ +function kotlinToolchain( + root: string, + env: NodeJS.ProcessEnv, +): toolchainVersion.IDerivation { + return toolchainVersion.derive([ + toolchainVersion.observe({ + root, + env, + command: "java", + override: TOOLCHAIN_OVERRIDE, + // `--version`, not `-version`. The single-dash form is the pre-Kotlin-9 + // spelling and writes to standard error, where the shared probe does not + // read; the JDK has answered the double-dash form on standard output + // since 9, and the SCIP lane already asks every toolchain that way. + args: ["--version"], + label: "java", + }), + toolchainVersion.observe({ + root, + env, + command: "scip-java", + override: OVERRIDE, + args: ["--version"], + label: "scip-java", + }), + ]); +} + +function publishesGraphOutput( + root: string, + env: NodeJS.ProcessEnv, + command: IGraphProvider.ICommand, +): boolean { + const invocation = spawnableCommand.append( + { ...command, args: [...command.args] }, + ["index", "--help"], + ); + const result = spawnSync(invocation.command, invocation.args, { + cwd: root, + encoding: "utf8", + env, + shell: false, + timeout: 30_000, + windowsHide: true, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + // No separate arm for a launch that never happened: a spawn error leaves + // both streams empty, which is already the answer this returns for a + // launcher that ran and published nothing. + /* c8 ignore start -- an executed spawnSync with UTF-8 encoding returns + * strings; the null arms exist only for Node's broader result type. */ + return `${result.stdout ?? ""}${result.stderr ?? ""}`.includes(GRAPH_OPTION); + /* c8 ignore stop */ +} + +function publishesResidentServer( + root: string, + env: NodeJS.ProcessEnv, + command: IGraphProvider.ICommand, +): boolean { + const invocation = spawnableCommand.append( + { ...command, args: [...command.args] }, + [SERVER_COMMAND, "--help"], + ); + const result = spawnSync(invocation.command, invocation.args, { + cwd: root, + encoding: "utf8", + env, + shell: false, + timeout: 30_000, + windowsHide: true, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + /* c8 ignore next -- an executed UTF-8 spawnSync returns both streams. */ + return `${result.stdout ?? ""}${result.stderr ?? ""}`.includes( + SERVER_CAPABILITY, + ); +} diff --git a/packages/graph/src/provider/rust/RUST_GRAPH_PRODUCER_COMMIT.ts b/packages/graph/src/provider/rust/RUST_GRAPH_PRODUCER_COMMIT.ts index eceecb30..8689de14 100644 --- a/packages/graph/src/provider/rust/RUST_GRAPH_PRODUCER_COMMIT.ts +++ b/packages/graph/src/provider/rust/RUST_GRAPH_PRODUCER_COMMIT.ts @@ -1,2 +1,2 @@ export const RUST_GRAPH_PRODUCER_COMMIT = - "2850ecba80311bebd4cdaa9fedc5321533b5b1e7"; + "378f220482c298775910f0fc46e8fda1bc516ecc"; diff --git a/packages/graph/src/provider/rust/RustGraphClient.ts b/packages/graph/src/provider/rust/RustGraphClient.ts index d3156e9d..2f5ed3b7 100644 --- a/packages/graph/src/provider/rust/RustGraphClient.ts +++ b/packages/graph/src/provider/rust/RustGraphClient.ts @@ -13,7 +13,6 @@ import { RustGraphSnapshotAdapter } from "./RustGraphSnapshotAdapter"; const GRAPH_METHOD = "samchon/graphSnapshot"; const SERVER_CANCELLED = -32802; const CONTENT_MODIFIED = -32801; -const DEFAULT_READY_TIMEOUT_MS = 300_000; const RETRY_DELAY_MS = 50; const MAX_RETRY_DELAY_MS = 5_000; @@ -31,7 +30,7 @@ export class RustGraphClient implements IBulkGraphSession { ) => void; private readonly initializationOptions: unknown; private readonly requestTimeoutMs: number | undefined; - private readonly readyTimeoutMs: number; + private readonly readyTimeoutMs: number | undefined; private readonly lifecycleAbort = new AbortController(); private queue: Promise = Promise.resolve(); private initialized: Promise | undefined; @@ -73,7 +72,7 @@ export class RustGraphClient implements IBulkGraphSession { this.version = 0; this.initializationOptions = options.initializationOptions; this.requestTimeoutMs = options.requestTimeoutMs; - this.readyTimeoutMs = options.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS; + this.readyTimeoutMs = options.readyTimeoutMs; /* c8 ignore start -- production native binaries need no arguments; the * protocol fixture itself is a JavaScript file and therefore needs one. */ const args = options.args ?? []; @@ -190,7 +189,10 @@ export class RustGraphClient implements IBulkGraphSession { } private async requestSnapshot(signal: AbortSignal): Promise { - const deadline = performance.now() + this.readyTimeoutMs; + const deadline = + this.readyTimeoutMs === undefined + ? undefined + : performance.now() + this.readyTimeoutMs; let checkpoint = this.checkpointPending ? this.adapter.persistedCheckpoint : undefined; @@ -232,20 +234,22 @@ export class RustGraphClient implements IBulkGraphSession { ) { throw error; } - if (performance.now() >= deadline) { + const remaining = + deadline === undefined ? undefined : deadline - performance.now(); + if (remaining !== undefined && remaining <= 0) { throw new Error( `rust HIR graph: producer did not become ready within ${String(this.readyTimeoutMs)} ms: ${error.message}`, ); } await delay( - Math.min(backoff, Math.max(0, deadline - performance.now())), + remaining === undefined ? backoff : Math.min(backoff, remaining), signal, ); // Same backoff, same clamp and same reset as the Clang client, for the // same reasons written out there. This lane has never demonstrated the // problem — rust-analyzer becomes ready quickly on the pinned corpus, - // and its row runs on the default timeout — but the loop is the same - // shape, so it should not be the one left to find out on a larger + // whose experiment supplies its own deadline — but the loop is the + // same shape, so it should not be the one left to find out on a larger // workspace. backoff = error.code === CONTENT_MODIFIED diff --git a/packages/graph/src/provider/scala/IScalaGraphSnapshot.ts b/packages/graph/src/provider/scala/IScalaGraphSnapshot.ts new file mode 100644 index 00000000..ff56567b --- /dev/null +++ b/packages/graph/src/provider/scala/IScalaGraphSnapshot.ts @@ -0,0 +1,119 @@ +/** Aggregate artifact committed by the BSP-driven Scala graph producer. */ +export interface IScalaGraphSnapshot { + schemaVersion: number; + projectRoot: string; + producer: IScalaGraphSnapshot.IProducer; + targets: IScalaGraphSnapshot.ITarget[]; +} + +export namespace IScalaGraphSnapshot { + export interface IProducer { + name: string; + version: string; + protocolVersion: number; + capabilities: ICapabilities; + } + + export interface ICapabilities { + atomicGenerations: boolean; + incremental: boolean; + diagnostics: boolean; + bsp: boolean; + semanticdb: boolean; + typedPlugins: boolean; + zinc: boolean; + } + + /** One BSP build target and its independently invalidated Scala universe. */ + export interface ITarget { + name: string; + generation: string; + universe: string; + bspUri: string; + scalaVersion: string; + scalaBinaryVersion: string; + platform: string; + sourceEncoding: string; + scalacOptionsDigest: string; + classpathDigest: string; + sourceRootsDigest: string; + semanticdbOptionsDigest: string; + compilerPluginsDigest: string; + zincAnalysisDigest: string; + generatedSourcesDigest: string; + coverage: Record; + shards: IShard[]; + } + + /** One source emitted by the typed plugin and cross-checked with SemanticDB. */ + export interface IShard { + schemaVersion: number; + language: string; + source: string; + checkerDigest: string; + diskDigest: string; + target: string; + compilerVersion: string; + compilerPlugin: "scala2" | "scala3"; + compilerPluginVersion: string; + semanticdbSchema: number; + semanticdbUri: string; + semanticdbMd5: string; + semanticdbBuildTarget: string; + nodes: INode[]; + edges: IEdge[]; + unresolved: IUnresolved[]; + diagnostics: IDiagnostic[]; + } + + export interface IEvidence { + file: string; + startLine: number; + startColumn: number; + endLine: number; + endColumn: number; + } + + export interface INode { + /** Stable structural identity, never a SemanticDB overload ordinal. */ + symbol: string; + kind: string; + name: string; + qualifiedName: string; + file: string; + exported: boolean; + modifiers: string[]; + signature: string; + origin: string; + evidence: IEvidence; + } + + export interface IEdge { + from: string; + to: string; + kind: string; + access: string | null; + provenance: string | null; + targetKind: string | null; + targetName: string | null; + targetQualifiedName: string | null; + evidence: IEvidence; + } + + export interface IUnresolved { + family: string; + reason: string; + evidence: IEvidence; + candidates: string[]; + } + + export interface IDiagnostic { + severity: string; + message: string; + evidence: IEvidence; + } + + export const SCHEMA_VERSION = 1; + export const PROTOCOL_VERSION = 1; + export const SEMANTICDB_SCHEMA = 4; +} diff --git a/packages/graph/src/provider/scala/SCALA_GRAPH_FACTS.ts b/packages/graph/src/provider/scala/SCALA_GRAPH_FACTS.ts new file mode 100644 index 00000000..63298ae4 --- /dev/null +++ b/packages/graph/src/provider/scala/SCALA_GRAPH_FACTS.ts @@ -0,0 +1,5 @@ +import { GRAPH_EDGE_KINDS, GraphEdgeKind } from "../../typings"; + +/** Families the typed plugins can publish; UI rendering and test intent need enrichers. */ +export const SCALA_GRAPH_FACTS: readonly GraphEdgeKind[] = + GRAPH_EDGE_KINDS.filter((kind) => !["renders", "tests"].includes(kind)); diff --git a/packages/graph/src/provider/scala/SCALA_GRAPH_PRODUCER.ts b/packages/graph/src/provider/scala/SCALA_GRAPH_PRODUCER.ts new file mode 100644 index 00000000..8fa58440 --- /dev/null +++ b/packages/graph/src/provider/scala/SCALA_GRAPH_PRODUCER.ts @@ -0,0 +1,2 @@ +/** Producer identity written by the paired Scala 2 and Scala 3 plugins. */ +export const SCALA_GRAPH_PRODUCER = "samchon-scala-graph"; diff --git a/packages/graph/src/provider/scala/SCALA_GRAPH_PROVIDER.ts b/packages/graph/src/provider/scala/SCALA_GRAPH_PROVIDER.ts new file mode 100644 index 00000000..5460e7db --- /dev/null +++ b/packages/graph/src/provider/scala/SCALA_GRAPH_PROVIDER.ts @@ -0,0 +1,2 @@ +/** Registry identity of the BSP-driven Scala compiler graph route. */ +export const SCALA_GRAPH_PROVIDER = "scalac-graph"; diff --git a/packages/graph/src/provider/scala/ScalaGraphSession.ts b/packages/graph/src/provider/scala/ScalaGraphSession.ts new file mode 100644 index 00000000..e746bad6 --- /dev/null +++ b/packages/graph/src/provider/scala/ScalaGraphSession.ts @@ -0,0 +1,59 @@ +import { GraphLanguage } from "../../typings"; +import { BatchGraphSession } from "../BatchGraphSession"; +import { IBulkGraphSession } from "../IBulkGraphSession"; +import { IGraphProvider } from "../IGraphProvider"; +import { CompilerGraphSession } from "../compiler/CompilerGraphSession"; +import { ScalaGraphSnapshotAdapter } from "./ScalaGraphSnapshotAdapter"; + +/** Resident BSP session backed by the paired Scala compiler plugins. */ +export class ScalaGraphSession implements IBulkGraphSession { + public readonly kind = "bulk" as const; + public readonly ownsProviderTopology = true; + public readonly languages: readonly GraphLanguage[]; + public readonly root: string; + + private readonly session: CompilerGraphSession; + + public constructor(options: ScalaGraphSession.IOptions) { + this.session = new CompilerGraphSession({ + ...options, + adapter: new ScalaGraphSnapshotAdapter(options.root), + serverCommand: "graph-server", + label: "Scala graph", + artifactName: "scala-graph.json", + }); + this.languages = this.session.languages; + this.root = this.session.root; + } + + public get generation(): number { + return this.session.generation; + } + + public get current(): IBulkGraphSession.ISnapshot | undefined { + return this.session.current; + } + + public refresh( + options: { signal?: AbortSignal } = {}, + ): Promise { + return this.session.refresh(options); + } + + public close(): Promise { + return this.session.close(); + } +} + +export namespace ScalaGraphSession { + export interface IOptions { + root: string; + languages: readonly GraphLanguage[]; + provider: string; + command: IGraphProvider.ICommand; + inputs: () => string[]; + configuration: NonNullable; + validate: (snapshot: IBulkGraphSession.ISnapshot) => void; + maxArtifactBytes?: number; + } +} diff --git a/packages/graph/src/provider/scala/ScalaGraphSnapshotAdapter.ts b/packages/graph/src/provider/scala/ScalaGraphSnapshotAdapter.ts new file mode 100644 index 00000000..0c2b0db1 --- /dev/null +++ b/packages/graph/src/provider/scala/ScalaGraphSnapshotAdapter.ts @@ -0,0 +1,131 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { CompilerGraphSnapshotAdapter } from "../compiler/CompilerGraphSnapshotAdapter"; +import { IScalaGraphSnapshot } from "./IScalaGraphSnapshot"; +import { SCALA_GRAPH_FACTS } from "./SCALA_GRAPH_FACTS"; +import { SCALA_GRAPH_PRODUCER } from "./SCALA_GRAPH_PRODUCER"; +import { SCALA_GRAPH_PROVIDER } from "./SCALA_GRAPH_PROVIDER"; + +const SHA256 = /^[0-9a-f]{64}$/u; +const MD5 = /^[0-9a-f]{32}$/u; + +/** Validate and adapt one atomic set of BSP target generations. */ +export class ScalaGraphSnapshotAdapter extends CompilerGraphSnapshotAdapter { + public constructor(root: string) { + super(root, { + label: "Scala graph", + language: "scala", + provider: SCALA_GRAPH_PROVIDER, + producer: SCALA_GRAPH_PRODUCER, + facts: SCALA_GRAPH_FACTS, + capabilities: ["bsp", "semanticdb", "typedPlugins", "zinc"], + diagnosticCode: "scalac", + shardKeyPrefix: "scala", + schemaVersion: IScalaGraphSnapshot.SCHEMA_VERSION, + protocolVersion: IScalaGraphSnapshot.PROTOCOL_VERSION, + identitySalt: (rawTarget) => { + const target = rawTarget as unknown as IScalaGraphSnapshot.ITarget; + return `${target.scalaBinaryVersion}\0${target.sourceEncoding.toLowerCase()}`; + }, + validateSnapshot: (raw) => { + const snapshot = raw as unknown as IScalaGraphSnapshot; + const capabilities = snapshot.producer.capabilities; + if ( + capabilities.bsp !== true || + capabilities.semanticdb !== true || + capabilities.typedPlugins !== true || + capabilities.zinc !== true + ) { + throw new Error( + "Scala graph: the producer does not prove BSP, SemanticDB, typed-plugin and Zinc ownership", + ); + } + }, + validateTarget: (raw) => { + const target = raw as unknown as IScalaGraphSnapshot.ITarget; + if ( + target.name !== target.bspUri || + !isUri(target.bspUri) || + !scalaVersion(target.scalaVersion) || + !scalaBinaryVersion(target.scalaVersion, target.scalaBinaryVersion) || + target.platform === "" || + !sourceEncoding(target.sourceEncoding) || + !SHA256.test(target.scalacOptionsDigest) || + !SHA256.test(target.classpathDigest) || + !SHA256.test(target.sourceRootsDigest) || + !SHA256.test(target.semanticdbOptionsDigest) || + !SHA256.test(target.compilerPluginsDigest) || + !SHA256.test(target.zincAnalysisDigest) || + !SHA256.test(target.generatedSourcesDigest) + ) { + throw new Error(`Scala graph: malformed BSP target ${target.name}`); + } + }, + validateShard: (raw, rawTarget, root) => { + const shard = raw as unknown as IScalaGraphSnapshot.IShard; + const target = rawTarget as unknown as IScalaGraphSnapshot.ITarget; + const expectedPlugin = target.scalaVersion.startsWith("2.") + ? "scala2" + : "scala3"; + if ( + shard.compilerVersion !== target.scalaVersion || + shard.compilerPlugin !== expectedPlugin || + shard.compilerPluginVersion === "" || + shard.semanticdbSchema !== IScalaGraphSnapshot.SEMANTICDB_SCHEMA || + shard.semanticdbUri !== shard.source || + shard.semanticdbBuildTarget !== target.bspUri || + !MD5.test(shard.semanticdbMd5) + ) { + throw new Error( + `Scala graph: malformed SemanticDB cross-check in ${shard.source}`, + ); + } + const source = path.resolve(root, shard.source); + let bytes: Buffer; + try { + bytes = fs.readFileSync(source); + } catch (error) { + throw new Error( + `Scala graph: cannot read SemanticDB source ${shard.source}: ${asError(error).message}`, + ); + } + const actual = createHash("md5").update(bytes).digest("hex"); + if (actual !== shard.semanticdbMd5) { + throw new Error( + `Scala graph: SemanticDB md5 does not match ${shard.source}`, + ); + } + }, + }); + } +} + +function scalaVersion(value: unknown): value is string { + return typeof value === "string" && /^(?:2\.1[23]|3)\./u.test(value); +} + +function scalaBinaryVersion(version: string, binary: unknown): boolean { + if (typeof binary !== "string") return false; + return version.startsWith("3.") + ? binary === "3" + : binary === version.split(".").slice(0, 2).join("."); +} + +function sourceEncoding(value: unknown): value is string { + return typeof value === "string" && /^[A-Za-z0-9._-]+$/u.test(value); +} + +function isUri(value: string): boolean { + try { + return new URL(value).protocol !== ""; + } catch { + return false; + } +} + +function asError(value: unknown): Error { + /* c8 ignore next -- Node's synchronous filesystem APIs throw Error objects. */ + return value instanceof Error ? value : new Error(String(value)); +} diff --git a/packages/graph/src/provider/scala/index.ts b/packages/graph/src/provider/scala/index.ts new file mode 100644 index 00000000..29fb75dd --- /dev/null +++ b/packages/graph/src/provider/scala/index.ts @@ -0,0 +1,7 @@ +export * from "./IScalaGraphSnapshot"; +export * from "./SCALA_GRAPH_FACTS"; +export * from "./SCALA_GRAPH_PRODUCER"; +export * from "./SCALA_GRAPH_PROVIDER"; +export * from "./ScalaGraphSession"; +export * from "./ScalaGraphSnapshotAdapter"; +export * from "./scalaGraphProvider"; diff --git a/packages/graph/src/provider/scala/scalaGraphProvider.ts b/packages/graph/src/provider/scala/scalaGraphProvider.ts new file mode 100644 index 00000000..a020d6da --- /dev/null +++ b/packages/graph/src/provider/scala/scalaGraphProvider.ts @@ -0,0 +1,169 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { compareOrdinal } from "@samchon/graph-sitter"; + +import { languageBuildInputs } from "../../indexer/languageBuildInputs"; +import { spawnableCommand } from "../../utils/spawnableCommand"; +import { assertGraphSnapshotContract } from "../assertGraphSnapshotContract"; +import { IGraphProvider } from "../IGraphProvider"; +import { providerInputFiles } from "../providerInputFiles"; +import { resolveProviderCommand } from "../resolveProviderCommand"; +import { toolchainVersion } from "../toolchainVersion"; +import { SCALA_GRAPH_FACTS } from "./SCALA_GRAPH_FACTS"; +import { SCALA_GRAPH_PRODUCER } from "./SCALA_GRAPH_PRODUCER"; +import { SCALA_GRAPH_PROVIDER } from "./SCALA_GRAPH_PROVIDER"; +import { ScalaGraphSession } from "./ScalaGraphSession"; + +const OVERRIDE = "SAMCHON_GRAPH_SCALA_GRAPH"; +const TOOLCHAIN_OVERRIDE = "SAMCHON_GRAPH_JAVA_TOOLCHAIN"; +const SERVER_CAPABILITY = + "Serve BSP-driven Scala compiler graph generations over NDJSON."; + +/** Scala sources, build definitions and usable BSP connection details. */ +const scalaInputs = (root: string): string[] => [ + ...new Set([ + ...providerInputFiles(root, ["scala"], []), + ...languageBuildInputs(root, ["scala"]), + ...bspFiles(root), + ]), +]; + +/** Compiler-owned Scala 2/3 facts produced in the repository's BSP compile. */ +export const scalaGraphProvider: IGraphProvider = { + name: SCALA_GRAPH_PROVIDER, + languages: ["scala"], + authority: "compiler", + facts: SCALA_GRAPH_FACTS, + resolution: { + commands: [SCALA_GRAPH_PRODUCER, "java"], + environmentOverrides: [OVERRIDE, TOOLCHAIN_OVERRIDE], + }, + buildInputs: (root) => [ + ...languageBuildInputs(root, ["scala"]), + ...bspFiles(root), + ], + configuration: (root, env) => [...scalaToolchain(root, env).rows], + configurationDerivation: (root, env) => scalaToolchain(root, env), + refuse: (options) => { + const refused = [ + options.server === undefined ? undefined : "server", + options.maxFiles === undefined ? undefined : "maxFiles", + options.lspReferenceLimit === undefined + ? undefined + : "lspReferenceLimit", + ].filter((value): value is string => value !== undefined); + return refused.length === 0 + ? undefined + : `scala: ${SCALA_GRAPH_PROVIDER} publishes complete BSP target generations and cannot honor ${refused.join(", ")}`; + }, + resolve: (root, env) => { + if (bspFiles(root).length === 0) return undefined; + const command = resolveProviderCommand(root, env, { + command: SCALA_GRAPH_PRODUCER, + override: OVERRIDE, + }); + return command !== undefined && + publishesResidentServer(root, env, command) && + supportsProject(root, env, command) + ? command + : undefined; + }, + open: (props) => + new ScalaGraphSession({ + root: props.root, + languages: props.languages, + provider: SCALA_GRAPH_PROVIDER, + command: props.command, + inputs: () => scalaInputs(props.root), + configuration: () => scalaToolchain(props.root, process.env), + validate: (snapshot) => + assertGraphSnapshotContract( + snapshot, + scalaGraphProvider, + props.languages, + props.root, + ), + }), +}; + +function bspFiles(root: string): string[] { + const directory = path.join(root, ".bsp"); + if (!fs.existsSync(directory)) return []; + return fs + .readdirSync(directory, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => `.bsp/${entry.name}`) + .sort(compareOrdinal); +} + +function scalaToolchain( + root: string, + env: NodeJS.ProcessEnv, +): toolchainVersion.IDerivation { + return toolchainVersion.derive([ + toolchainVersion.observe({ + root, + env, + command: "java", + override: TOOLCHAIN_OVERRIDE, + args: ["--version"], + label: "java", + }), + toolchainVersion.observe({ + root, + env, + command: SCALA_GRAPH_PRODUCER, + override: OVERRIDE, + args: ["--version"], + label: SCALA_GRAPH_PRODUCER, + }), + ]); +} + +function publishesResidentServer( + root: string, + env: NodeJS.ProcessEnv, + command: IGraphProvider.ICommand, +): boolean { + const invocation = spawnableCommand.append( + { ...command, args: [...command.args] }, + ["graph-server", "--help"], + ); + const result = spawnSync(invocation.command, invocation.args, { + cwd: root, + encoding: "utf8", + env, + shell: false, + timeout: 30_000, + windowsHide: true, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + return ( + result.status === 0 && + `${result.stdout}${result.stderr}`.includes(SERVER_CAPABILITY) + ); +} + +function supportsProject( + root: string, + env: NodeJS.ProcessEnv, + command: IGraphProvider.ICommand, +): boolean { + const invocation = spawnableCommand.append( + { ...command, args: [...command.args] }, + ["supports", "--cwd", root], + ); + return ( + spawnSync(invocation.command, invocation.args, { + cwd: root, + encoding: "utf8", + env, + shell: false, + timeout: 120_000, + windowsHide: true, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }).status === 0 + ); +} diff --git a/packages/graph/src/provider/scip/ScipSession.ts b/packages/graph/src/provider/scip/ScipSession.ts index 0e20ddf7..a94fd2c6 100644 --- a/packages/graph/src/provider/scip/ScipSession.ts +++ b/packages/graph/src/provider/scip/ScipSession.ts @@ -272,9 +272,7 @@ export class ScipSession implements IBulkGraphSession { if (projectRoot === "") { throw new Error(`${this.options.provider}: the index has no project root`); } - const declared = projectRoot.startsWith("file://") - ? fileFromUri(projectRoot) - : projectRoot; + const declared = fileFromUri(projectRoot); if (!samePath(declared, this.root)) { throw new Error( `${this.options.provider}: the index was produced for ${declared}, not ${this.root}`, diff --git a/packages/graph/src/provider/scip/standardScipProviders.ts b/packages/graph/src/provider/scip/standardScipProviders.ts index 3c0e95d0..73555022 100644 --- a/packages/graph/src/provider/scip/standardScipProviders.ts +++ b/packages/graph/src/provider/scip/standardScipProviders.ts @@ -244,7 +244,14 @@ const dotnetScipProvider = createScipProvider({ "packages.lock.json", "nuget.config", ], - buildExtensions: [".sln", ".csproj", ".fsproj", ".props", ".targets"], + buildExtensions: [ + ".sln", + ".slnx", + ".csproj", + ".fsproj", + ".props", + ".targets", + ], indexArgs: (artifact) => ["index", "--output", artifact], }); diff --git a/packages/graph/src/provider/sidecar/SidecarSession.ts b/packages/graph/src/provider/sidecar/SidecarSession.ts index 0270d817..76334a99 100644 --- a/packages/graph/src/provider/sidecar/SidecarSession.ts +++ b/packages/graph/src/provider/sidecar/SidecarSession.ts @@ -170,9 +170,7 @@ export class SidecarSession implements IBulkGraphSession { `${this.options.provider}: the snapshot has no project root`, ); } - const declared = projectRoot.startsWith("file://") - ? fileFromUri(projectRoot) - : projectRoot; + const declared = fileFromUri(projectRoot); if (!samePath(declared, this.root)) { throw new Error( `${this.options.provider}: the snapshot was produced for ${declared}, not ${this.root}`, diff --git a/packages/graph/src/provider/sidecar/standardSidecarProviders.ts b/packages/graph/src/provider/sidecar/standardSidecarProviders.ts index 9820c621..03aa2a4c 100644 --- a/packages/graph/src/provider/sidecar/standardSidecarProviders.ts +++ b/packages/graph/src/provider/sidecar/standardSidecarProviders.ts @@ -19,18 +19,9 @@ import { IGraphProvider } from "../IGraphProvider"; * analysis engine, so the producer became a script inside the server rather than * a program beside it, in `provider/lua`. * - * The last two leave by being withdrawn, because for each the honest answer is - * not a program yet. - * - * **swift.** The channel is settled: `swift build` emits an index store during - * an ordinary debug build, and its records carry `RelChild`, so an occurrence - * arrives already naming the declaration that encloses it — the one thing lua's - * exporter could not answer. But the store's on-disk format is toolchain - * internal, versioned `v5` with no third-party stability claim, and described - * conceptually rather than as a binary specification. Reading it means linking - * IndexStoreDB, so the producer must be a compiled Swift program. That is - * well-defined work nobody has started, and swift is not in the benchmark - * corpus, so nothing here can yet be measured either. + * The last entry leaves by being withdrawn, because its honest answer is not a + * program yet. Swift now has its compiled IndexStoreDB route in + * `provider/swift`; this placeholder list no longer owns it. * * **zig.** There is a channel, and it is stranger. ZLS has no batch mode, but * `zig build-obj -femit-docs` emits `sources.tar` beside a `main.wasm` that is diff --git a/packages/graph/src/provider/swift/ISwiftGraphSnapshot.ts b/packages/graph/src/provider/swift/ISwiftGraphSnapshot.ts new file mode 100644 index 00000000..43debe72 --- /dev/null +++ b/packages/graph/src/provider/swift/ISwiftGraphSnapshot.ts @@ -0,0 +1,124 @@ +/** Aggregate artifact committed by the SwiftPM/IndexStoreDB producer. */ +export interface ISwiftGraphSnapshot { + schemaVersion: number; + projectRoot: string; + producer: ISwiftGraphSnapshot.IProducer; + targets: ISwiftGraphSnapshot.ITarget[]; +} + +export namespace ISwiftGraphSnapshot { + export interface IProducer { + name: string; + version: string; + protocolVersion: number; + capabilities: ICapabilities; + } + + export interface ICapabilities { + atomicGenerations: boolean; + incremental: boolean; + diagnostics: boolean; + explicitOutputUnits: boolean; + indexStoreDB: boolean; + sourceEnrichment: boolean; + swiftpm: boolean; + sourceKitResident: boolean; + } + + /** One Swift module, build triple and configuration universe. */ + export interface ITarget { + name: string; + generation: string; + universe: string; + moduleName: string; + targetTriple: string; + sdk: string; + configuration: string; + swiftLanguageVersion: string; + compilerFlagsDigest: string; + moduleDependenciesDigest: string; + packageResolutionDigest: string; + pluginsDigest: string; + generatedSourcesDigest: string; + indexStoreDBCommit: string; + outputUnits: IOutputUnit[]; + coverage: Record; + shards: IShard[]; + } + + /** Exact compiler output admitted to the explicit IndexStoreDB view. */ + export interface IOutputUnit { + path: string; + digest: string; + } + + /** One source queried from the frozen store and enriched exactly once. */ + export interface IShard { + schemaVersion: number; + language: string; + source: string; + checkerDigest: string; + diskDigest: string; + target: string; + compilerVersion: string; + moduleName: string; + targetTriple: string; + sourceEnrichmentPasses: number; + nodes: INode[]; + edges: IEdge[]; + unresolved: IUnresolved[]; + diagnostics: IDiagnostic[]; + } + + export interface IEvidence { + file: string; + startLine: number; + startColumn: number; + endLine: number; + endColumn: number; + } + + export interface INode { + /** Swift/Clang USR from the compiler index, including local USRs. */ + symbol: string; + kind: string; + name: string; + qualifiedName: string; + file: string; + exported: boolean; + modifiers: string[]; + signature: string; + origin: string; + evidence: IEvidence; + } + + export interface IEdge { + from: string; + to: string; + kind: string; + access: string | null; + provenance: string | null; + targetKind: string | null; + targetName: string | null; + targetQualifiedName: string | null; + evidence: IEvidence; + } + + export interface IUnresolved { + family: string; + reason: string; + evidence: IEvidence; + candidates: string[]; + } + + export interface IDiagnostic { + severity: string; + message: string; + evidence: IEvidence; + } + + export const SCHEMA_VERSION = 1; + export const PROTOCOL_VERSION = 1; + export const INDEX_STORE_DB_COMMIT = + "54212fce1aecb199070808bdb265e7f17e396015"; +} diff --git a/packages/graph/src/provider/swift/SWIFT_GRAPH_FACTS.ts b/packages/graph/src/provider/swift/SWIFT_GRAPH_FACTS.ts new file mode 100644 index 00000000..3dfdafc4 --- /dev/null +++ b/packages/graph/src/provider/swift/SWIFT_GRAPH_FACTS.ts @@ -0,0 +1,5 @@ +import { GRAPH_EDGE_KINDS, GraphEdgeKind } from "../../typings"; + +/** Families IndexStoreDB plus one source enrichment pass can publish. */ +export const SWIFT_GRAPH_FACTS: readonly GraphEdgeKind[] = + GRAPH_EDGE_KINDS.filter((kind) => kind !== "renders"); diff --git a/packages/graph/src/provider/swift/SWIFT_GRAPH_PRODUCER.ts b/packages/graph/src/provider/swift/SWIFT_GRAPH_PRODUCER.ts new file mode 100644 index 00000000..193b1049 --- /dev/null +++ b/packages/graph/src/provider/swift/SWIFT_GRAPH_PRODUCER.ts @@ -0,0 +1,2 @@ +/** Executable name of the shipped SwiftPM/IndexStoreDB graph producer. */ +export const SWIFT_GRAPH_PRODUCER = "samchon-swift-graph"; diff --git a/packages/graph/src/provider/swift/SWIFT_GRAPH_PROVIDER.ts b/packages/graph/src/provider/swift/SWIFT_GRAPH_PROVIDER.ts new file mode 100644 index 00000000..70817350 --- /dev/null +++ b/packages/graph/src/provider/swift/SWIFT_GRAPH_PROVIDER.ts @@ -0,0 +1,2 @@ +/** Provider identity published by the strict Swift route. */ +export const SWIFT_GRAPH_PROVIDER = "swift-indexstore"; diff --git a/packages/graph/src/provider/swift/SwiftGraphSession.ts b/packages/graph/src/provider/swift/SwiftGraphSession.ts new file mode 100644 index 00000000..770b5a7e --- /dev/null +++ b/packages/graph/src/provider/swift/SwiftGraphSession.ts @@ -0,0 +1,59 @@ +import { GraphLanguage } from "../../typings"; +import { BatchGraphSession } from "../BatchGraphSession"; +import { IBulkGraphSession } from "../IBulkGraphSession"; +import { IGraphProvider } from "../IGraphProvider"; +import { CompilerGraphSession } from "../compiler/CompilerGraphSession"; +import { SwiftGraphSnapshotAdapter } from "./SwiftGraphSnapshotAdapter"; + +/** Resident sidecar process over repeated native SwiftPM index generations. */ +export class SwiftGraphSession implements IBulkGraphSession { + public readonly kind = "bulk" as const; + public readonly ownsProviderTopology = true; + public readonly languages: readonly GraphLanguage[]; + public readonly root: string; + + private readonly session: CompilerGraphSession; + + public constructor(options: SwiftGraphSession.IOptions) { + this.session = new CompilerGraphSession({ + ...options, + adapter: new SwiftGraphSnapshotAdapter(options.root), + serverCommand: "graph-server", + label: "Swift graph", + artifactName: "swift-graph.json", + }); + this.languages = this.session.languages; + this.root = this.session.root; + } + + public get generation(): number { + return this.session.generation; + } + + public get current(): IBulkGraphSession.ISnapshot | undefined { + return this.session.current; + } + + public refresh( + options: { signal?: AbortSignal } = {}, + ): Promise { + return this.session.refresh(options); + } + + public close(): Promise { + return this.session.close(); + } +} + +export namespace SwiftGraphSession { + export interface IOptions { + root: string; + languages: readonly GraphLanguage[]; + provider: string; + command: IGraphProvider.ICommand; + inputs: () => string[]; + configuration: NonNullable; + validate: (snapshot: IBulkGraphSession.ISnapshot) => void; + maxArtifactBytes?: number; + } +} diff --git a/packages/graph/src/provider/swift/SwiftGraphSnapshotAdapter.ts b/packages/graph/src/provider/swift/SwiftGraphSnapshotAdapter.ts new file mode 100644 index 00000000..62586ea0 --- /dev/null +++ b/packages/graph/src/provider/swift/SwiftGraphSnapshotAdapter.ts @@ -0,0 +1,132 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { CompilerGraphSnapshotAdapter } from "../compiler/CompilerGraphSnapshotAdapter"; +import { ISwiftGraphSnapshot } from "./ISwiftGraphSnapshot"; +import { SWIFT_GRAPH_FACTS } from "./SWIFT_GRAPH_FACTS"; +import { SWIFT_GRAPH_PRODUCER } from "./SWIFT_GRAPH_PRODUCER"; +import { SWIFT_GRAPH_PROVIDER } from "./SWIFT_GRAPH_PROVIDER"; + +const SHA256 = /^[0-9a-f]{64}$/u; + +/** Validate and adapt one explicit set of Swift compiler output units. */ +export class SwiftGraphSnapshotAdapter extends CompilerGraphSnapshotAdapter { + public constructor(root: string) { + super(root, { + label: "Swift graph", + language: "swift", + provider: SWIFT_GRAPH_PROVIDER, + producer: SWIFT_GRAPH_PRODUCER, + facts: SWIFT_GRAPH_FACTS, + capabilities: [ + "explicitOutputUnits", + "indexStoreDB", + "sourceEnrichment", + "swiftpm", + ], + diagnosticCode: "swiftc", + shardKeyPrefix: "swift", + schemaVersion: ISwiftGraphSnapshot.SCHEMA_VERSION, + protocolVersion: ISwiftGraphSnapshot.PROTOCOL_VERSION, + identitySalt: (rawTarget) => { + const target = rawTarget as unknown as ISwiftGraphSnapshot.ITarget; + return `${target.moduleName}\0${target.targetTriple}\0${target.configuration}`; + }, + validateSnapshot: (raw) => { + const snapshot = raw as unknown as ISwiftGraphSnapshot; + const capabilities = snapshot.producer.capabilities; + if ( + capabilities.explicitOutputUnits !== true || + capabilities.indexStoreDB !== true || + capabilities.sourceEnrichment !== true || + capabilities.swiftpm !== true || + capabilities.sourceKitResident !== false + ) { + throw new Error( + "Swift graph: the standalone producer must prove explicit IndexStoreDB output units, one source enrichment lane and SwiftPM ownership without claiming SourceKit residency", + ); + } + }, + validateTarget: (raw) => { + const target = raw as unknown as ISwiftGraphSnapshot.ITarget; + if ( + target.moduleName === "" || + target.targetTriple === "" || + target.configuration === "" || + target.swiftLanguageVersion === "" || + target.name !== targetIdentity(target) || + target.indexStoreDBCommit !== + ISwiftGraphSnapshot.INDEX_STORE_DB_COMMIT || + ![ + target.compilerFlagsDigest, + target.moduleDependenciesDigest, + target.packageResolutionDigest, + target.pluginsDigest, + target.generatedSourcesDigest, + ].every((value) => SHA256.test(value)) || + !outputUnits(target.outputUnits, root) + ) { + throw new Error(`Swift graph: malformed build target ${target.name}`); + } + }, + validateShard: (raw, rawTarget) => { + const shard = raw as unknown as ISwiftGraphSnapshot.IShard; + const target = rawTarget as unknown as ISwiftGraphSnapshot.ITarget; + if ( + shard.moduleName !== target.moduleName || + shard.targetTriple !== target.targetTriple || + shard.sourceEnrichmentPasses !== 1 + ) { + throw new Error( + `Swift graph: malformed source enrichment in ${shard.source}`, + ); + } + }, + }); + } +} + +function targetIdentity(target: ISwiftGraphSnapshot.ITarget): string { + return `${target.moduleName}@${target.targetTriple}/${target.configuration}`; +} + +function outputUnits( + units: readonly ISwiftGraphSnapshot.IOutputUnit[], + root: string, +): boolean { + if (units.length === 0) return false; + let previous: string | undefined; + for (const unit of units) { + if ( + unit.path === "" || + path.isAbsolute(unit.path) || + !SHA256.test(unit.digest) || + (previous !== undefined && previous >= unit.path) + ) { + return false; + } + const resolved = path.resolve(root, unit.path); + if (!confined(root, resolved)) return false; + let bytes: Buffer; + try { + bytes = fs.readFileSync(resolved); + } catch { + return false; + } + if (createHash("sha256").update(bytes).digest("hex") !== unit.digest) { + return false; + } + previous = unit.path; + } + return true; +} + +function confined(root: string, candidate: string): boolean { + const relative = path.relative(path.resolve(root), candidate); + return ( + relative !== "" && + relative !== ".." && + !relative.startsWith(`..${path.sep}`) + ); +} diff --git a/packages/graph/src/provider/swift/index.ts b/packages/graph/src/provider/swift/index.ts new file mode 100644 index 00000000..480a62c7 --- /dev/null +++ b/packages/graph/src/provider/swift/index.ts @@ -0,0 +1,8 @@ +export * from "./ISwiftGraphSnapshot"; +export * from "./resolveSwiftGraphCommand"; +export * from "./swiftGraphProvider"; +export * from "./SwiftGraphSession"; +export * from "./SwiftGraphSnapshotAdapter"; +export * from "./SWIFT_GRAPH_FACTS"; +export * from "./SWIFT_GRAPH_PRODUCER"; +export * from "./SWIFT_GRAPH_PROVIDER"; diff --git a/packages/graph/src/provider/swift/resolveSwiftGraphCommand.ts b/packages/graph/src/provider/swift/resolveSwiftGraphCommand.ts new file mode 100644 index 00000000..1ca6d4c3 --- /dev/null +++ b/packages/graph/src/provider/swift/resolveSwiftGraphCommand.ts @@ -0,0 +1,80 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { spawnableCommand } from "../../utils/spawnableCommand"; +import { IGraphProvider } from "../IGraphProvider"; +import { resolveProviderCommand } from "../resolveProviderCommand"; +import { SWIFT_GRAPH_PRODUCER } from "./SWIFT_GRAPH_PRODUCER"; + +const OVERRIDE = "SAMCHON_GRAPH_SWIFT_GRAPH"; +const SERVER_CAPABILITY = + "Serve explicit-output-unit SwiftPM IndexStoreDB generations over NDJSON."; + +/** Resolve a compatible standalone producer on one explicitly supported host. */ +export function resolveSwiftGraphCommand( + root: string, + env: NodeJS.ProcessEnv, + platform: NodeJS.Platform = process.platform, +): IGraphProvider.ICommand | undefined { + if ( + !["darwin", "linux"].includes(platform) || + !fs.existsSync(path.join(root, "Package.swift")) + ) { + return undefined; + } + const command = resolveProviderCommand(root, env, { + command: SWIFT_GRAPH_PRODUCER, + override: OVERRIDE, + }); + return command !== undefined && + publishesResidentServer(root, env, command) && + supportsProject(root, env, command) + ? command + : undefined; +} + +function publishesResidentServer( + root: string, + env: NodeJS.ProcessEnv, + command: IGraphProvider.ICommand, +): boolean { + const invocation = spawnableCommand.append( + { ...command, args: [...command.args] }, + ["graph-server", "--help"], + ); + const result = spawnSync(invocation.command, invocation.args, { + cwd: root, + encoding: "utf8", + env, + shell: false, + timeout: 30_000, + windowsHide: true, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + return `${result.stdout}${result.stderr}`.includes( + SERVER_CAPABILITY, + ); +} + +function supportsProject( + root: string, + env: NodeJS.ProcessEnv, + command: IGraphProvider.ICommand, +): boolean { + const invocation = spawnableCommand.append( + { ...command, args: [...command.args] }, + ["supports", "--cwd", root], + ); + return ( + spawnSync(invocation.command, invocation.args, { + cwd: root, + encoding: "utf8", + env, + shell: false, + timeout: 120_000, + windowsHide: true, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }).status === 0 + ); +} diff --git a/packages/graph/src/provider/swift/swiftGraphProvider.ts b/packages/graph/src/provider/swift/swiftGraphProvider.ts new file mode 100644 index 00000000..37d2972c --- /dev/null +++ b/packages/graph/src/provider/swift/swiftGraphProvider.ts @@ -0,0 +1,90 @@ +import { languageBuildInputs } from "../../indexer/languageBuildInputs"; +import { assertGraphSnapshotContract } from "../assertGraphSnapshotContract"; +import { IGraphProvider } from "../IGraphProvider"; +import { providerInputFiles } from "../providerInputFiles"; +import { toolchainVersion } from "../toolchainVersion"; +import { ISwiftGraphSnapshot } from "./ISwiftGraphSnapshot"; +import { resolveSwiftGraphCommand } from "./resolveSwiftGraphCommand"; +import { SWIFT_GRAPH_FACTS } from "./SWIFT_GRAPH_FACTS"; +import { SWIFT_GRAPH_PRODUCER } from "./SWIFT_GRAPH_PRODUCER"; +import { SWIFT_GRAPH_PROVIDER } from "./SWIFT_GRAPH_PROVIDER"; +import { SwiftGraphSession } from "./SwiftGraphSession"; + +const OVERRIDE = "SAMCHON_GRAPH_SWIFT_GRAPH"; +const TOOLCHAIN_OVERRIDE = "SAMCHON_GRAPH_SWIFT_TOOLCHAIN"; + +const swiftInputs = (root: string): string[] => [ + ...new Set([ + ...providerInputFiles(root, ["swift"], []), + ...languageBuildInputs(root, ["swift"]), + ]), +]; + +/** Standalone SwiftPM fallback over a frozen explicit IndexStoreDB unit set. */ +export const swiftGraphProvider: IGraphProvider = { + name: SWIFT_GRAPH_PROVIDER, + languages: ["swift"], + authority: "compiler", + facts: SWIFT_GRAPH_FACTS, + resolution: { + commands: [SWIFT_GRAPH_PRODUCER, "swift"], + environmentOverrides: [OVERRIDE, TOOLCHAIN_OVERRIDE], + }, + buildInputs: (root) => languageBuildInputs(root, ["swift"]), + configuration: (root, env) => [...swiftToolchain(root, env).rows], + configurationDerivation: (root, env) => swiftToolchain(root, env), + refuse: (options) => { + const refused = [ + options.server === undefined ? undefined : "server", + options.maxFiles === undefined ? undefined : "maxFiles", + options.lspReferenceLimit === undefined + ? undefined + : "lspReferenceLimit", + ].filter((value): value is string => value !== undefined); + return refused.length === 0 + ? undefined + : `swift: ${SWIFT_GRAPH_PROVIDER} publishes complete SwiftPM module generations and cannot honor ${refused.join(", ")}`; + }, + resolve: (root, env) => resolveSwiftGraphCommand(root, env), + open: (props) => + new SwiftGraphSession({ + root: props.root, + languages: props.languages, + provider: SWIFT_GRAPH_PROVIDER, + command: props.command, + inputs: () => swiftInputs(props.root), + configuration: () => swiftToolchain(props.root, process.env), + validate: (snapshot) => + assertGraphSnapshotContract( + snapshot, + swiftGraphProvider, + props.languages, + props.root, + ), + }), +}; + +function swiftToolchain( + root: string, + env: NodeJS.ProcessEnv, +): toolchainVersion.IDerivation { + return toolchainVersion.derive([ + toolchainVersion.observe({ + root, + env, + command: "swift", + override: TOOLCHAIN_OVERRIDE, + args: ["--version"], + label: "swift", + }), + toolchainVersion.observe({ + root, + env, + command: SWIFT_GRAPH_PRODUCER, + override: OVERRIDE, + args: ["--version"], + label: SWIFT_GRAPH_PRODUCER, + }), + `indexstore-db=${ISwiftGraphSnapshot.INDEX_STORE_DB_COMMIT}`, + ]); +} diff --git a/packages/graph/src/repository/cargoRepositoryContextProvider.ts b/packages/graph/src/repository/cargoRepositoryContextProvider.ts index c4bed4d4..33da6201 100644 --- a/packages/graph/src/repository/cargoRepositoryContextProvider.ts +++ b/packages/graph/src/repository/cargoRepositoryContextProvider.ts @@ -3,10 +3,12 @@ import fs from "node:fs"; import path from "node:path"; import { ISamchonRepositoryContextDump } from "../structures"; -import { spawnableCommand } from "../utils/spawnableCommand"; +import { isSubPath } from "../utils/isSubPath"; import { IRepositoryContextProvider } from "./IRepositoryContextProvider"; import { createRepositoryContextSession } from "./createRepositoryContextSession"; import { repositoryContextFacts } from "./repositoryContextFacts"; +import { resolveCargoCommand } from "./resolveCargoCommand"; +import { workspaceDiscoveryDirectories } from "./workspaceDiscoveryDirectories"; const { compareRepositoryText, @@ -207,10 +209,12 @@ function collectCargoRepositoryContext( files: [...files].sort(compareRepositoryText), sources: uniqueRepositorySources([ ...sources, - ...metadata.packages - .filter((pkg) => members.has(pkg.id)) - .map((pkg) => path.dirname(path.dirname(pkg.manifest_path))) - .map((directory) => repositoryContextSource(props.root, directory)), + ...workspaceDiscoveryDirectories( + metadata.workspace_root, + metadata.packages + .filter((pkg) => members.has(pkg.id)) + .map((pkg) => path.dirname(pkg.manifest_path)), + ).map((directory) => repositoryContextSource(props.root, directory)), ...["Cargo.lock", "rust-toolchain", "rust-toolchain.toml"] .map((file) => path.join(props.root, file)) .filter((file) => fs.existsSync(file)) @@ -276,7 +280,7 @@ function appendCargoTargets( ecosystem: ECOSYSTEM, coordinate: targetCoordinate, configuration, - external: !isInside(root, target.src_path), + external: !isSubPath(root, target.src_path), file, evidence, }, @@ -288,7 +292,7 @@ function appendCargoTargets( ecosystem: ECOSYSTEM, coordinate: targetCoordinate, configuration, - external: !isInside(root, target.src_path), + external: !isSubPath(root, target.src_path), file, evidence, }, @@ -347,7 +351,7 @@ function appendCargoTargets( ecosystem: ECOSYSTEM, coordinate: targetCoordinate, configuration, - external: !isInside(root, target.src_path), + external: !isSubPath(root, target.src_path), file, evidence, }); @@ -384,13 +388,16 @@ function executeCargoMetadata( root: string, env: NodeJS.ProcessEnv, ): ICargoMetadata { - /* c8 ignore next -- each coverage host has exactly one native shim suffix. */ - const command = process.platform === "win32" ? "cargo.cmd" : "cargo"; - const invocation = spawnableCommand( - command, - ["metadata", "--format-version", "1", "--locked", "--offline"], + const invocation = resolveCargoCommand( + root, env, + ["metadata", "--format-version", "1", "--locked", "--offline"], ); + if (invocation === undefined) { + throw new Error( + "cargo metadata failed without changing the project: cargo was not found", + ); + } const result = spawnSync(invocation.command, invocation.args, { cwd: root, env, @@ -420,9 +427,8 @@ function executeCargoMetadata( } function cargoVersion(root: string, env: NodeJS.ProcessEnv): string { - /* c8 ignore next -- each coverage host has exactly one native shim suffix. */ - const command = process.platform === "win32" ? "cargo.cmd" : "cargo"; - const invocation = spawnableCommand(command, ["--version"], env); + const invocation = resolveCargoCommand(root, env, ["--version"]); + if (invocation === undefined) return ""; const result = spawnSync(invocation.command, invocation.args, { cwd: root, env, @@ -448,11 +454,6 @@ function dedupeEdges( ); } -function isInside(root: string, file: string): boolean { - const relative = path.relative(root, file); - return relative !== ".." && !relative.startsWith(`..${path.sep}`); -} - function throwIfAborted(signal: AbortSignal | undefined): void { if (signal?.aborted) { throw new Error("cargo repository context cancelled"); diff --git a/packages/graph/src/repository/cmakeRepositoryContextProvider.ts b/packages/graph/src/repository/cmakeRepositoryContextProvider.ts index 378338a6..b3a63d14 100644 --- a/packages/graph/src/repository/cmakeRepositoryContextProvider.ts +++ b/packages/graph/src/repository/cmakeRepositoryContextProvider.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { ISamchonRepositoryContextDump } from "../structures"; +import { isSubPath } from "../utils/isSubPath"; import { IRepositoryContextProvider } from "./IRepositoryContextProvider"; import { createRepositoryContextSession } from "./createRepositoryContextSession"; import { repositoryContextFacts } from "./repositoryContextFacts"; @@ -375,7 +376,7 @@ function cmakeConfigurationShard( ecosystem: ECOSYSTEM, coordinate, configuration: target, - external: !isInside(root, artifactPath), + external: !isSubPath(root, artifactPath), evidence, }); edges.push( @@ -476,7 +477,7 @@ function appendCmakeSources( ecosystem: ECOSYSTEM, coordinate, configuration, - external: !isInside(root, directory), + external: !isSubPath(root, directory), evidence: repositoryContextEvidence( root, path.join(detail.paths.source, "CMakeLists.txt"), @@ -625,11 +626,6 @@ function dedupeEdges( ); } -function isInside(root: string, file: string): boolean { - const relative = path.relative(root, file); - return relative !== ".." && !relative.startsWith(`..${path.sep}`); -} - function throwIfAborted(signal: AbortSignal | undefined): void { if (signal?.aborted) { throw new Error("CMake repository context cancelled"); diff --git a/packages/graph/src/repository/createRepositoryContextSession.ts b/packages/graph/src/repository/createRepositoryContextSession.ts index cb5f304c..464c2ed6 100644 --- a/packages/graph/src/repository/createRepositoryContextSession.ts +++ b/packages/graph/src/repository/createRepositoryContextSession.ts @@ -4,6 +4,7 @@ import { IRepositoryContextProvider } from "./IRepositoryContextProvider"; import { IRepositoryContextSession } from "./IRepositoryContextSession"; import { RepositoryContextProtocol } from "./RepositoryContextProtocol"; import { repositoryContextFacts } from "./repositoryContextFacts"; +import { topologyPhaseTrace } from "./topologyPhaseTrace"; const { repositoryContextPathDigest } = repositoryContextFacts; @@ -55,6 +56,8 @@ export function createRepositoryContextSession( } const collected = await collect({ ...props, signal: options.signal }); + const tracing = props.env.SAMCHON_GRAPH_TOPOLOGY_TRACE === "1"; + const normalizationStarted = tracing ? performance.now() : 0; assertOpen(); throwIfAborted(options.signal); const sources = collected.shards.flatMap((shard) => shard.sources); @@ -158,6 +161,12 @@ export function createRepositoryContextSession( contentDigest: RepositoryContextProtocol.contentDigest(facts), }); const snapshot = store.apply(frames, options); + if (tracing) { + topologyPhaseTrace(provider.name, "normalization", normalizationStarted, { + nodes: facts.nodes.length, + edges: facts.edges.length, + }); + } generation = sequence; currentWarnings = [...collected.warnings]; inputState = createRepositoryContextSession.observeInputGeneration( diff --git a/packages/graph/src/repository/gradleRepositoryContextProvider.ts b/packages/graph/src/repository/gradleRepositoryContextProvider.ts index 514144a7..0ec5a670 100644 --- a/packages/graph/src/repository/gradleRepositoryContextProvider.ts +++ b/packages/graph/src/repository/gradleRepositoryContextProvider.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import { ISamchonRepositoryContextDump } from "../structures"; +import { isSubPath } from "../utils/isSubPath"; import { IRepositoryContextProvider } from "./IRepositoryContextProvider"; import { createRepositoryContextSession } from "./createRepositoryContextSession"; import { parseGradleRepositoryContextModel } from "./parseGradleRepositoryContextModel"; @@ -172,7 +173,7 @@ function collectGradleRepositoryContext( ecosystem: ECOSYSTEM, coordinate, configuration: source.kind, - external: !isInside(props.root, source.directory), + external: !isSubPath(props.root, source.directory), root: repositoryContextFile(props.root, source.directory), evidence, }); @@ -394,11 +395,6 @@ function dedupeEdges( ); } -function isInside(root: string, file: string): boolean { - const relative = path.relative(root, file); - return relative !== ".." && !relative.startsWith(`..${path.sep}`); -} - function throwIfAborted(signal: AbortSignal | undefined): void { if (signal?.aborted) { throw new Error("Gradle repository context cancelled"); diff --git a/packages/graph/src/repository/index.ts b/packages/graph/src/repository/index.ts index 2215b202..6053b0bb 100644 --- a/packages/graph/src/repository/index.ts +++ b/packages/graph/src/repository/index.ts @@ -9,6 +9,7 @@ export * from "./pnpmRepositoryContextProvider"; export * from "./REPOSITORY_CONTEXT_PROVIDERS"; export * from "./repositoryContextFacts"; export * from "./RepositoryContextProtocol"; +export * from "./resolveCargoCommand"; export * from "./createResidentRepositoryContextSource"; export * from "./createResidentRepositoryContextMemorySource"; export * from "./SamchonRepositoryContextMemory"; diff --git a/packages/graph/src/repository/pnpmRepositoryContextProvider.ts b/packages/graph/src/repository/pnpmRepositoryContextProvider.ts index 83d59053..0c43657b 100644 --- a/packages/graph/src/repository/pnpmRepositoryContextProvider.ts +++ b/packages/graph/src/repository/pnpmRepositoryContextProvider.ts @@ -3,10 +3,13 @@ import fs from "node:fs"; import path from "node:path"; import { ISamchonRepositoryContextDump } from "../structures"; +import { isSubPath } from "../utils/isSubPath"; import { spawnableCommand } from "../utils/spawnableCommand"; import { IRepositoryContextProvider } from "./IRepositoryContextProvider"; import { createRepositoryContextSession } from "./createRepositoryContextSession"; import { repositoryContextFacts } from "./repositoryContextFacts"; +import { topologyPhaseTrace } from "./topologyPhaseTrace"; +import { workspaceDiscoveryDirectories } from "./workspaceDiscoveryDirectories"; const { compareRepositoryText, @@ -139,7 +142,7 @@ function collectPnpmRepositoryContext( ecosystem: ECOSYSTEM, coordinate, configuration: "default", - external: false, + external: !isSubPath(props.root, absolute), evidence: repositoryContextEvidence(props.root, manifestFile), }); edges.push({ @@ -158,13 +161,11 @@ function collectPnpmRepositoryContext( files, ); } - for (const parent of new Set( - packages - .map((entry) => path.resolve(entry.path)) - .filter((directory) => directory !== path.resolve(props.root)) - .map((directory) => path.dirname(directory)), + for (const directory of workspaceDiscoveryDirectories( + props.root, + packages.map((entry) => path.resolve(entry.path)), )) { - sources.push(repositoryContextSource(props.root, parent)); + sources.push(repositoryContextSource(props.root, directory)); } for (const entry of packages) { @@ -235,9 +236,10 @@ function appendManifestFacts( path.join(packageRoot, "package.json"), ); for (const rootName of manifest.files ?? []) { - if (!isSimplePath(rootName)) continue; - const coordinate = `${repositoryContextFile(root, packageRoot)}/${rootName}`; - const generated = isGeneratedRoot(rootName); + const declared = declaredRoot(packageRoot, rootName); + if (declared === undefined) continue; + const coordinate = repositoryContextFile(root, declared.absolute); + const generated = declared.generated; const id = repositoryContextId( ECOSYSTEM, generated ? "generated-root" : "source-root", @@ -251,8 +253,8 @@ function appendManifestFacts( ecosystem: ECOSYSTEM, coordinate, configuration: "default", - external: false, - root: repositoryContextFile(root, path.resolve(packageRoot, rootName)), + external: !isSubPath(root, declared.absolute), + root: repositoryContextFile(root, declared.absolute), evidence, }); edges.push( @@ -267,7 +269,8 @@ function appendManifestFacts( "entrypoint", coordinate, ); - const file = repositoryContextFile(root, path.resolve(packageRoot, target)); + const absoluteTarget = path.resolve(packageRoot, target); + const file = repositoryContextFile(root, absoluteTarget); files.add(file); nodes.push({ id, @@ -277,7 +280,7 @@ function appendManifestFacts( ecosystem: ECOSYSTEM, coordinate, configuration: "default", - external: false, + external: !isSubPath(root, absoluteTarget), file, evidence, }); @@ -363,7 +366,39 @@ function dependencyRows(entry: IPnpmPackage): IPnpmDependency[] { } function readManifest(file: string): IPackageManifest { - return JSON.parse(fs.readFileSync(file, "utf8")) as IPackageManifest; + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(file, "utf8")); + } catch (error) { + malformedManifest(file, String(error)); + } + if (!isRecord(parsed)) malformedManifest(file, "root must be an object"); + for (const field of ["name", "main", "module", "types", "typings"] as const) { + if (parsed[field] !== undefined && typeof parsed[field] !== "string") { + malformedManifest(file, `${field} must be a string`); + } + } + if ( + parsed.files !== undefined && + (!Array.isArray(parsed.files) || + parsed.files.some((entry) => typeof entry !== "string")) + ) { + malformedManifest(file, "files must be a string array"); + } + if (!optionalStringRecord(parsed.scripts)) { + malformedManifest(file, "scripts must be a string record"); + } + if ( + parsed.bin !== undefined && + typeof parsed.bin !== "string" && + !stringRecord(parsed.bin) + ) { + malformedManifest(file, "bin must be a string or string record"); + } + if (!validExports(parsed.exports)) { + malformedManifest(file, "exports must contain only string, null, array, or object targets"); + } + return parsed as IPackageManifest; } function workspaceInputs(root: string): string[] { @@ -388,49 +423,57 @@ function executePnpm( root: string, env: NodeJS.ProcessEnv, ): IPnpmPackage[] { - /* c8 ignore next -- each coverage host has exactly one native shim suffix. */ - const command = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; - const invocation = spawnableCommand( - command, - ["list", "-r", "--json", "--depth", "0"], - env, - ); - const result = spawnSync(invocation.command, invocation.args, { - cwd: root, - env, - encoding: "utf8", - windowsHide: true, - windowsVerbatimArguments: invocation.windowsVerbatimArguments, - }); - if (result.status !== 0) { - /* c8 ignore start -- direct-spawn errors and silent nonzero exits are - * operating-system fallbacks; stderr failures are exercised here. */ - const failure = - result.stderr || result.error?.message || "unknown error"; - /* c8 ignore stop */ - throw new Error( - `pnpm repository context failed: ${failure.trim()}`, + const tracing = env.SAMCHON_GRAPH_TOPOLOGY_TRACE === "1"; + const started = tracing ? performance.now() : 0; + try { + /* c8 ignore next -- each coverage host has exactly one native shim suffix. */ + const command = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; + const invocation = spawnableCommand( + command, + ["list", "-r", "--json", "--depth", "0"], + env, ); + const result = spawnSync(invocation.command, invocation.args, { + cwd: root, + env, + encoding: "utf8", + windowsHide: true, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + if (result.status !== 0) { + /* c8 ignore start -- direct-spawn errors and silent nonzero exits are + * operating-system fallbacks; stderr failures are exercised here. */ + const failure = + result.stderr || result.error?.message || "unknown error"; + /* c8 ignore stop */ + throw new Error( + `pnpm repository context failed: ${failure.trim()}`, + ); + } + return parsePnpmPackages(result.stdout); + } finally { + if (tracing) topologyPhaseTrace(PROVIDER, "model-query", started); } - const parsed = JSON.parse(result.stdout) as IPnpmPackage[]; - if (!Array.isArray(parsed) || parsed.some((entry) => !entry.path)) { - throw new Error("pnpm repository context returned a malformed package list"); - } - return parsed; } function detectPnpmVersion(root: string, env: NodeJS.ProcessEnv): string { - /* c8 ignore next -- each coverage host has exactly one native shim suffix. */ - const command = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; - const invocation = spawnableCommand(command, ["--version"], env); - const result = spawnSync(invocation.command, invocation.args, { - cwd: root, - env, - encoding: "utf8", - windowsHide: true, - windowsVerbatimArguments: invocation.windowsVerbatimArguments, - }); - return result.status === 0 ? result.stdout.trim() : ""; + const tracing = env.SAMCHON_GRAPH_TOPOLOGY_TRACE === "1"; + const started = tracing ? performance.now() : 0; + try { + /* c8 ignore next -- each coverage host has exactly one native shim suffix. */ + const command = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; + const invocation = spawnableCommand(command, ["--version"], env); + const result = spawnSync(invocation.command, invocation.args, { + cwd: root, + env, + encoding: "utf8", + windowsHide: true, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + return result.status === 0 ? result.stdout.trim() : ""; + } finally { + if (tracing) topologyPhaseTrace(PROVIDER, "tool-startup", started); + } } function dedupeEdges( @@ -453,10 +496,98 @@ function isSimplePath(value: string): boolean { value.trim() !== "" && !value.includes("*") && !value.startsWith("!") && - !path.isAbsolute(value) + !path.isAbsolute(value) && + path.win32.parse(value).root === "" && + !value.replaceAll("\\", "/").split("/").includes("..") ); } +function declaredRoot( + packageRoot: string, + value: string, +): { absolute: string; generated: boolean } | undefined { + if (!isSimplePath(value)) return undefined; + const absolute = path.resolve(packageRoot, value); + const generated = isGeneratedRoot(value); + const stat = fs.statSync(absolute, { throwIfNoEntry: false }); + if (stat?.isDirectory() === true || (stat === undefined && generated)) { + return { absolute, generated }; + } + return undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stringRecord(value: unknown): value is Record { + return ( + isRecord(value) && + Object.values(value).every((entry) => typeof entry === "string") + ); +} + +function optionalStringRecord(value: unknown): boolean { + return value === undefined || stringRecord(value); +} + +function parsePnpmPackages(text: string): IPnpmPackage[] { + const parsed: unknown = JSON.parse(text); + if (!Array.isArray(parsed)) malformedPnpmModel("root must be an array"); + for (const [index, entry] of parsed.entries()) { + if (!isRecord(entry)) malformedPnpmModel(`[${index}] must be an object`); + if (typeof entry.path !== "string" || entry.path.trim() === "") { + malformedPnpmModel(`[${index}].path must be a nonempty string`); + } + for (const field of ["name", "version"] as const) { + if (entry[field] !== undefined && typeof entry[field] !== "string") { + malformedPnpmModel(`[${index}].${field} must be a string`); + } + } + if (entry.private !== undefined && typeof entry.private !== "boolean") { + malformedPnpmModel(`[${index}].private must be a boolean`); + } + for (const field of [ + "dependencies", + "devDependencies", + "optionalDependencies", + ] as const) { + const dependencies = entry[field]; + if (dependencies === undefined) continue; + if (!isRecord(dependencies)) { + malformedPnpmModel(`[${index}].${field} must be an object`); + } + for (const [name, dependency] of Object.entries(dependencies)) { + if ( + !isRecord(dependency) || + (dependency.path !== undefined && typeof dependency.path !== "string") + ) { + malformedPnpmModel( + `[${index}].${field}.${name} must be an object with an optional string path`, + ); + } + } + } + } + return parsed as IPnpmPackage[]; +} + +function malformedPnpmModel(reason: string): never { + throw new Error(`pnpm repository context returned a malformed package list: ${reason}`); +} + +function validExports(value: unknown): boolean { + if (value === undefined || value === null || typeof value === "string") { + return true; + } + if (Array.isArray(value)) return value.every(validExports); + return isRecord(value) && Object.values(value).every(validExports); +} + +function malformedManifest(file: string, reason: string): never { + throw new Error(`pnpm package manifest ${file} is malformed: ${reason}`); +} + function isGeneratedRoot(value: string): boolean { return /^(?:lib|dist|build|out)(?:\/|$)/.test(value.replaceAll("\\", "/")); } diff --git a/packages/graph/src/repository/resolveCargoCommand.ts b/packages/graph/src/repository/resolveCargoCommand.ts new file mode 100644 index 00000000..46e60c72 --- /dev/null +++ b/packages/graph/src/repository/resolveCargoCommand.ts @@ -0,0 +1,14 @@ +import { resolveProviderCommand } from "../provider/resolveProviderCommand"; + +/** Resolve rustup's native cargo.exe before optional command shims on Windows. */ +export function resolveCargoCommand( + root: string, + env: NodeJS.ProcessEnv, + args: readonly string[] = [], +) { + return resolveProviderCommand(root, env, { + command: "cargo", + override: "SAMCHON_GRAPH_CARGO", + args, + }); +} diff --git a/packages/graph/src/repository/topologyPhaseTrace.ts b/packages/graph/src/repository/topologyPhaseTrace.ts new file mode 100644 index 00000000..736d5ed0 --- /dev/null +++ b/packages/graph/src/repository/topologyPhaseTrace.ts @@ -0,0 +1,17 @@ +/** Emit one opt-in timing row for repository-orientation experiments. */ +export function topologyPhaseTrace( + provider: string, + phase: "tool-startup" | "model-query" | "normalization" | "join", + started: number, + details: Record = {}, +): void { + process.stderr.write( + `@samchon/graph: topology-phase=${JSON.stringify({ + schemaVersion: 1, + provider, + phase, + durationMs: Math.max(0, performance.now() - started), + ...details, + })}\n`, + ); +} diff --git a/packages/graph/src/repository/workspaceDiscoveryDirectories.ts b/packages/graph/src/repository/workspaceDiscoveryDirectories.ts new file mode 100644 index 00000000..5860ef37 --- /dev/null +++ b/packages/graph/src/repository/workspaceDiscoveryDirectories.ts @@ -0,0 +1,25 @@ +import path from "node:path"; + +import { isSubPath } from "../utils/isSubPath"; + +/** Directory identities whose immediate entries can reveal a new member. */ +export function workspaceDiscoveryDirectories( + workspaceRoot: string, + members: readonly string[], +): string[] { + const root = path.resolve(workspaceRoot); + const found = new Set([root]); + for (const member of members) { + let directory = path.dirname(path.resolve(member)); + while (isSubPath(root, directory)) { + found.add(directory); + if (isSubPath(directory, root)) break; + directory = path.dirname(directory); + } + } + return [...found].sort(compareText); +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : 1; +} diff --git a/packages/graph/src/routeSummary.ts b/packages/graph/src/routeSummary.ts new file mode 100644 index 00000000..7ce5a329 --- /dev/null +++ b/packages/graph/src/routeSummary.ts @@ -0,0 +1,30 @@ +import { ISamchonGraphDump } from "./structures"; + +/** Bounded machine-readable identity for the producer(s) that actually served. */ +export function routeSummary(dump: ISamchonGraphDump): string { + const summary = { + schemaVersion: 1, + indexer: dump.indexer, + provenance: (dump.provenance ?? []).map((row) => ({ + provider: row.provider, + languages: row.languages, + authority: row.authority, + producer: { + tool: row.producer.tool, + version: row.producer.version, + schemaVersion: row.producer.schemaVersion, + protocolVersion: row.producer.protocolVersion, + }, + })), + }; + const encoded = JSON.stringify(summary); + if (Buffer.byteLength(encoded, "utf8") <= ROUTE_SUMMARY_LIMIT) return encoded; + return JSON.stringify({ + schemaVersion: 1, + indexer: dump.indexer, + provenance: [], + truncated: true, + }); +} + +const ROUTE_SUMMARY_LIMIT = 16 * 1024; diff --git a/packages/graph/src/runGraph.ts b/packages/graph/src/runGraph.ts index e613d78e..361c4639 100644 --- a/packages/graph/src/runGraph.ts +++ b/packages/graph/src/runGraph.ts @@ -2,6 +2,7 @@ import packageJson from "../package.json"; import { buildGraphDump } from "./indexer/buildGraphDump"; import { startServer } from "./mcp/startServer"; import { parseGraphArgs } from "./parseGraphArgs"; +import { routeSummary } from "./routeSummary"; import { ISamchonGraphDump } from "./structures"; import { runView } from "./view"; @@ -112,7 +113,10 @@ function dumpSummary(dump: ISamchonGraphDump): string { const by = served.length === 0 ? "no strict provider served" : served.join(" "); /* c8 ignore stop */ - const lines = [`@samchon/graph: indexer=${dump.indexer} ${by}`]; + const lines = [ + `@samchon/graph: indexer=${dump.indexer} ${by}`, + `@samchon/graph: route=${routeSummary(dump)}`, + ]; /* c8 ignore start -- the field is optional in the dump contract and always * present in practice, so the empty-fallback arm guards a shape no producer * in this repository emits. */ diff --git a/packages/graph/src/utils/fileFromUri.ts b/packages/graph/src/utils/fileFromUri.ts index ba6c32c4..d2e16c02 100644 --- a/packages/graph/src/utils/fileFromUri.ts +++ b/packages/graph/src/utils/fileFromUri.ts @@ -1,7 +1,7 @@ import { fileURLToPath } from "node:url"; export function fileFromUri(uri: string): string { - if (!uri.startsWith("file://")) return uri; + if (uri.slice(0, 6).toLowerCase() !== "file:/") return uri; // An LSP owns the URI encoder on responses. Pyright percent-encodes reserved // path characters such as the drive colon and `@`, while other servers leave // some of them literal. `decodeURI` deliberately preserves reserved escapes, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cc56f174..16e0a34b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -151,6 +151,9 @@ importers: tests/experiment: dependencies: + '@modelcontextprotocol/sdk': + specifier: ^1.26.0 + version: 1.29.0(zod@4.4.3) '@samchon/graph': specifier: workspace:* version: link:../../packages/graph diff --git a/sidecars/csharp/GraphExtractor.cs b/sidecars/csharp/GraphExtractor.cs new file mode 100644 index 00000000..ce8e7dd6 --- /dev/null +++ b/sidecars/csharp/GraphExtractor.cs @@ -0,0 +1,2147 @@ +using System.Text; +using System.Text.Json.Nodes; +using System.Collections.Immutable; +using System.Collections.Concurrent; +using System.Diagnostics; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Operations; + +namespace Samchon.Graph.CSharp; + +internal static class GraphExtractor +{ + private static readonly string[] Families = + [ + "contains", "exports", "imports", "calls", "accesses", + "instantiates", "type_ref", "extends", "implements", "overrides", + "dispatches", "decorates", "renders", "tests", "references", + ]; + + public static async Task ExtractAsync( + Solution solution, + string root, + IReadOnlyList workspaceDiagnostics, + GraphDraft? previous, + IReadOnlySet changedFiles, + bool forceFull, + CancellationToken cancellationToken) + { + var timing = Stopwatch.StartNew(); + var priorElapsed = 0L; + void Trace(string phase) + { + if (Environment.GetEnvironmentVariable("SAMCHON_GRAPH_ROSLYN_TRACE") != "1") + { + return; + } + var elapsed = timing.ElapsedMilliseconds; + Console.Error.WriteLine( + $"{{\"phase\":\"roslyn-{phase}\",\"elapsedMs\":{elapsed - priorElapsed}}}"); + priorElapsed = elapsed; + } + + var topologicalOrder = solution.GetProjectDependencyGraph() + .GetTopologicallySortedProjects() + .Select((project, index) => (project, index)) + .ToDictionary(entry => entry.project, entry => entry.index); + var projects = solution.Projects + .Where(project => project.Language == LanguageNames.CSharp) + .OrderBy(project => topologicalOrder[project.Id]) + .ThenBy(project => project.FilePath ?? project.Name, StringComparer.Ordinal) + .ToArray(); + var previousContexts = previous?.ProviderState as IReadOnlyList; + var changed = changedFiles + .Select(Path.GetFullPath) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var directlyChangedProjects = projects + .Where(project => project.Documents.Any(document => + document.FilePath is { Length: > 0 } file + && changed.Contains(Path.GetFullPath(file))) + || previousContexts?.Any(context => + context.Project.Id == project.Id + && context.Project.Documents.Any(document => + document.FilePath is { Length: > 0 } file + && changed.Contains(Path.GetFullPath(file)))) == true) + .Select(project => project.Id) + .ToHashSet(); + // Roslyn's topological order does not define an order between independent + // projects, and their reload-specific ProjectIds may therefore reorder + // otherwise identical Solutions. Downstream order assigns the one shard + // that carries shared build sources, so seal it by semantic target. + var contexts = (await Task.WhenAll(projects.Select(async project => + { + var prior = previousContexts?.FirstOrDefault(context => + context.Project.Id == project.Id); + if (!forceFull + && prior is not null + && !directlyChangedProjects.Contains(project.Id)) + { + return new ProjectContext( + prior.Project, + prior.Compilation, + prior.Target, + prior.Assembly, + prior.Framework, + prior.ProjectFile, + [], + prior.Documents, + prior.GeneratedDocuments); + } + cancellationToken.ThrowIfCancellationRequested(); + var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Roslyn produced no compilation for {project.Name}"); + var framework = TargetFramework(compilation); + var assembly = compilation.Assembly.Identity.GetDisplayName(); + var projectFile = project.FilePath is null + ? $"bundled:///csharp/projects/{Safe(project.Name)}" + : Path.GetFullPath(project.FilePath); + var target = $"roslyn:{GraphProtocol.HashText($"{projectFile}\0{assembly}\0{framework}")}"; + return new ProjectContext( + project, + compilation, + target, + assembly, + framework, + projectFile, + [], + null, + forceFull ? null : prior?.GeneratedDocuments); + }))) + .OrderBy(context => context.Target, StringComparer.Ordinal) + .ToList(); + Trace("compilations"); + if (contexts.Count == 0) + { + throw new InvalidOperationException("Roslyn workspace contains no C# compilation"); + } + var catalog = new ProjectCatalog(contexts); + foreach (var context in contexts) + { + context.Catalog = catalog; + } + + var reuseBuildUniverse = !forceFull + && previous is not null + && changed.All(file => Path.GetExtension(file).Equals( + ".cs", + StringComparison.OrdinalIgnoreCase)) + && previous.Targets.Order(StringComparer.Ordinal).SequenceEqual( + contexts.Select(context => context.Target).Order(StringComparer.Ordinal)); + List buildSources; + JsonObject universe; + if (reuseBuildUniverse) + { + buildSources = []; + universe = previous!.Universe.DeepClone().AsObject(); + } + else + { + buildSources = BuildSources(root, contexts); + var universeRows = new JsonArray(); + foreach (var context in contexts) + { + universeRows.Add(new JsonObject + { + ["target"] = context.Target, + ["project"] = context.ProjectFile, + ["assembly"] = context.Assembly, + ["framework"] = context.Framework, + ["parseOptions"] = context.Project.ParseOptions?.ToString() ?? "", + ["compilationOptions"] = context.Project.CompilationOptions?.ToString() ?? "", + ["output"] = context.Project.OutputFilePath ?? "", + ["analyzers"] = new JsonArray(context.Project.AnalyzerReferences + .Select(reference => reference.FullPath ?? reference.Display ?? "") + .Order(StringComparer.Ordinal) + .Select(value => JsonValue.Create(value)) + .ToArray()), + ["projectReferences"] = new JsonArray(context.Project.ProjectReferences + .Select(reference => solution.GetProject(reference.ProjectId)?.FilePath ?? reference.ProjectId.Id.ToString()) + .Order(StringComparer.Ordinal) + .Select(value => JsonValue.Create(value)) + .ToArray()), + ["metadataReferences"] = new JsonArray(context.Project.MetadataReferences + .Select(reference => reference.Display ?? "") + .Order(StringComparer.Ordinal) + .Select(value => JsonValue.Create(value)) + .ToArray()), + }); + } + var inputRows = new JsonArray(buildSources + .OrderBy(source => source["file"]!.GetValue(), StringComparer.Ordinal) + .Select(source => source.DeepClone()) + .ToArray()); + universe = new JsonObject + { + ["workspaceRoot"] = Path.GetFullPath(root), + ["projects"] = universeRows, + ["inputs"] = inputRows, + }; + } + var universeDigest = GraphProtocol.Hash(universe); + + var shards = new List(); + var diagnosticsByFile = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var globalDiagnostics = contexts.ToDictionary( + context => context.Target, + _ => new List(), + StringComparer.Ordinal); + + var documentWork = new List(); + foreach (var context in contexts) + { + IReadOnlyList projectDocuments; + if (context.Documents is not null) + { + projectDocuments = context.Documents; + } + else + { + var documents = context.Project.Documents.Cast() + .Select(document => new ContextDocument( + document, + document.FilePath is { Length: > 0 } file + && Ignored(root, Path.GetFullPath(file)))) + .ToList(); + IReadOnlyList generatedDocuments; + if (context.GeneratedDocuments is not null + && await GeneratedOutputsMatch( + context, + context.GeneratedDocuments, + cancellationToken).ConfigureAwait(false)) + { + generatedDocuments = context.GeneratedDocuments; + } + else + { + generatedDocuments = (await context.Project + .GetSourceGeneratedDocumentsAsync(cancellationToken) + .ConfigureAwait(false)) + .Select(document => new ContextDocument(document, true)) + .ToArray(); + } + context.GeneratedDocuments = generatedDocuments; + documents.AddRange(generatedDocuments); + projectDocuments = documents + .OrderBy(entry => DocumentIdentity(entry.Document), StringComparer.Ordinal) + .ToArray(); + context.Documents = projectDocuments; + } + foreach (var entry in projectDocuments) + { + var document = entry.Document; + var generated = entry.Generated; + if (generated && document.FilePath is { Length: > 0 } generatedFile) + { + context.GeneratedSources[Path.GetFullPath(generatedFile)] = + GeneratedSourceIdentity(context, document); + } + documentWork.Add(new DocumentWork( + context, + document, + DocumentShardKey(context, document, root, generated), + generated)); + } + } + Trace("documents"); + + var previousShards = previous?.Shards + .ToDictionary(shard => shard.Key, StringComparer.Ordinal) + ?? new Dictionary(StringComparer.Ordinal); + var reuse = !forceFull + && previous is not null + && GraphProtocol.Hash(previous.Universe) == universeDigest; + var extracted = new Dictionary(StringComparer.Ordinal); + var signatureChanged = new HashSet(); + var diagnosticProjects = new HashSet(); + using var extractionGate = new SemaphoreSlim( + Math.Max(1, Math.Min(Environment.ProcessorCount, 8))); + async Task<(DocumentWork Work, ShardDraft? Prior, ShardDraft? Next)> ExtractChanged( + DocumentWork work) + { + previousShards.TryGetValue(work.Key, out var prior); + if (reuse + && prior is not null + && !changed.Contains(SourceIdentity( + work.Context, + work.Document, + work.Generated)) + && (!work.Generated + || await CheckerDigest(work.Document, cancellationToken).ConfigureAwait(false) + == SourceCheckerDigest(prior))) + { + return (work, prior, null); + } + ShardDraft next; + if (reuse) + { + await extractionGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + next = await Task.Run( + () => ExtractDocumentAsync( + work.Context, + work.Document, + root, + universeDigest, + diagnosticsByFile, + work.Generated, + cancellationToken), + cancellationToken).ConfigureAwait(false); + } + finally + { + extractionGate.Release(); + } + } + else + { + next = await ExtractDocumentAsync( + work.Context, + work.Document, + root, + universeDigest, + diagnosticsByFile, + work.Generated, + cancellationToken).ConfigureAwait(false); + } + return (work, prior, next); + } + IReadOnlyList<(DocumentWork Work, ShardDraft? Prior, ShardDraft? Next)> + extractionResults; + if (reuse) + { + extractionResults = await Task.WhenAll(documentWork.Select(ExtractChanged)) + .ConfigureAwait(false); + } + else + { + var initialResults = + new List<(DocumentWork Work, ShardDraft? Prior, ShardDraft? Next)>(); + foreach (var work in documentWork) + { + initialResults.Add(await ExtractChanged(work).ConfigureAwait(false)); + } + extractionResults = initialResults; + } + foreach (var result in extractionResults) + { + if (result.Next is null) + { + continue; + } + extracted[result.Work.Key] = result.Next; + if (result.Prior is null + || result.Prior.InterfaceFingerprint != result.Next.InterfaceFingerprint) + { + signatureChanged.Add(result.Work.Context.Project.Id); + } + } + Trace("extraction"); + if (reuse) + { + var currentKeys = documentWork + .Select(work => work.Key) + .ToHashSet(StringComparer.Ordinal); + foreach (var context in contexts) + { + var prefix = $"csharp-shard-v1|{context.Target}|document:"; + var removed = previousShards.Keys.FirstOrDefault(key => + key.StartsWith(prefix, StringComparison.Ordinal) + && !currentKeys.Contains(key)); + if (removed is not null) + { + signatureChanged.Add(context.Project.Id); + diagnosticProjects.Add(context.Project.Id); + } + } + } + if (reuse && signatureChanged.Count != 0) + { + return await ExtractAsync( + solution, + root, + workspaceDiagnostics, + previous, + changedFiles, + true, + cancellationToken).ConfigureAwait(false); + } + var affected = DependentClosure(solution, signatureChanged); + diagnosticProjects.UnionWith(affected); + if (!reuse) + { + diagnosticProjects.UnionWith(contexts.Select(context => context.Project.Id)); + } + var incrementalDocuments = documentWork + .Where(work => extracted.ContainsKey(work.Key) + && !diagnosticProjects.Contains(work.Context.Project.Id)) + .GroupBy(work => work.Context.Project.Id) + .ToDictionary( + group => group.Key, + group => (IReadOnlyList)group + .Select(work => work.Document) + .DistinctBy(document => document.Id) + .ToArray()); + await Task.WhenAll(contexts + .Where(context => diagnosticProjects.Contains(context.Project.Id) + || incrementalDocuments.ContainsKey(context.Project.Id)) + .Select(async context => + { + context.Diagnostics = diagnosticProjects.Contains(context.Project.Id) + ? await DiagnosticsOf( + context.Project, + context.Compilation, + cancellationToken).ConfigureAwait(false) + : await IncrementalDiagnosticsOf( + incrementalDocuments[context.Project.Id], + cancellationToken).ConfigureAwait(false); + })).ConfigureAwait(false); + foreach (var context in contexts + .Where(context => diagnosticProjects.Contains(context.Project.Id) + || incrementalDocuments.ContainsKey(context.Project.Id))) + { + foreach (var diagnostic in context.Diagnostics) + { + if (diagnostic.Location.IsInSource + && diagnostic.Location.SourceTree?.FilePath is { Length: > 0 } file) + { + var absolute = Path.GetFullPath(file); + var key = $"{context.Target}\0{absolute}"; + if (!diagnosticsByFile.TryGetValue(key, out var rows)) + { + rows = []; + diagnosticsByFile[key] = rows; + } + rows.Add(diagnostic); + } + else + { + globalDiagnostics[context.Target].Add(diagnostic); + } + } + } + Trace("diagnostics"); + foreach (var work in documentWork) + { + cancellationToken.ThrowIfCancellationRequested(); + if (extracted.TryGetValue(work.Key, out var shard)) + { + previousShards.TryGetValue(work.Key, out var prior); + shard = ReuseWithDiagnostics( + shard, + DiagnosticsFor( + work.Context, + work.Document, + root, + diagnosticsByFile, + diagnosticProjects.Contains(work.Context.Project.Id) + ? null + : prior?.Payload["diagnostics"]?.AsArray())); + } + else + { + if (!reuse + || affected.Contains(work.Context.Project.Id) + || !previousShards.TryGetValue(work.Key, out var prior)) + { + shard = await ExtractDocumentAsync( + work.Context, + work.Document, + root, + universeDigest, + diagnosticsByFile, + work.Generated, + cancellationToken).ConfigureAwait(false); + } + else + { + shard = diagnosticProjects.Contains(work.Context.Project.Id) + ? ReuseWithDiagnostics( + prior, + DiagnosticsFor( + work.Context, + work.Document, + root, + diagnosticsByFile)) + : prior; + } + } + shards.Add(shard); + } + + for (var index = 0; index < contexts.Count; index++) + { + var context = contexts[index]; + var metadataKey = $"csharp-shard-v1|{context.Target}|metadata"; + if (reuseBuildUniverse + && previousShards.TryGetValue(metadataKey, out var cachedMetadata)) + { + shards.Add(cachedMetadata); + continue; + } + var projectId = ProjectNodeId(context); + var projectFile = GraphFile(root, context.ProjectFile); + var evidence = new JsonObject + { + ["file"] = projectFile, + ["startLine"] = 1, + ["startCol"] = 1, + ["endLine"] = 1, + ["endCol"] = 1, + }; + var nodes = new JsonArray(new JsonObject + { + ["id"] = projectId, + ["kind"] = "package", + ["language"] = "csharp", + ["name"] = context.Project.Name, + ["qualifiedName"] = context.Assembly, + ["file"] = projectFile, + ["external"] = false, + ["exported"] = true, + ["evidence"] = evidence.DeepClone(), + }); + var projectEdges = new JsonArray(); + foreach (var reference in context.Project.ProjectReferences) + { + var referenced = contexts.FirstOrDefault(candidate => + candidate.Project.Id == reference.ProjectId); + if (referenced is null) + { + continue; + } + var referencedId = ProjectNodeId(referenced); + nodes.Add(new JsonObject + { + ["id"] = referencedId, + ["kind"] = "package", + ["language"] = "csharp", + ["name"] = referenced.Project.Name, + ["qualifiedName"] = referenced.Assembly, + ["file"] = GraphFile(root, referenced.ProjectFile), + ["external"] = false, + ["exported"] = true, + }); + projectEdges.Add(new JsonObject + { + ["from"] = projectId, + ["to"] = referencedId, + ["kind"] = "imports", + ["evidence"] = evidence.DeepClone(), + }); + } + var coverage = new JsonArray(); + var unresolved = new JsonArray(); + foreach (var family in Families) + { + var state = family == "renders" ? "unsupported" : "partial"; + coverage.Add(new JsonObject + { + ["provider"] = GraphProtocol.Provider, + ["language"] = "csharp", + ["target"] = context.Target, + ["family"] = family, + ["state"] = state, + }); + if (state == "partial") + { + unresolved.Add(new JsonObject + { + ["provider"] = GraphProtocol.Provider, + ["language"] = "csharp", + ["target"] = context.Target, + ["universe"] = universeDigest, + ["family"] = family, + ["evidence"] = evidence.DeepClone(), + ["reason"] = "provider-gap", + ["candidates"] = new JsonArray(), + }); + } + } + JsonArray diagnostics; + if (reuse + && !diagnosticProjects.Contains(context.Project.Id) + && previousShards.TryGetValue(metadataKey, out var priorMetadata)) + { + diagnostics = priorMetadata.Payload["diagnostics"]!.DeepClone().AsArray(); + } + else + { + diagnostics = new JsonArray(); + foreach (var diagnostic in globalDiagnostics[context.Target] + .Select(diagnostic => DiagnosticNode(context, root, diagnostic)) + .OrderBy(DiagnosticKey, StringComparer.Ordinal)) + { + diagnostics.Add(diagnostic); + } + foreach (var diagnostic in workspaceDiagnostics) + { + diagnostics.Add(new JsonObject + { + ["file"] = "", + ["line"] = 0, + ["column"] = 0, + ["code"] = "MSBUILD", + ["message"] = diagnostic, + ["severity"] = "warning", + }); + } + } + var sources = new JsonArray(); + sources.Add(SourceOf(context.ProjectFile)); + if (index == 0) + { + foreach (var source in buildSources) + { + if (source["file"]!.GetValue() != context.ProjectFile) + { + sources.Add(source.DeepClone()); + } + } + } + shards.Add(Shard( + metadataKey, + context.Target, + nodes, + projectEdges, + diagnostics, + coverage, + unresolved, + sources)); + } + + var errorDiagnostics = contexts + .SelectMany(context => context.Diagnostics) + .Where(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error) + .Take(3) + .Select(diagnostic => diagnostic.GetMessage()) + .ToArray(); + Trace("assembly"); + return new GraphDraft( + contexts.Select(context => context.Target).Order(StringComparer.Ordinal).ToArray(), + universe, + universeDigest, + shards, + errorDiagnostics.Length != 0, + string.Join("; ", errorDiagnostics), + contexts); + } + + private static async Task ExtractDocumentAsync( + ProjectContext context, + Document document, + string root, + string universe, + IReadOnlyDictionary> diagnosticsByFile, + bool generated, + CancellationToken cancellationToken) + { + var syntax = await document.GetSyntaxRootAsync(cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Roslyn produced no syntax tree for {document.Name}"); + var model = await document.GetSemanticModelAsync(cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Roslyn produced no semantic model for {document.Name}"); + var text = await document.GetTextAsync(cancellationToken).ConfigureAwait(false); + var sourceFile = SourceIdentity(context, document, generated); + var graphFile = GraphFile(root, sourceFile); + var fileId = SemanticNodeId( + $"file:{context.Target}:{graphFile}", + "file", + Path.GetFileName(graphFile), + context.Target); + var projectId = ProjectNodeId(context); + var nodes = new Dictionary(StringComparer.Ordinal); + var edges = new Dictionary(StringComparer.Ordinal); + var unresolved = new JsonArray(); + var declarationFingerprints = new List(); + nodes[fileId] = new JsonObject + { + ["id"] = fileId, + ["kind"] = "file", + ["language"] = "csharp", + ["name"] = Path.GetFileName(graphFile), + ["file"] = graphFile, + ["external"] = false, + }; + AddEdge(edges, projectId, fileId, "contains", null); + + var allSyntax = syntax.DescendantNodesAndSelf().ToArray(); + foreach (var declaration in allSyntax) + { + cancellationToken.ThrowIfCancellationRequested(); + var symbol = DeclaredSymbol(model, declaration, cancellationToken); + if (symbol is null || symbol is IAliasSymbol || symbol.Name == "") + { + continue; + } + declarationFingerprints.Add(DeclarationFingerprint(symbol)); + var node = NodeForSymbol(context, symbol, root); + nodes.TryAdd(node["id"]!.GetValue(), node); + var owner = OwnerSymbol(symbol); + var ownerId = owner is null || owner is INamespaceSymbol { IsGlobalNamespace: true } + ? fileId + : EnsureNode(nodes, context, owner, root); + var nodeId = node["id"]!.GetValue(); + AddEdge(edges, ownerId, nodeId, "contains", Evidence(context, root, declaration.GetLocation())); + if (Exported(symbol)) + { + AddEdge(edges, owner is null ? projectId : ownerId, nodeId, "exports", Evidence(context, root, declaration.GetLocation())); + } + if (symbol is INamedTypeSymbol type) + { + AddTypeRelations(nodes, edges, context, type, nodeId, root, declaration.GetLocation()); + AddSynthesizedRecordMembers(nodes, edges, context, type, nodeId, root); + } + AddOverride(nodes, edges, context, symbol, nodeId, root, declaration.GetLocation()); + } + + foreach (var item in allSyntax) + { + cancellationToken.ThrowIfCancellationRequested(); + switch (item) + { + case UsingDirectiveSyntax usingDirective: + { + var target = usingDirective.Alias is null + ? model.GetSymbolInfo(usingDirective.Name!, cancellationToken).Symbol + : model.GetSymbolInfo(usingDirective.Name!, cancellationToken).Symbol; + if (target is not null) + { + AddEdge(edges, fileId, EnsureNode(nodes, context, target, root), "imports", Evidence(context, root, usingDirective.GetLocation())); + } + break; + } + case InvocationExpressionSyntax invocation + when model.GetOperation(invocation, cancellationToken) is IInvocationOperation operation: + { + var owner = model.GetEnclosingSymbol(invocation.SpanStart, cancellationToken); + if (owner is null) + { + break; + } + var from = EnsureNode(nodes, context, owner, root); + var target = CanonicalMethod(operation.TargetMethod); + var to = EnsureNode(nodes, context, target, root); + AddEdge(edges, from, to, "calls", Evidence(context, root, invocation.GetLocation())); + AddDispatchCandidates( + nodes, + unresolved, + context, + target, + invocation, + root, + universe); + if (IsTest(owner)) + { + AddEdge(edges, from, to, "tests", Evidence(context, root, invocation.GetLocation())); + } + break; + } + case ObjectCreationExpressionSyntax creation + when model.GetOperation(creation, cancellationToken) is IObjectCreationOperation operation: + AddSemanticEdge(nodes, edges, context, model, operation.Type, creation, root, "instantiates", cancellationToken); + break; + case ImplicitObjectCreationExpressionSyntax creation + when model.GetOperation(creation, cancellationToken) is IObjectCreationOperation operation: + AddSemanticEdge(nodes, edges, context, model, operation.Type, creation, root, "instantiates", cancellationToken); + break; + case AttributeSyntax attribute: + { + var owner = AttributeOwner(model, attribute, cancellationToken); + var constructor = model.GetSymbolInfo(attribute, cancellationToken).Symbol as IMethodSymbol; + if (owner is not null && constructor is not null) + { + var from = EnsureNode(nodes, context, owner, root); + var type = EnsureNode(nodes, context, constructor.ContainingType, root); + AddEdge( + edges, + from, + type, + "decorates", + Evidence(context, root, attribute.GetLocation())); + AddEdge( + edges, + from, + EnsureNode(nodes, context, constructor, root), + "references", + Evidence(context, root, attribute.GetLocation())); + AddEdge( + edges, + from, + type, + "type_ref", + Evidence(context, root, attribute.GetLocation())); + } + break; + } + case AnonymousFunctionExpressionSyntax anonymous + when model.GetOperation(anonymous, cancellationToken) + is IAnonymousFunctionOperation operation: + { + var lambda = EnsureNode(nodes, context, operation.Symbol, root); + var owner = operation.Symbol.ContainingSymbol; + if (owner is not null) + { + AddEdge( + edges, + EnsureNode(nodes, context, owner, root), + lambda, + "contains", + Evidence(context, root, anonymous.GetLocation())); + } + unresolved.Add(new JsonObject + { + ["provider"] = GraphProtocol.Provider, + ["language"] = "csharp", + ["target"] = context.Target, + ["universe"] = universe, + ["family"] = "contains", + ["evidence"] = Evidence(context, root, anonymous.GetLocation()), + ["reason"] = "identity-unstable", + ["candidates"] = new JsonArray(lambda), + }); + break; + } + case IdentifierNameSyntax identifier: + { + var symbol = model.GetSymbolInfo(identifier, cancellationToken).Symbol; + if (symbol is null || IsDeclarationIdentifier(identifier)) + { + break; + } + var owner = model.GetEnclosingSymbol(identifier.SpanStart, cancellationToken); + if (owner is null) + { + break; + } + var from = EnsureNode(nodes, context, owner, root); + var to = EnsureNode(nodes, context, symbol, root); + AddEdge(edges, from, to, "references", Evidence(context, root, identifier.GetLocation())); + if (symbol is ITypeSymbol) + { + AddEdge(edges, from, to, "type_ref", Evidence(context, root, identifier.GetLocation())); + } + if (symbol is IFieldSymbol or IPropertySymbol or IEventSymbol or ILocalSymbol or IParameterSymbol) + { + AddEdge(edges, from, to, "accesses", Evidence(context, root, identifier.GetLocation())); + } + break; + } + case TypeSyntax typeSyntax: + { + var type = model.GetTypeInfo(typeSyntax, cancellationToken).Type; + AddSemanticEdge(nodes, edges, context, model, type, typeSyntax, root, "type_ref", cancellationToken); + break; + } + } + } + + var diagnostics = DiagnosticsFor(context, document, root, diagnosticsByFile); + var source = new JsonObject + { + ["file"] = sourceFile, + ["checkerDigest"] = GraphProtocol.HashText(text.ToString()), + ["diskDigest"] = !generated + && document.FilePath is { Length: > 0 } disk + && File.Exists(disk) + ? GraphProtocol.HashBytes(await File.ReadAllBytesAsync(disk, cancellationToken).ConfigureAwait(false)) + : "", + }; + return Shard( + DocumentShardKey(context, document, root, generated), + context.Target, + new JsonArray(nodes.Values.OrderBy(node => node["id"]!.GetValue(), StringComparer.Ordinal).Select(node => node.DeepClone()).ToArray()), + new JsonArray(edges.Values.OrderBy(edge => EdgeKey(edge), StringComparer.Ordinal).Select(edge => edge.DeepClone()).ToArray()), + diagnostics, + new JsonArray(), + unresolved, + new JsonArray(source), + GraphProtocol.Hash(new JsonArray(declarationFingerprints + .Order(StringComparer.Ordinal) + .Select(value => JsonValue.Create(value)) + .ToArray()))); + } + + private static string DocumentShardKey( + ProjectContext context, + Document document, + string root, + bool generated) => + $"csharp-shard-v1|{context.Target}|document:{GraphFile(root, SourceIdentity(context, document, generated))}"; + + private static async Task CheckerDigest( + Document document, + CancellationToken cancellationToken) => + GraphProtocol.HashText( + (await document.GetTextAsync(cancellationToken).ConfigureAwait(false)).ToString()); + + private static string SourceCheckerDigest(ShardDraft shard) => + shard.Payload["sources"]!.AsArray().Single()!["checkerDigest"]!.GetValue(); + + private static async Task GeneratedOutputsMatch( + ProjectContext context, + IReadOnlyList cached, + CancellationToken cancellationToken) + { + var sourcePaths = context.Project.Documents + .Where(document => document.FilePath is { Length: > 0 }) + .Select(document => Path.GetFullPath(document.FilePath!)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var current = context.Compilation.SyntaxTrees + .Where(tree => tree.FilePath is { Length: > 0 }) + .Select(tree => new + { + File = Path.GetFullPath(tree.FilePath), + Tree = tree, + }) + .Where(entry => !sourcePaths.Contains(entry.File)) + .GroupBy(entry => entry.File, StringComparer.OrdinalIgnoreCase) + .ToDictionary( + group => group.Key, + group => GraphProtocol.HashText( + group.First().Tree.GetText(cancellationToken).ToString()), + StringComparer.OrdinalIgnoreCase); + if (current.Count != cached.Count + || cached.Any(document => document.Document.FilePath is not { Length: > 0 })) + { + return false; + } + foreach (var document in cached) + { + var file = Path.GetFullPath(document.Document.FilePath!); + if (!current.TryGetValue(file, out var digest) + || digest != await CheckerDigest(document.Document, cancellationToken) + .ConfigureAwait(false)) + { + return false; + } + } + return true; + } + + private static HashSet DependentClosure( + Solution solution, + IReadOnlySet changed) + { + var output = changed.ToHashSet(); + var pending = new Queue(changed); + var graph = solution.GetProjectDependencyGraph(); + while (pending.TryDequeue(out var project)) + { + foreach (var dependent in graph.GetProjectsThatDirectlyDependOnThisProject(project)) + { + if (output.Add(dependent)) + { + pending.Enqueue(dependent); + } + } + } + return output; + } + + private static JsonArray DiagnosticsFor( + ProjectContext context, + Document document, + string root, + IReadOnlyDictionary> diagnosticsByFile, + JsonArray? cachedDiagnostics = null) + { + var diagnostics = new List(); + if (document.FilePath is { Length: > 0 } filePath + && diagnosticsByFile.TryGetValue( + $"{context.Target}\0{Path.GetFullPath(filePath)}", + out var rows)) + { + foreach (var diagnostic in rows + .Select(diagnostic => DiagnosticNode(context, root, diagnostic)) + .OrderBy(DiagnosticKey, StringComparer.Ordinal)) + { + diagnostics.Add(diagnostic); + } + } + if (cachedDiagnostics is not null) + { + diagnostics.AddRange(cachedDiagnostics + .Where(diagnostic => diagnostic is JsonObject value + && !IsCompilerDiagnostic(value)) + .Select(diagnostic => diagnostic!.DeepClone().AsObject())); + } + return new JsonArray(diagnostics + .GroupBy(DiagnosticKey, StringComparer.Ordinal) + .Select(group => group.First()) + .OrderBy(DiagnosticKey, StringComparer.Ordinal) + .ToArray()); + } + + private static bool IsCompilerDiagnostic(JsonObject diagnostic) + { + var code = diagnostic["code"]?.GetValue(); + return code is { Length: > 2 } + && code.StartsWith("CS", StringComparison.Ordinal) + && code.AsSpan(2).IndexOfAnyExceptInRange('0', '9') == -1; + } + + private static ShardDraft ReuseWithDiagnostics( + ShardDraft prior, + JsonArray diagnostics) + { + if (GraphProtocol.Hash(prior.Payload["diagnostics"]!) == GraphProtocol.Hash(diagnostics)) + { + return prior; + } + var payload = prior.Payload.DeepClone().AsObject(); + payload["diagnostics"] = diagnostics; + return new ShardDraft( + prior.Key, + payload, + prior.InterfaceFingerprint, + GraphProtocol.ShardFactHash(payload), + GraphProtocol.Hash(payload)); + } + + private static string DeclarationFingerprint(ISymbol symbol) + { + symbol = Canonical(symbol); + var builder = new StringBuilder(); + builder.Append(DocumentationCommentId.CreateDeclarationId(symbol)); + builder.Append('|').Append(symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)); + builder.Append('|').Append(symbol.DeclaredAccessibility); + foreach (var attribute in symbol.GetAttributes() + .OrderBy(attribute => attribute.AttributeClass?.ToDisplayString(), StringComparer.Ordinal)) + { + builder.Append('|').Append(attribute.ToString()); + } + if (symbol is IFieldSymbol { HasConstantValue: true } field) + { + builder.Append("|const:").Append(field.ConstantValue); + } + if (symbol is INamedTypeSymbol type) + { + builder.Append("|base:").Append(type.BaseType?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)); + foreach (var @interface in type.Interfaces + .OrderBy(value => value.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), StringComparer.Ordinal)) + { + builder.Append("|interface:") + .Append(@interface.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)); + } + } + return builder.ToString(); + } + + private static void AddDispatchCandidates( + Dictionary nodes, + JsonArray unresolved, + ProjectContext context, + IMethodSymbol target, + InvocationExpressionSyntax invocation, + string root, + string universe) + { + if (!target.IsAbstract + && !target.IsVirtual + && target.ContainingType.TypeKind != TypeKind.Interface) + { + return; + } + var candidates = context.Catalog.DispatchCandidates(target) + .Select(candidate => EnsureNode(nodes, candidate.Context, candidate.Symbol, root)) + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal) + .ToArray(); + unresolved.Add(new JsonObject + { + ["provider"] = GraphProtocol.Provider, + ["language"] = "csharp", + ["target"] = context.Target, + ["universe"] = universe, + ["family"] = "dispatches", + ["evidence"] = Evidence(context, root, invocation.GetLocation()), + ["reason"] = candidates.Length == 0 ? "external-boundary" : "dynamic", + ["candidates"] = new JsonArray(candidates + .Select(candidate => JsonValue.Create(candidate)) + .ToArray()), + }); + } + + private static void AddSemanticEdge( + Dictionary nodes, + Dictionary edges, + ProjectContext context, + SemanticModel model, + ISymbol? target, + SyntaxNode syntax, + string root, + string kind, + CancellationToken cancellationToken) + { + var owner = model.GetEnclosingSymbol(syntax.SpanStart, cancellationToken); + if (owner is null || target is null) + { + return; + } + AddEdge( + edges, + EnsureNode(nodes, context, owner, root), + EnsureNode(nodes, context, target, root), + kind, + Evidence(context, root, syntax.GetLocation())); + } + + private static void AddTypeRelations( + Dictionary nodes, + Dictionary edges, + ProjectContext context, + INamedTypeSymbol type, + string from, + string root, + Location location) + { + if (type.BaseType is { SpecialType: not SpecialType.System_Object } baseType) + { + AddEdge(edges, from, EnsureNode(nodes, context, baseType, root), "extends", Evidence(context, root, location)); + } + foreach (var contract in type.Interfaces) + { + AddEdge( + edges, + from, + EnsureNode(nodes, context, contract, root), + type.TypeKind == TypeKind.Interface ? "extends" : "implements", + Evidence(context, root, location)); + } + } + + private static void AddSynthesizedRecordMembers( + Dictionary nodes, + Dictionary edges, + ProjectContext context, + INamedTypeSymbol type, + string owner, + string root) + { + if (!type.IsRecord) + { + return; + } + foreach (var member in type.GetMembers() + .Where(member => member.IsImplicitlyDeclared) + .OrderBy(member => DocumentationCommentId.CreateDeclarationId(member) + ?? member.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), StringComparer.Ordinal)) + { + var memberId = EnsureNode(nodes, context, member, root); + AddEdge(edges, owner, memberId, "contains", null); + AddOverride(nodes, edges, context, member, memberId, root, + member.Locations.FirstOrDefault(location => location.IsInSource)); + } + } + + private static void AddOverride( + Dictionary nodes, + Dictionary edges, + ProjectContext context, + ISymbol symbol, + string from, + string root, + Location? location) + { + ISymbol? overridden = symbol switch + { + IMethodSymbol method => method.OverriddenMethod, + IPropertySymbol property => property.OverriddenProperty, + IEventSymbol @event => @event.OverriddenEvent, + _ => null, + }; + if (overridden is not null) + { + AddEdge( + edges, + from, + EnsureNode(nodes, context, overridden, root), + "overrides", + location is null ? null : Evidence(context, root, location)); + } + if (symbol.ContainingType is { } containingType) + { + foreach (var contract in containingType.AllInterfaces + .SelectMany(@interface => @interface.GetMembers()) + .Where(contract => SymbolEqualityComparer.Default.Equals( + containingType.FindImplementationForInterfaceMember(contract), + symbol)) + .OrderBy(contract => DocumentationCommentId.CreateDeclarationId(contract) + ?? contract.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), StringComparer.Ordinal)) + { + AddEdge( + edges, + from, + EnsureNode(nodes, context, contract, root), + "implements", + location is null ? null : Evidence(context, root, location)); + } + } + } + + private static string EnsureNode( + Dictionary nodes, + ProjectContext context, + ISymbol symbol, + string root) + { + var canonical = Canonical(symbol); + var identityContext = context.Catalog.Resolve(context, canonical); + var node = NodeForSymbol(identityContext, canonical, root); + var id = node["id"]!.GetValue(); + nodes.TryAdd(id, node); + return id; + } + + private static JsonObject NodeForSymbol( + ProjectContext context, + ISymbol symbol, + string root) + { + symbol = Canonical(symbol); + return context.Nodes.GetOrAdd(symbol, resolved => + CreateNodeForSymbol(context, resolved, root)); + } + + private static JsonObject CreateNodeForSymbol( + ProjectContext context, + ISymbol symbol, + string root) + { + // A symbol is repeated in its declaration shard and every shard that + // references it. Build one canonical representation in all of them: + // Roslyn's symbol location is the declaration token, while a caller- + // supplied syntax location may span the whole body and move on body edits. + var location = SourceLocation(symbol); + var declaration = symbol.GetAttributes().Length == 0 + ? null + : CanonicalDeclaration(symbol); + var source = location?.SourceTree?.FilePath; + var generated = source is null ? null : GeneratedGraphFile(context, root, source); + var external = source is null || generated is null && !IsWithin(root, source); + var kind = NodeKind(symbol, external); + var name = symbol is IMethodSymbol { MethodKind: MethodKind.Constructor } constructor + ? constructor.ContainingType.Name + : symbol.Name == "" ? symbol.ContainingAssembly?.Name ?? "external" : symbol.Name; + var qualified = symbol.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + var node = new JsonObject + { + ["id"] = SymbolId(context, symbol, root), + ["kind"] = kind, + ["language"] = "csharp", + ["name"] = name, + ["file"] = generated ?? (external + ? $"bundled:///csharp/dependencies/{Safe(symbol.ContainingAssembly?.Name ?? "unknown")}" + : GraphFile(root, Path.GetFullPath(source!))), + ["external"] = external, + ["signature"] = symbol.ToDisplayString(SymbolDisplayFormat.MinimallyQualifiedFormat), + }; + if (qualified != name) + { + node["qualifiedName"] = qualified; + } + if (Exported(symbol)) + { + node["exported"] = true; + } + if (location is { IsInSource: true }) + { + node["evidence"] = Evidence(context, root, location); + } + var modifiers = Modifiers(symbol); + if (modifiers.Count != 0) + { + node["modifiers"] = new JsonArray(modifiers.Select(value => JsonValue.Create(value)).ToArray()); + } + if (declaration is MemberDeclarationSyntax member && member.AttributeLists.Count != 0) + { + node["decorators"] = new JsonArray(member.AttributeLists + .SelectMany(list => list.Attributes) + .Select(attribute => new JsonObject + { + ["name"] = attribute.Name.ToString(), + ["arguments"] = new JsonArray(attribute.ArgumentList?.Arguments + .Select(argument => DecoratorArgument(argument.Expression)) + .ToArray() ?? []), + }) + .ToArray()); + } + return node; + } + + private static JsonNode? Literal(ExpressionSyntax expression) => expression switch + { + LiteralExpressionSyntax literal when literal.Token.Value is string value => value, + LiteralExpressionSyntax literal when literal.Token.Value is bool value => value, + LiteralExpressionSyntax literal when literal.Token.Value is int value => value, + LiteralExpressionSyntax literal when literal.Token.Value is long value => value, + LiteralExpressionSyntax literal when literal.Token.Value is double value => value, + _ => null, + }; + + private static JsonObject DecoratorArgument(ExpressionSyntax expression) + { + var output = new JsonObject(); + if (Literal(expression) is { } literal) + { + output["literal"] = literal; + } + return output; + } + + private static ISymbol? DeclaredSymbol( + SemanticModel model, + SyntaxNode syntax, + CancellationToken cancellationToken) => syntax switch + { + BaseNamespaceDeclarationSyntax or BaseTypeDeclarationSyntax or DelegateDeclarationSyntax + or BaseMethodDeclarationSyntax or PropertyDeclarationSyntax or IndexerDeclarationSyntax + or EventDeclarationSyntax or EnumMemberDeclarationSyntax or ParameterSyntax + or LocalFunctionStatementSyntax => model.GetDeclaredSymbol(syntax, cancellationToken), + VariableDeclaratorSyntax variable + when variable.Parent?.Parent is FieldDeclarationSyntax or EventFieldDeclarationSyntax + or LocalDeclarationStatementSyntax => model.GetDeclaredSymbol(variable, cancellationToken), + _ => null, + }; + + private static ISymbol Canonical(ISymbol symbol) => symbol switch + { + IMethodSymbol method => CanonicalMethod(method), + INamedTypeSymbol type => type.OriginalDefinition, + IPropertySymbol property => property.OriginalDefinition, + IEventSymbol @event => @event.OriginalDefinition, + IFieldSymbol field => field.OriginalDefinition, + _ => symbol, + }; + + private static IMethodSymbol CanonicalMethod(IMethodSymbol method) + { + method = method.ReducedFrom ?? method; + method = method.PartialDefinitionPart ?? method; + return method.OriginalDefinition; + } + + private static ISymbol? OwnerSymbol(ISymbol symbol) => symbol switch + { + INamespaceSymbol namespaceSymbol => namespaceSymbol.ContainingNamespace, + _ => symbol.ContainingSymbol, + }; + + private static ISymbol? AttributeOwner( + SemanticModel model, + AttributeSyntax attribute, + CancellationToken cancellationToken) + { + for (var node = attribute.Parent?.Parent; node is not null; node = node.Parent) + { + var symbol = DeclaredSymbol(model, node, cancellationToken); + if (symbol is not null) + { + return symbol; + } + } + return null; + } + + private static string SymbolId(ProjectContext context, ISymbol symbol, string root) + { + symbol = Canonical(symbol); + var source = SourceLocation(symbol)?.SourceTree?.FilePath; + var external = source is null + || GeneratedGraphFile(context, root, source) is null && !IsWithin(root, source); + var kind = NodeKind(symbol, external); + var name = symbol is IMethodSymbol { MethodKind: MethodKind.Constructor } constructor + ? constructor.ContainingType.Name + : symbol.Name == "" ? symbol.ContainingAssembly?.Name ?? "external" : symbol.Name; + var qualified = symbol.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + var display = qualified == name ? name : qualified; + var assembly = symbol.ContainingAssembly?.Identity.GetDisplayName() ?? context.Assembly; + var documentation = DocumentationCommentId.CreateDeclarationId(symbol); + if (documentation is null) + { + var owner = symbol.ContainingSymbol is null + ? "global" + : DocumentationCommentId.CreateDeclarationId(Canonical(symbol.ContainingSymbol)) + ?? symbol.ContainingSymbol.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + documentation = symbol switch + { + IParameterSymbol parameter => $"parameter:{owner}:{parameter.Ordinal}:{parameter.Name}", + ITypeParameterSymbol parameter => $"type-parameter:{owner}:{parameter.Ordinal}:{parameter.Name}", + ILocalSymbol local => $"local:{owner}:{LexicalIdentity(local, root)}:{local.Name}:{local.Type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)}", + _ => $"structural:{owner}:{LexicalIdentity(symbol, root)}:{symbol.Kind}:{symbol.Name}", + }; + } + var native = $"csharp-v1|{assembly.Length}:{assembly}|{context.Framework.Length}:{context.Framework}|{documentation}|{kind}"; + var unstable = symbol is IMethodSymbol { MethodKind: MethodKind.AnonymousFunction } + or INamedTypeSymbol { IsAnonymousType: true }; + return SemanticNodeId( + native, + kind, + display, + context.Target, + unstable ? "generation" : "persistent", + DocumentationCommentId.CreateDeclarationId(symbol) is null + ? "structural" + : "semantic"); + } + + private static string LexicalIdentity(ISymbol symbol, string root) + { + var location = SourceLocation(symbol); + if (location?.SourceTree is not { } tree) + { + return "generated"; + } + var syntaxRoot = tree.GetRoot(); + var syntax = syntaxRoot.FindNode(location.SourceSpan, getInnermostNodeForTie: true); + var parts = new Stack(); + for (var node = syntax; node.Parent is { } parent; node = parent) + { + var ordinal = parent.ChildNodes() + .Where(sibling => sibling.RawKind == node.RawKind) + .TakeWhile(sibling => sibling != node) + .Count(); + parts.Push($"{node.RawKind}:{ordinal}"); + if (DeclaredSymbolFromShape(node)) + { + break; + } + } + return $"{GraphFile(root, Path.GetFullPath(tree.FilePath))}:{string.Join('/', parts)}"; + } + + private static Location? SourceLocation(ISymbol symbol) + { + Location? best = null; + foreach (var location in symbol.Locations) + { + if (location.IsInSource + && (best is null || CompareSourcePosition(location, best) < 0)) + { + best = location; + } + } + return best; + } + + private static SyntaxNode? CanonicalDeclaration(ISymbol symbol) + { + SyntaxReference? best = null; + foreach (var reference in symbol.DeclaringSyntaxReferences) + { + if (best is null || CompareSourcePosition(reference, best) < 0) + { + best = reference; + } + } + return best?.GetSyntax(); + } + + private static int CompareSourcePosition(Location left, Location right) + { + var compared = CompareSourcePath( + left.SourceTree?.FilePath ?? "", + right.SourceTree?.FilePath ?? ""); + return compared != 0 + ? compared + : left.SourceSpan.Start != right.SourceSpan.Start + ? left.SourceSpan.Start.CompareTo(right.SourceSpan.Start) + : left.SourceSpan.Length.CompareTo(right.SourceSpan.Length); + } + + private static int CompareSourcePosition( + SyntaxReference left, + SyntaxReference right) + { + var compared = CompareSourcePath( + left.SyntaxTree.FilePath ?? "", + right.SyntaxTree.FilePath ?? ""); + return compared != 0 + ? compared + : left.Span.Start != right.Span.Start + ? left.Span.Start.CompareTo(right.Span.Start) + : left.Span.Length.CompareTo(right.Span.Length); + } + + private static int CompareSourcePath(string left, string right) + { + var compared = StringComparer.OrdinalIgnoreCase.Compare(left, right); + return compared != 0 ? compared : StringComparer.Ordinal.Compare(left, right); + } + + private static bool DeclaredSymbolFromShape(SyntaxNode syntax) => syntax is + BaseNamespaceDeclarationSyntax or BaseTypeDeclarationSyntax or DelegateDeclarationSyntax + or BaseMethodDeclarationSyntax or PropertyDeclarationSyntax or IndexerDeclarationSyntax + or EventDeclarationSyntax or EnumMemberDeclarationSyntax or LocalFunctionStatementSyntax; + + private static string ProjectNodeId(ProjectContext context) => + SemanticNodeId( + $"project:{context.ProjectFile}:{context.Assembly}:{context.Framework}", + "package", + context.Assembly, + context.Target); + + private static string SemanticNodeId( + string symbol, + string role, + string display, + string target, + string stability = "persistent", + string nativeStability = "semantic") + { + var fields = new (string Name, string Value)[] + { + ("version", "2"), + ("language", "csharp"), + ("role", role), + ("symbol", symbol), + ("stability", stability), + ("scope.target", target), + ("native.stability", nativeStability), + ("native.key", symbol), + ("display", display), + }; + var encoded = string.Concat(fields.Select(field => + $"{Encoding.UTF8.GetByteCount(field.Name)}:{field.Name}{Encoding.UTF8.GetByteCount(field.Value)}:{field.Value}")); + return $"@v2/csharp/{GraphProtocol.HashText(encoded)}#{EncodeComponent(display)}:{role}"; + } + + private static string EncodeComponent(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + var output = new StringBuilder(bytes.Length); + foreach (var valueByte in bytes) + { + var character = (char)valueByte; + if ((character >= 'A' && character <= 'Z') + || (character >= 'a' && character <= 'z') + || (character >= '0' && character <= '9') + || character is '-' or '_' or '.' or '!' or '~' or '*' or '\'' or '(' or ')') + { + output.Append(character); + } + else + { + output.Append('%'); + output.Append(valueByte.ToString("X2", System.Globalization.CultureInfo.InvariantCulture)); + } + } + return output.ToString(); + } + + private static string NodeKind(ISymbol symbol, bool external) + { + if (external) + { + return "external_symbol"; + } + return symbol switch + { + INamespaceSymbol => "namespace", + INamedTypeSymbol { TypeKind: TypeKind.Class } => "class", + INamedTypeSymbol { TypeKind: TypeKind.Interface } => "interface", + INamedTypeSymbol { TypeKind: TypeKind.Enum } => "enum", + INamedTypeSymbol => "type", + IMethodSymbol { MethodKind: MethodKind.Constructor or MethodKind.StaticConstructor } => "constructor", + IMethodSymbol { MethodKind: MethodKind.LocalFunction or MethodKind.AnonymousFunction } => "function", + IMethodSymbol => "method", + IPropertySymbol => "property", + IFieldSymbol => "field", + IEventSymbol => "property", + IParameterSymbol or ITypeParameterSymbol => "parameter", + ILocalSymbol => "variable", + _ => "variable", + }; + } + + private static bool Exported(ISymbol symbol) => symbol.DeclaredAccessibility is + Accessibility.Public or Accessibility.Protected or Accessibility.ProtectedOrInternal; + + private static List Modifiers(ISymbol symbol) + { + var output = new List(); + var accessibility = symbol.DeclaredAccessibility switch + { + Accessibility.Public => "public", + Accessibility.Private => "private", + Accessibility.Protected => "protected", + Accessibility.Internal => "internal", + Accessibility.ProtectedOrInternal => "protected", + Accessibility.ProtectedAndInternal => "private", + _ => null, + }; + if (accessibility is not null) + { + output.Add(accessibility); + } + if (symbol.IsStatic) + { + output.Add("static"); + } + if (symbol.IsAbstract) + { + output.Add("abstract"); + } + if (symbol is IFieldSymbol { IsReadOnly: true } or IPropertySymbol { IsReadOnly: true }) + { + output.Add("readonly"); + } + if (symbol is IMethodSymbol { IsAsync: true }) + { + output.Add("async"); + } + if (symbol is IFieldSymbol { IsConst: true }) + { + output.Add("const"); + } + return output; + } + + private static bool IsTest(ISymbol symbol) => symbol.GetAttributes().Any(attribute => + attribute.AttributeClass?.ToDisplayString() is + "Xunit.FactAttribute" or "Xunit.TheoryAttribute" + or "NUnit.Framework.TestAttribute" or "NUnit.Framework.TestCaseAttribute" + or "Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute"); + + private static bool IsDeclarationIdentifier(IdentifierNameSyntax identifier) => + identifier.Parent is NameEqualsSyntax or NameColonSyntax + || identifier.Ancestors().Any(ancestor => ancestor is AttributeSyntax attribute && attribute.Name == identifier); + + private static void AddEdge( + Dictionary edges, + string from, + string to, + string kind, + JsonObject? evidence) + { + if (from == to) + { + return; + } + var key = $"{kind}\0{from}\0{to}"; + if (edges.ContainsKey(key)) + { + return; + } + var edge = new JsonObject + { + ["from"] = from, + ["to"] = to, + ["kind"] = kind, + }; + if (evidence is not null) + { + edge["evidence"] = evidence; + } + edges[key] = edge; + } + + private static string EdgeKey(JsonObject edge) => string.Join('\0', + edge["kind"]!.GetValue(), + edge["from"]!.GetValue(), + edge["to"]!.GetValue()); + + private static JsonObject Evidence( + ProjectContext context, + string root, + Location location) + { + var span = location.GetLineSpan(); + var start = span.StartLinePosition; + var end = span.EndLinePosition; + return new JsonObject + { + ["file"] = location.SourceTree?.FilePath is { Length: > 0 } file + ? GeneratedGraphFile(context, root, file) + ?? GraphFile(root, Path.GetFullPath(file)) + : "", + ["startLine"] = start.Line + 1, + ["startCol"] = start.Character + 1, + ["endLine"] = end.Line + 1, + ["endCol"] = end.Character + 1, + }; + } + + private static JsonObject DiagnosticNode( + ProjectContext context, + string root, + Diagnostic diagnostic) + { + if (!diagnostic.Location.IsInSource) + { + return new JsonObject + { + ["file"] = "", + ["line"] = 0, + ["column"] = 0, + ["code"] = diagnostic.Id, + ["message"] = diagnostic.GetMessage(), + ["severity"] = Severity(diagnostic.Severity), + }; + } + var span = diagnostic.Location.GetLineSpan(); + return new JsonObject + { + ["file"] = GeneratedGraphFile(context, root, span.Path) + ?? GraphFile(root, Path.GetFullPath(span.Path)), + ["line"] = span.StartLinePosition.Line + 1, + ["column"] = span.StartLinePosition.Character + 1, + ["code"] = diagnostic.Id, + ["message"] = diagnostic.GetMessage(), + ["severity"] = Severity(diagnostic.Severity), + }; + } + + private static string DiagnosticKey(JsonObject diagnostic) => string.Join('\0', + diagnostic["file"]!.GetValue(), + diagnostic["line"]!.GetValue().ToString("D10", + System.Globalization.CultureInfo.InvariantCulture), + diagnostic["column"]!.GetValue().ToString("D10", + System.Globalization.CultureInfo.InvariantCulture), + diagnostic["code"]!.GetValue(), + diagnostic["severity"]!.GetValue(), + diagnostic["message"]!.GetValue()); + + private static string Severity(DiagnosticSeverity severity) => severity switch + { + DiagnosticSeverity.Error => "error", + DiagnosticSeverity.Warning => "warning", + DiagnosticSeverity.Info => "info", + _ => "hint", + }; + + private static ShardDraft Shard( + string key, + string target, + JsonArray nodes, + JsonArray edges, + JsonArray diagnostics, + JsonArray coverage, + JsonArray unresolved, + JsonArray sources, + string interfaceFingerprint = "") + { + var payload = new JsonObject + { + ["key"] = key, + ["target"] = target, + ["languages"] = new JsonArray("csharp"), + ["nodes"] = nodes, + ["edges"] = edges, + ["diagnostics"] = diagnostics, + ["coverage"] = coverage, + ["unresolved"] = unresolved, + ["sources"] = sources, + }; + return new ShardDraft( + key, + payload, + interfaceFingerprint, + GraphProtocol.ShardFactHash(payload), + GraphProtocol.Hash(payload)); + } + + private static async Task> DiagnosticsOf( + Project project, + Compilation compilation, + CancellationToken cancellationToken) + { + var analyzers = project.AnalyzerReferences + .SelectMany(reference => reference.GetAnalyzers(LanguageNames.CSharp)) + .Distinct() + .ToImmutableArray(); + return analyzers.Length == 0 + ? compilation.GetDiagnostics(cancellationToken) + : await compilation + .WithAnalyzers( + analyzers, + project.AnalyzerOptions) + .GetAllDiagnosticsAsync(cancellationToken) + .ConfigureAwait(false); + } + + private static async Task> IncrementalDiagnosticsOf( + IReadOnlyList documents, + CancellationToken cancellationToken) + { + var diagnostics = new List(); + foreach (var document in documents) + { + var model = await document.GetSemanticModelAsync(cancellationToken).ConfigureAwait(false); + if (model is null) + { + continue; + } + diagnostics.AddRange(model.GetDiagnostics(cancellationToken: cancellationToken)); + } + return diagnostics; + } + + private static List BuildSources( + string root, + IReadOnlyList contexts) + { + var names = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "global.json", "Directory.Build.props", "Directory.Build.targets", + "Directory.Packages.props", "packages.lock.json", "nuget.config", + }; + var files = EnumerateWorkspaceFiles(root) + .Where(file => Path.GetExtension(file).ToLowerInvariant() is ".sln" or ".slnx" or ".csproj" or ".props" or ".targets" + || names.Contains(Path.GetFileName(file))) + .Concat(contexts.SelectMany(context => context.Project.MetadataReferences + .Select(reference => reference.Display))) + .Concat(contexts.SelectMany(context => context.Project.AnalyzerReferences + .Select(reference => reference.FullPath))) + .Concat(contexts.SelectMany(context => context.Project.AdditionalDocuments + .Concat(context.Project.AnalyzerConfigDocuments) + .Select(document => document.FilePath))) + .Concat(contexts.Select(context => context.Project.FilePath is null + ? null + : Path.Combine( + Path.GetDirectoryName(context.Project.FilePath)!, + "obj", + "project.assets.json"))) + .Where(file => file is { Length: > 0 } && File.Exists(file)) + .Select(file => Path.GetFullPath(file!)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Select(SourceOf) + .OrderBy(source => source["file"]!.GetValue(), StringComparer.Ordinal) + .ToList(); + return files; + } + + private static IEnumerable EnumerateWorkspaceFiles(string root) + { + var pending = new Stack(); + pending.Push(root); + while (pending.Count != 0) + { + var directory = pending.Pop(); + foreach (var entry in Directory.EnumerateFileSystemEntries(directory) + .Order(StringComparer.Ordinal)) + { + if (Directory.Exists(entry)) + { + if (!Ignored(root, entry)) + { + pending.Push(entry); + } + } + else if (File.Exists(entry)) + { + yield return entry; + } + } + } + } + + private static JsonObject SourceOf(string file) + { + if (file.StartsWith("bundled:///", StringComparison.Ordinal)) + { + var digest = GraphProtocol.HashText(file); + return new JsonObject + { + ["file"] = file, + ["checkerDigest"] = digest, + ["diskDigest"] = "", + }; + } + var absolute = Path.GetFullPath(file); + var bytes = File.ReadAllBytes(absolute); + var byteDigest = GraphProtocol.HashBytes(bytes); + return new JsonObject + { + ["file"] = absolute, + // Build inputs include binary metadata and analyzers as well as + // XML/MSBuild text. Roslyn consumes their bytes, so one atomic byte + // read is both the checker identity and the publication fence. + ["checkerDigest"] = byteDigest, + ["diskDigest"] = byteDigest, + }; + } + + private static string SourceIdentity( + ProjectContext context, + Document document, + bool generated) + { + if (generated) + { + return GeneratedSourceIdentity(context, document); + } + if (document.FilePath is { Length: > 0 } file) + { + return Path.GetFullPath(file); + } + return $"bundled:///csharp/documents/{GraphProtocol.HashText(context.Target)}/{Safe(document.Name)}"; + } + + private static string GeneratedSourceIdentity( + ProjectContext context, + Document document) + { + var origin = document.FilePath is { Length: > 0 } file + ? Path.GetFullPath(file) + : string.Join('/', document.Folders.Append(document.Name)); + return $"bundled:///csharp/generated/{GraphProtocol.HashText(context.Target)}/{GraphProtocol.HashText(origin)}/{Safe(document.Name)}"; + } + + private static string? GeneratedGraphFile( + ProjectContext context, + string root, + string source) + { + var absolute = Path.GetFullPath(source); + if (context.GeneratedSources.TryGetValue(absolute, out var generated)) + { + return generated; + } + return IsWithin(root, absolute) && Ignored(root, absolute) + ? $"bundled:///csharp/generated/{GraphProtocol.HashText(context.Target)}/{GraphProtocol.HashText(absolute)}/{Safe(Path.GetFileName(absolute))}" + : null; + } + + private static string DocumentIdentity(Document document) => + document.FilePath is { Length: > 0 } file ? Path.GetFullPath(file) : document.Name; + + private static string GraphFile(string root, string source) + { + if (source.StartsWith("bundled:///", StringComparison.Ordinal)) + { + return source; + } + var relative = Path.GetRelativePath(Path.GetFullPath(root), Path.GetFullPath(source)); + if (Path.IsPathRooted(relative) || relative == ".") + { + var absolute = Path.GetFullPath(source); + return $"bundled:///csharp/external-source/{GraphProtocol.HashText(absolute)}/{Safe(Path.GetFileName(absolute))}"; + } + return relative.Replace(Path.DirectorySeparatorChar, '/').Replace(Path.AltDirectorySeparatorChar, '/'); + } + + private static bool IsWithin(string root, string file) + { + var relative = Path.GetRelativePath(Path.GetFullPath(root), Path.GetFullPath(file)); + return relative != ".." + && !relative.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) + && !Path.IsPathRooted(relative); + } + + private static bool Ignored(string root, string file) + { + var parts = Path.GetRelativePath(root, file) + .Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return parts.Any(part => part is ".git" or ".wiki" or "bin" or "node_modules" or "obj"); + } + + private static string TargetFramework(Compilation compilation) + { + var attribute = compilation.Assembly.GetAttributes().FirstOrDefault(candidate => + candidate.AttributeClass?.ToDisplayString() == "System.Runtime.Versioning.TargetFrameworkAttribute"); + return attribute?.ConstructorArguments.FirstOrDefault().Value as string ?? "unspecified"; + } + + private static string Safe(string value) + { + var builder = new StringBuilder(value.Length); + foreach (var character in value) + { + builder.Append(char.IsLetterOrDigit(character) || character is '.' or '-' or '_' + ? character + : '_'); + } + return builder.Length == 0 ? "unknown" : builder.ToString(); + } + + private sealed class ProjectContext( + Project project, + Compilation compilation, + string target, + string assembly, + string framework, + string projectFile, + IReadOnlyList diagnostics, + IReadOnlyList? documents = null, + IReadOnlyList? generatedDocuments = null) + { + public Project Project { get; } = project; + public Compilation Compilation { get; } = compilation; + public string Target { get; } = target; + public string Assembly { get; } = assembly; + public string Framework { get; } = framework; + public string ProjectFile { get; } = projectFile; + public IReadOnlyList Diagnostics { get; set; } = diagnostics; + public Dictionary GeneratedSources { get; } = + new(StringComparer.OrdinalIgnoreCase); + public IReadOnlyList? Documents { get; set; } = documents; + public IReadOnlyList? GeneratedDocuments { get; set; } = + generatedDocuments; + public ConcurrentDictionary Nodes { get; } = + new(SymbolEqualityComparer.Default); + public ProjectCatalog Catalog { get; set; } = null!; + } + + private sealed record ContextDocument(Document Document, bool Generated); + + private sealed class ProjectCatalog(IReadOnlyList contexts) + { + private readonly ConcurrentDictionary> dispatchCache = + new(StringComparer.Ordinal); + + public ProjectContext Resolve(ProjectContext current, ISymbol symbol) + { + var assembly = symbol.ContainingAssembly?.Identity.GetDisplayName(); + if (assembly is null) + { + return current; + } + if (current.Assembly == assembly) + { + return current; + } + var candidates = contexts + .Where(context => context.Assembly == assembly) + .ToArray(); + if (candidates.Length == 0) + { + return current; + } + var sourceFiles = symbol.Locations + .Where(location => location.IsInSource && location.SourceTree?.FilePath is { Length: > 0 }) + .Select(location => Path.GetFullPath(location.SourceTree!.FilePath)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var matchingFramework = candidates + .Where(context => context.Framework == current.Framework) + .ToArray(); + var bySource = matchingFramework + .Concat(candidates.Except(matchingFramework)) + .FirstOrDefault(context => context.Project.Documents + .Any(document => document.FilePath is { Length: > 0 } file + && sourceFiles.Contains(Path.GetFullPath(file))) + || sourceFiles.Any(context.GeneratedSources.ContainsKey)); + return bySource + ?? matchingFramework.FirstOrDefault() + ?? candidates[0]; + } + + public IReadOnlyList DispatchCandidates(IMethodSymbol target) + { + target = CanonicalMethod(target); + var declarationId = DocumentationCommentId.CreateDeclarationId(target); + if (declarationId is null) + { + return []; + } + var assembly = target.ContainingAssembly?.Identity.GetDisplayName(); + var cacheKey = $"{assembly}\0{declarationId}"; + if (dispatchCache.TryGetValue(cacheKey, out var cached)) + { + return cached; + } + var output = new List(); + foreach (var context in contexts) + { + var localTarget = DocumentationCommentId + .GetSymbolsForDeclarationId(declarationId, context.Compilation) + .OfType() + .FirstOrDefault(candidate => assembly is null + || candidate.ContainingAssembly?.Identity.GetDisplayName() == assembly); + if (localTarget is null) + { + continue; + } + foreach (var type in AllTypes(context.Compilation.Assembly.GlobalNamespace) + .Where(type => !type.IsAbstract)) + { + IMethodSymbol? implementation; + if (localTarget.ContainingType.TypeKind == TypeKind.Interface) + { + implementation = type.FindImplementationForInterfaceMember(localTarget) + as IMethodSymbol; + } + else + { + implementation = type.GetMembers(localTarget.Name) + .OfType() + .FirstOrDefault(candidate => Overrides(candidate, localTarget)); + if (implementation is null + && !localTarget.IsAbstract + && SymbolEqualityComparer.Default.Equals(type, localTarget.ContainingType)) + { + implementation = localTarget; + } + } + if (implementation is not null && !implementation.IsAbstract) + { + output.Add(new DispatchCandidate( + context, + CanonicalMethod(implementation))); + } + } + } + var result = output + .DistinctBy(candidate => $"{candidate.Context.Target}\0{DocumentationCommentId.CreateDeclarationId(candidate.Symbol)}") + .ToArray(); + return dispatchCache.GetOrAdd(cacheKey, result); + } + + private static bool Overrides(IMethodSymbol candidate, IMethodSymbol target) + { + for (var current = candidate.OverriddenMethod; + current is not null; + current = current.OverriddenMethod) + { + if (SymbolEqualityComparer.Default.Equals( + current.OriginalDefinition, + target.OriginalDefinition)) + { + return true; + } + } + return false; + } + + private static IEnumerable AllTypes(INamespaceSymbol root) + { + foreach (var type in root.GetTypeMembers()) + { + foreach (var nested in AllTypes(type)) + { + yield return nested; + } + } + foreach (var child in root.GetNamespaceMembers()) + { + foreach (var type in AllTypes(child)) + { + yield return type; + } + } + } + + private static IEnumerable AllTypes(INamedTypeSymbol root) + { + yield return root; + foreach (var child in root.GetTypeMembers()) + { + foreach (var nested in AllTypes(child)) + { + yield return nested; + } + } + } + } + + private sealed record DispatchCandidate(ProjectContext Context, IMethodSymbol Symbol); + + private sealed record DocumentWork( + ProjectContext Context, + Document Document, + string Key, + bool Generated); +} diff --git a/sidecars/csharp/GraphProtocol.cs b/sidecars/csharp/GraphProtocol.cs new file mode 100644 index 00000000..052b6ec0 --- /dev/null +++ b/sidecars/csharp/GraphProtocol.cs @@ -0,0 +1,640 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json.Nodes; +using System.Globalization; + +namespace Samchon.Graph.CSharp; + +internal static class GraphProtocol +{ + public const string Provider = "roslyn-workspace"; + public const string Producer = "samchon-roslyn"; + public const string Version = "1.0.0"; + + public static readonly string[] Facts = + [ + "contains", + "exports", + "imports", + "calls", + "accesses", + "instantiates", + "type_ref", + "extends", + "implements", + "overrides", + "dispatches", + "decorates", + "tests", + "references", + ]; + + public static readonly string[] Capabilities = + [ + "coverage", + "diagnostics", + "diskDigests", + "incremental", + "sourceDigests", + "universe", + "unresolved", + "immutableSolution", + "sourceGeneratedDocuments", + ]; + + public static JsonObject Unchanged(GraphGeneration generation) => new() + { + ["protocolVersion"] = 1, + ["mode"] = "unchanged", + ["sequence"] = generation.Sequence, + ["generation"] = generation.Generation, + ["universe"] = generation.Universe, + ["frames"] = new JsonArray(), + }; + + public static JsonObject Replay(GraphGeneration generation, string? knownGeneration) + { + var original = generation.Envelope.DeepClone().AsObject(); + var originalBegin = original["frames"]!.AsArray() + .Single(frame => frame!["type"]!.GetValue() == "begin")!; + if ((knownGeneration is null + && original["mode"]!.GetValue() == "initial") + || originalBegin["baseGeneration"]?.GetValue() == knownGeneration) + { + return original; + } + var envelope = Commit(null, generation.Draft, generation.CompilerVersion); + if (envelope["generation"]!.GetValue() != generation.Generation + || envelope["universe"]!.GetValue() != generation.Universe) + { + throw new InvalidOperationException( + "C# workspace graph cannot replay a generation whose sealed draft moved"); + } + envelope["mode"] = knownGeneration is null ? "initial" : "rebuild"; + envelope["sequence"] = generation.Sequence; + var frames = envelope["frames"]!.AsArray(); + frames.Single(frame => frame!["type"]!.GetValue() == "begin")!["sequence"] = generation.Sequence; + frames.Single(frame => frame!["type"]!.GetValue() == "commit")!["sequence"] = generation.Sequence; + return envelope; + } + + public static JsonObject Commit( + GraphGeneration? previous, + GraphDraft draft, + string compilerVersion) + { + var universe = draft.UniverseFingerprint; + var full = previous is null || previous.Universe != universe; + var sequence = (previous?.Sequence ?? 0) + 1; + var shards = draft.Shards + .OrderBy(shard => shard.Key, StringComparer.Ordinal) + .ToDictionary(shard => shard.Key, StringComparer.Ordinal); + var previousShards = previous?.Draft.Shards + .ToDictionary(shard => shard.Key, StringComparer.Ordinal); + var manifest = shards.Values + .Select(shard => new ManifestEntry(shard.Key, shard.PayloadDigest)) + .ToArray(); + var generation = Hash(new JsonObject + { + ["universe"] = universe, + ["shards"] = new JsonArray(manifest.Select(entry => entry.Json()).ToArray()), + }); + if (previous is not null && previous.Generation == generation) + { + return Unchanged(previous); + } + + var hello = Hello(compilerVersion); + var allSources = Sources(shards.Values); + var begin = new JsonObject + { + ["type"] = "begin", + ["sequence"] = sequence, + ["generation"] = generation, + ["universe"] = universe, + ["manifest"] = ManifestDigest(allSources), + ["targets"] = new JsonArray(draft.Targets.Order(StringComparer.Ordinal).Select(value => JsonValue.Create(value)).ToArray()), + }; + if (!full) + { + begin["baseSequence"] = previous!.Sequence; + begin["baseGeneration"] = previous.Generation; + } + + var frames = new JsonArray(hello, begin); + var factsUnchanged = previous is not null + && !full + && previous.CompilerVersion == compilerVersion; + if (full) + { + foreach (var entry in manifest) + { + frames.Add(Upsert(entry, shards[entry.Key].Payload)); + } + } + else + { + var oldManifest = previous!.Manifest.ToDictionary(entry => entry.Key, StringComparer.Ordinal); + foreach (var entry in manifest) + { + if (!oldManifest.TryGetValue(entry.Key, out var old) || old.Digest != entry.Digest) + { + frames.Add(Upsert(entry, shards[entry.Key].Payload)); + factsUnchanged = factsUnchanged + && previousShards!.TryGetValue(entry.Key, out var prior) + && prior.FactFingerprint == shards[entry.Key].FactFingerprint; + } + } + foreach (var old in previous.Manifest) + { + if (!shards.ContainsKey(old.Key)) + { + factsUnchanged = false; + frames.Add(new JsonObject + { + ["type"] = "deleteShard", + ["key"] = old.Key, + }); + } + } + } + + var factDigest = factsUnchanged + ? PreviousFactDigest(previous!) + : FactDigest(hello, universe, manifest, shards); + frames.Add(new JsonObject + { + ["type"] = "commit", + ["sequence"] = sequence, + ["generation"] = generation, + ["shards"] = new JsonArray(manifest.Select(entry => entry.Json()).ToArray()), + ["factDigest"] = factDigest, + }); + return new JsonObject + { + ["protocolVersion"] = 1, + ["mode"] = previous is null ? "initial" : full ? "reload" : "incremental", + ["sequence"] = sequence, + ["generation"] = generation, + ["universe"] = universe, + ["frames"] = frames, + }; + } + + public static GraphGeneration GenerationFrom( + JsonObject envelope, + GraphDraft draft, + string compilerVersion) + { + var manifest = draft.Shards + .OrderBy(shard => shard.Key, StringComparer.Ordinal) + .Select(shard => new ManifestEntry(shard.Key, shard.PayloadDigest)) + .ToArray(); + return new GraphGeneration( + envelope["sequence"]!.GetValue(), + envelope["generation"]!.GetValue(), + envelope["universe"]!.GetValue(), + manifest, + draft, + compilerVersion, + envelope.DeepClone().AsObject()); + } + + public static string HashBytes(ReadOnlySpan bytes) => + Convert.ToHexStringLower(SHA256.HashData(bytes)); + + public static string HashText(string text) => HashBytes(Encoding.UTF8.GetBytes(text)); + + public static string Hash(JsonNode node) => HashText(CanonicalText(node)); + + public static string ShardFactHash(JsonObject shard) + { + var builder = new StringBuilder(); + builder.Append('{'); + var first = true; + foreach (var name in new[] { "coverage", "diagnostics", "edges", "nodes", "unresolved" }) + { + if (!first) + { + builder.Append(','); + } + first = false; + WriteQuoted(builder, name); + builder.Append(':'); + WriteCanonical(builder, shard[name]); + } + builder.Append('}'); + return HashText(builder.ToString()); + } + + private static string PreviousFactDigest(GraphGeneration generation) => generation.Envelope["frames"]! + .AsArray() + .Single(frame => frame!["type"]!.GetValue() == "commit")!["factDigest"]! + .GetValue(); + + private static JsonObject Hello(string compilerVersion) => new() + { + ["type"] = "hello", + ["protocolVersion"] = 1, + ["schemaVersion"] = 1, + ["producerSchemaVersion"] = 1, + ["provider"] = Provider, + ["producer"] = Producer, + ["producerVersion"] = Version, + ["compilerVersion"] = compilerVersion, + ["languages"] = new JsonArray("csharp"), + ["authority"] = "compiler", + ["supportedFacts"] = new JsonArray(Facts.Select(value => JsonValue.Create(value)).ToArray()), + ["capabilities"] = new JsonArray(Capabilities.Select(value => JsonValue.Create(value)).ToArray()), + }; + + private static JsonObject Upsert(ManifestEntry entry, JsonObject shard) => new() + { + ["type"] = "upsertShard", + ["digest"] = entry.Digest, + ["shard"] = shard.DeepClone(), + }; + + private static IReadOnlyList Sources(IEnumerable shards) + { + var sources = new SortedDictionary(StringComparer.Ordinal); + foreach (var shard in shards) + { + foreach (var node in shard.Payload["sources"]!.AsArray()) + { + var source = node!.AsObject(); + var file = source["file"]!.GetValue(); + if (sources.TryGetValue(file, out var prior) && Hash(prior) != Hash(source)) + { + throw new InvalidOperationException($"Shards disagree about source {file}"); + } + sources[file] = source; + } + } + return sources.Values.ToArray(); + } + + private static string ManifestDigest(IReadOnlyList sources) + { + var array = new JsonArray(sources + .OrderBy(source => source["file"]!.GetValue(), StringComparer.Ordinal) + .Select(source => source.DeepClone()) + .ToArray()); + return Hash(array); + } + + private static string FactDigest( + JsonObject hello, + string universe, + IReadOnlyList manifest, + IReadOnlyDictionary shards) + { + var nodes = new JsonArray(); + var edges = new JsonArray(); + var diagnostics = new JsonArray(); + var coverage = new JsonArray(); + var unresolved = new JsonArray(); + var seenNodes = new HashSet(StringComparer.Ordinal); + var seenEdges = new HashSet(StringComparer.Ordinal); + var seenDiagnostics = new HashSet(StringComparer.Ordinal); + var seenUnresolved = new HashSet(StringComparer.Ordinal); + foreach (var entry in manifest) + { + var shard = shards[entry.Key].Payload; + foreach (var node in shard["nodes"]!.AsArray()) + { + if (seenNodes.Add(node!["id"]!.GetValue())) + { + nodes.Add(node.DeepClone()); + } + } + foreach (var edge in shard["edges"]!.AsArray()) + { + var key = string.Join('\0', + edge!["from"]!.GetValue(), + edge["to"]!.GetValue(), + edge["kind"]!.GetValue()); + if (seenEdges.Add(key)) + { + edges.Add(edge.DeepClone()); + } + } + foreach (var diagnostic in shard["diagnostics"]!.AsArray()) + { + var key = string.Join('\0', + diagnostic!["file"]!.GetValue(), + diagnostic["line"]!.ToJsonString(), + diagnostic["column"]?.ToJsonString() ?? "", + diagnostic["code"]!.ToJsonString(), + diagnostic["severity"]?.GetValue() ?? "", + diagnostic["message"]!.GetValue()); + if (seenDiagnostics.Add(key)) + { + diagnostics.Add(diagnostic.DeepClone()); + } + } + foreach (var row in shard["coverage"]!.AsArray()) + { + coverage.Add(row!.DeepClone()); + } + foreach (var site in shard["unresolved"]!.AsArray()) + { + var key = CanonicalText(site!); + if (seenUnresolved.Add(key)) + { + unresolved.Add(site!.DeepClone()); + } + } + } + var provenance = new JsonObject + { + ["provider"] = hello["provider"]!.DeepClone(), + ["authority"] = hello["authority"]!.DeepClone(), + ["facts"] = hello["supportedFacts"]!.DeepClone(), + ["schemaVersion"] = hello["producerSchemaVersion"]!.DeepClone(), + ["tool"] = hello["producer"]!.DeepClone(), + ["toolVersion"] = hello["producerVersion"]!.DeepClone(), + ["compilerVersion"] = hello["compilerVersion"]!.DeepClone(), + ["protocolVersion"] = hello["protocolVersion"]!.DeepClone(), + ["universe"] = universe, + ["capabilities"] = hello["capabilities"]!.DeepClone(), + }; + return Hash(new JsonObject + { + ["languages"] = hello["languages"]!.DeepClone(), + ["nodes"] = nodes, + ["edges"] = edges, + ["diagnostics"] = diagnostics, + ["coverage"] = coverage, + ["unresolved"] = unresolved, + ["provenance"] = provenance, + }); + } + + private static string CanonicalText(JsonNode? node) + { + var builder = new StringBuilder(); + WriteCanonical(builder, node); + return builder.ToString(); + } + + private static void WriteCanonical(StringBuilder builder, JsonNode? node) + { + if (node is null) + { + builder.Append("null"); + return; + } + if (node is JsonObject valueObject) + { + builder.Append('{'); + var first = true; + foreach (var property in valueObject.OrderBy(property => property.Key, StringComparer.Ordinal)) + { + if (!first) + { + builder.Append(','); + } + first = false; + WriteQuoted(builder, property.Key); + builder.Append(':'); + WriteCanonical(builder, property.Value); + } + builder.Append('}'); + return; + } + if (node is JsonArray array) + { + builder.Append('['); + var first = true; + foreach (var item in array) + { + if (!first) + { + builder.Append(','); + } + first = false; + WriteCanonical(builder, item); + } + builder.Append(']'); + return; + } + if (node is JsonValue value && value.TryGetValue(out var text)) + { + WriteQuoted(builder, text); + return; + } + if (node is JsonValue boolean && boolean.TryGetValue(out var flag)) + { + builder.Append(flag ? "true" : "false"); + return; + } + if (node is JsonValue number && TryNumber(number, out var numeric)) + { + WriteJavaScriptNumber(builder, numeric); + return; + } + builder.Append(node.ToJsonString()); + } + + private static bool TryNumber(JsonValue value, out double number) + { + if (value.TryGetValue(out var integer)) + { + number = integer; + return true; + } + if (value.TryGetValue(out var longInteger)) + { + number = longInteger; + return true; + } + if (value.TryGetValue(out var unsignedInteger)) + { + number = unsignedInteger; + return true; + } + if (value.TryGetValue(out var unsignedLongInteger)) + { + number = unsignedLongInteger; + return true; + } + if (value.TryGetValue(out var single)) + { + number = single; + return true; + } + if (value.TryGetValue(out var floating)) + { + number = floating; + return true; + } + if (value.TryGetValue(out var decimalNumber)) + { + number = (double)decimalNumber; + return true; + } + number = 0; + return false; + } + + private static void WriteJavaScriptNumber(StringBuilder builder, double number) + { + if (number == 0) + { + builder.Append('0'); + return; + } + if (!double.IsFinite(number)) + { + builder.Append("null"); + return; + } + + if (number < 0) + { + builder.Append('-'); + number = -number; + } + var roundTrip = number.ToString("R", CultureInfo.InvariantCulture); + var exponentIndex = roundTrip.IndexOfAny(['E', 'e']); + var mantissa = exponentIndex == -1 + ? roundTrip + : roundTrip[..exponentIndex]; + var exponent = exponentIndex == -1 + ? 0 + : int.Parse(roundTrip[(exponentIndex + 1)..], CultureInfo.InvariantCulture); + var decimalIndex = mantissa.IndexOf('.'); + var decimalPosition = (decimalIndex == -1 ? mantissa.Length : decimalIndex) + exponent; + var digits = mantissa.Replace(".", "", StringComparison.Ordinal); + var leading = 0; + while (leading < digits.Length && digits[leading] == '0') + { + leading++; + } + decimalPosition -= leading; + digits = digits[leading..].TrimEnd('0'); + + if (number >= 1e-6 && number < 1e21) + { + if (decimalPosition <= 0) + { + builder.Append("0."); + builder.Append('0', -decimalPosition); + builder.Append(digits); + } + else if (decimalPosition >= digits.Length) + { + builder.Append(digits); + builder.Append('0', decimalPosition - digits.Length); + } + else + { + builder.Append(digits.AsSpan(0, decimalPosition)); + builder.Append('.'); + builder.Append(digits.AsSpan(decimalPosition)); + } + return; + } + + builder.Append(digits[0]); + if (digits.Length > 1) + { + builder.Append('.'); + builder.Append(digits.AsSpan(1)); + } + builder.Append('e'); + var scientificExponent = decimalPosition - 1; + if (scientificExponent >= 0) + { + builder.Append('+'); + } + builder.Append(scientificExponent.ToString(CultureInfo.InvariantCulture)); + } + + private static void WriteQuoted(StringBuilder builder, string value) + { + builder.Append('"'); + for (var index = 0; index < value.Length; index++) + { + var character = value[index]; + switch (character) + { + case '"': + builder.Append("\\\""); + break; + case '\\': + builder.Append("\\\\"); + break; + case '\b': + builder.Append("\\b"); + break; + case '\f': + builder.Append("\\f"); + break; + case '\n': + builder.Append("\\n"); + break; + case '\r': + builder.Append("\\r"); + break; + case '\t': + builder.Append("\\t"); + break; + default: + if (character < ' ' || char.IsSurrogate(character) + && (index + 1 == value.Length + || !char.IsSurrogatePair(character, value[index + 1]))) + { + builder.Append("\\u"); + builder.Append(((int)character).ToString("x4", + System.Globalization.CultureInfo.InvariantCulture)); + } + else + { + builder.Append(character); + if (char.IsHighSurrogate(character)) + { + builder.Append(value[++index]); + } + } + break; + } + } + builder.Append('"'); + } +} + +internal sealed record ManifestEntry(string Key, string Digest) +{ + public JsonObject Json() => new() + { + ["key"] = Key, + ["digest"] = Digest, + }; +} + +internal sealed record ShardDraft( + string Key, + JsonObject Payload, + string InterfaceFingerprint, + string FactFingerprint, + string PayloadDigest); + +internal sealed record GraphDraft( + IReadOnlyList Targets, + JsonObject Universe, + string UniverseFingerprint, + IReadOnlyList Shards, + bool HasErrors, + string ErrorSummary, + object? ProviderState = null); + +internal sealed record GraphGeneration( + int Sequence, + string Generation, + string Universe, + IReadOnlyList Manifest, + GraphDraft Draft, + string CompilerVersion, + JsonObject Envelope); diff --git a/sidecars/csharp/Program.cs b/sidecars/csharp/Program.cs new file mode 100644 index 00000000..92b097f8 --- /dev/null +++ b/sidecars/csharp/Program.cs @@ -0,0 +1,351 @@ +using System.Collections.Concurrent; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Diagnostics; + +namespace Samchon.Graph.CSharp; + +internal static class Program +{ + public static async Task Main(string[] args) + { + if (args is ["--dotnet-host", var host, .. var trailing]) + { + MakeDotNetDiscoverable(host); + args = trailing; + } + if (args is ["--measure-load", var root]) + { + var elapsed = await WorkspaceGraphService.MeasureLoadAsync( + root, + CancellationToken.None).ConfigureAwait(false); + Console.WriteLine(JsonSerializer.Serialize(new + { + phase = "msbuild-workspace-load", + elapsedMs = elapsed, + })); + return; + } + if (args.Length != 0) + { + throw new ArgumentException("Usage: samchon-roslyn [--measure-load ]"); + } + using var input = Console.OpenStandardInput(); + using var output = Console.OpenStandardOutput(); + await new JsonRpcServer(input, output).RunAsync().ConfigureAwait(false); + } + + private static void MakeDotNetDiscoverable(string host) + { + if (!Path.IsPathFullyQualified(host) || !File.Exists(host)) + { + throw new ArgumentException("--dotnet-host must name an absolute executable."); + } + var directory = Path.GetDirectoryName(host)!; + Environment.SetEnvironmentVariable("DOTNET_ROOT", directory); + var path = Environment.GetEnvironmentVariable("PATH") ?? string.Empty; + if (!path.Split(Path.PathSeparator).Any(entry => + string.Equals(entry.TrimEnd(Path.DirectorySeparatorChar), + directory.TrimEnd(Path.DirectorySeparatorChar), + StringComparison.OrdinalIgnoreCase))) + { + Environment.SetEnvironmentVariable( + "PATH", + directory + (path.Length == 0 ? string.Empty : Path.PathSeparator + path)); + } + } +} + +internal sealed class JsonRpcServer(Stream input, Stream output) +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + private readonly ConcurrentDictionary requests = new(); + private readonly SemaphoreSlim writes = new(1, 1); + private WorkspaceGraphService? graph; + private bool shutdown; + + public async Task RunAsync() + { + var pending = new List(); + while (await ReadMessageAsync(input, CancellationToken.None).ConfigureAwait(false) is { } message) + { + if (!message.RootElement.TryGetProperty("method", out var methodElement)) + { + message.Dispose(); + continue; + } + var method = methodElement.GetString() ?? ""; + if (method == "$/cancelRequest") + { + Cancel(message.RootElement); + message.Dispose(); + continue; + } + if (method == "exit") + { + message.Dispose(); + break; + } + pending.Add(HandleAsync(message, method)); + } + await Task.WhenAll(pending).ConfigureAwait(false); + if (graph is not null) + { + await graph.DisposeAsync().ConfigureAwait(false); + } + } + + private async Task HandleAsync(JsonDocument message, string method) + { + using (message) + { + var root = message.RootElement; + var hasId = root.TryGetProperty("id", out var idElement); + var id = hasId ? idElement.Clone() : default; + var requestKey = hasId ? id.GetRawText() : ""; + using var cancellation = new CancellationTokenSource(); + if (hasId) + { + requests[requestKey] = cancellation; + } + try + { + var result = await DispatchAsync(root, method, cancellation.Token).ConfigureAwait(false); + if (hasId) + { + await RespondAsync(id, result, cancellation.Token).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + if (hasId) + { + await ErrorAsync(id, -32800, "Request cancelled", CancellationToken.None) + .ConfigureAwait(false); + } + } + catch (SnapshotInvalidatedException error) + { + if (hasId) + { + await ErrorAsync(id, -32801, error.Message, CancellationToken.None) + .ConfigureAwait(false); + } + } + catch (Exception error) + { + if (hasId) + { + await ErrorAsync(id, -32603, error.Message, CancellationToken.None) + .ConfigureAwait(false); + } + Console.Error.WriteLine(error); + } + finally + { + if (hasId) + { + requests.TryRemove(requestKey, out _); + } + } + } + } + + private async Task DispatchAsync( + JsonElement request, + string method, + CancellationToken cancellationToken) + { + switch (method) + { + case "initialize": + { + if (shutdown) + { + throw new InvalidOperationException("Roslyn graph server is shutting down"); + } + var parameters = request.GetProperty("params"); + var rootUri = parameters.TryGetProperty("rootUri", out var rootElement) + ? rootElement.GetString() + : null; + if (string.IsNullOrWhiteSpace(rootUri)) + { + throw new InvalidOperationException("initialize.rootUri is required"); + } + var root = Path.GetFullPath(new Uri(rootUri).LocalPath); + graph = new WorkspaceGraphService(root); + return new JsonObject + { + ["capabilities"] = new JsonObject + { + ["executeCommandProvider"] = new JsonObject + { + ["commands"] = new JsonArray("csharp.graph.snapshot"), + }, + }, + ["serverInfo"] = new JsonObject + { + ["name"] = GraphProtocol.Producer, + ["version"] = GraphProtocol.Version, + }, + }; + } + case "initialized": + return null; + case "workspace/didChangeWatchedFiles": + RequireGraph().NotifyChangedFiles(request.GetProperty("params")); + return null; + case "workspace/executeCommand": + { + var parameters = request.GetProperty("params"); + var command = parameters.GetProperty("command").GetString(); + if (command != "csharp.graph.snapshot") + { + throw new InvalidOperationException($"Unsupported command: {command}"); + } + string? knownGeneration = null; + if (parameters.TryGetProperty("arguments", out var arguments) + && arguments.ValueKind == JsonValueKind.Array + && arguments.GetArrayLength() != 0 + && arguments[0].ValueKind == JsonValueKind.Object + && arguments[0].TryGetProperty("knownGeneration", out var known) + && known.ValueKind == JsonValueKind.String) + { + knownGeneration = known.GetString(); + } + return await RequireGraph().SnapshotAsync(knownGeneration, cancellationToken) + .ConfigureAwait(false); + } + case "shutdown": + shutdown = true; + return null; + default: + throw new InvalidOperationException($"Unsupported method: {method}"); + } + } + + private WorkspaceGraphService RequireGraph() => graph + ?? throw new InvalidOperationException("Roslyn graph server is not initialized"); + + private void Cancel(JsonElement message) + { + if (!message.TryGetProperty("params", out var parameters) + || !parameters.TryGetProperty("id", out var id)) + { + return; + } + if (requests.TryGetValue(id.GetRawText(), out var cancellation)) + { + cancellation.Cancel(); + } + } + + private Task RespondAsync(JsonElement id, JsonNode? result, CancellationToken cancellationToken) => + WriteAsync(new JsonObject + { + ["jsonrpc"] = "2.0", + ["id"] = JsonNode.Parse(id.GetRawText()), + ["result"] = result, + }, cancellationToken); + + private Task ErrorAsync( + JsonElement id, + int code, + string message, + CancellationToken cancellationToken) => WriteAsync(new JsonObject + { + ["jsonrpc"] = "2.0", + ["id"] = JsonNode.Parse(id.GetRawText()), + ["error"] = new JsonObject + { + ["code"] = code, + ["message"] = message, + }, + }, cancellationToken); + + private async Task WriteAsync(JsonObject message, CancellationToken cancellationToken) + { + var timing = Stopwatch.StartNew(); + var body = JsonSerializer.SerializeToUtf8Bytes(message, JsonOptions); + Trace("json-serialize", timing.ElapsedMilliseconds, body.Length); + var header = Encoding.ASCII.GetBytes($"Content-Length: {body.Length}\r\n\r\n"); + timing.Restart(); + await writes.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await output.WriteAsync(header, cancellationToken).ConfigureAwait(false); + await output.WriteAsync(body, cancellationToken).ConfigureAwait(false); + await output.FlushAsync(cancellationToken).ConfigureAwait(false); + Trace("json-write", timing.ElapsedMilliseconds, body.Length); + } + finally + { + writes.Release(); + } + } + + private static void Trace(string phase, long elapsedMs, int bytes) + { + if (Environment.GetEnvironmentVariable("SAMCHON_GRAPH_ROSLYN_TRACE") == "1") + { + Console.Error.WriteLine( + $"{{\"phase\":\"roslyn-{phase}\",\"elapsedMs\":{elapsedMs},\"bytes\":{bytes}}}"); + } + } + + private static async Task ReadMessageAsync( + Stream stream, + CancellationToken cancellationToken) + { + var header = new List(); + var suffix = 0; + while (suffix != 4) + { + var value = new byte[1]; + if (await stream.ReadAsync(value, cancellationToken).ConfigureAwait(false) == 0) + { + return header.Count == 0 + ? null + : throw new EndOfStreamException("Truncated LSP header"); + } + header.Add(value[0]); + suffix = value[0] == "\r\n\r\n"[suffix] + ? suffix + 1 + : value[0] == '\r' ? 1 : 0; + if (header.Count > 16 * 1024) + { + throw new InvalidDataException("LSP header is too large"); + } + } + var text = Encoding.ASCII.GetString([.. header]); + var length = text.Split("\r\n", StringSplitOptions.RemoveEmptyEntries) + .Select(line => line.Split(':', 2)) + .Where(parts => parts.Length == 2 && parts[0].Equals("Content-Length", StringComparison.OrdinalIgnoreCase)) + .Select(parts => int.Parse(parts[1].Trim(), System.Globalization.CultureInfo.InvariantCulture)) + .Single(); + if (length <= 0) + { + throw new InvalidDataException("LSP Content-Length must be positive"); + } + var body = new byte[length]; + var offset = 0; + while (offset != body.Length) + { + var read = await stream.ReadAsync(body.AsMemory(offset), cancellationToken).ConfigureAwait(false); + if (read == 0) + { + throw new EndOfStreamException("Truncated LSP body"); + } + offset += read; + } + return JsonDocument.Parse(body); + } +} + +internal sealed class SnapshotInvalidatedException(string message) : Exception(message); diff --git a/sidecars/csharp/Samchon.Graph.CSharp.csproj b/sidecars/csharp/Samchon.Graph.CSharp.csproj new file mode 100644 index 00000000..5c443fa9 --- /dev/null +++ b/sidecars/csharp/Samchon.Graph.CSharp.csproj @@ -0,0 +1,21 @@ + + + Exe + net10.0 + samchon-roslyn + Samchon.Graph.CSharp + enable + enable + latest + true + true + true + + + + + + + + + diff --git a/sidecars/csharp/WorkspaceGraphService.cs b/sidecars/csharp/WorkspaceGraphService.cs new file mode 100644 index 00000000..76ee096f --- /dev/null +++ b/sidecars/csharp/WorkspaceGraphService.cs @@ -0,0 +1,611 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Diagnostics; +using Microsoft.Build.Locator; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.MSBuild; +using Microsoft.CodeAnalysis.Text; + +namespace Samchon.Graph.CSharp; + +internal sealed class WorkspaceGraphService : IAsyncDisposable +{ + private readonly string root; + private readonly object eventGate = new(); + private readonly HashSet changedFiles = new(StringComparer.OrdinalIgnoreCase); + private readonly SemaphoreSlim snapshots = new(1, 1); + private readonly FileSystemWatcher watcher; + private HashSet compilerInputs = new(StringComparer.OrdinalIgnoreCase); + private Dictionary observedSources = + new(StringComparer.OrdinalIgnoreCase); + private List compilerInputWatchers = []; + private MSBuildWorkspace? workspace; + private Solution? solution; + private GraphGeneration? generation; + private long eventEpoch; + private bool dirty = true; + private bool buildDirty = true; + + public WorkspaceGraphService(string root) + { + this.root = Path.GetFullPath(root); + if (!Directory.Exists(this.root)) + { + throw new DirectoryNotFoundException($"C# workspace root does not exist: {this.root}"); + } + watcher = new FileSystemWatcher(this.root) + { + IncludeSubdirectories = true, + NotifyFilter = NotifyFilters.FileName + | NotifyFilters.DirectoryName + | NotifyFilters.LastWrite + | NotifyFilters.CreationTime + | NotifyFilters.Size, + EnableRaisingEvents = true, + }; + watcher.Changed += OnChanged; + watcher.Created += OnChanged; + watcher.Deleted += OnChanged; + watcher.Renamed += OnRenamed; + watcher.Error += OnWatcherError; + } + + public static async Task MeasureLoadAsync( + string root, + CancellationToken cancellationToken) + { + var resolved = Path.GetFullPath(root); + if (!MSBuildLocator.IsRegistered) + { + MSBuildLocator.RegisterDefaults(); + } + using var measured = MSBuildWorkspace.Create(); + measured.SkipUnrecognizedProjects = false; + var entry = SelectEntryPoint(resolved); + var started = Stopwatch.StartNew(); + var loaded = Path.GetExtension(entry).Equals(".csproj", StringComparison.OrdinalIgnoreCase) + ? (await measured.OpenProjectAsync(entry, cancellationToken: cancellationToken) + .ConfigureAwait(false)).Solution + : await measured.OpenSolutionAsync(entry, cancellationToken: cancellationToken) + .ConfigureAwait(false); + started.Stop(); + if (!loaded.Projects.Any(project => project.Language == LanguageNames.CSharp)) + { + throw new InvalidOperationException($"C# workspace contains no C# projects: {entry}"); + } + return started.ElapsedMilliseconds; + } + + public void NotifyChangedFiles(JsonElement parameters) + { + if (!parameters.TryGetProperty("changes", out var changes) + || changes.ValueKind != JsonValueKind.Array) + { + return; + } + foreach (var change in changes.EnumerateArray()) + { + if (change.TryGetProperty("uri", out var uri) + && Uri.TryCreate(uri.GetString(), UriKind.Absolute, out var parsed) + && parsed.IsFile) + { + MarkChanged(parsed.LocalPath); + } + } + } + + public async Task SnapshotAsync( + string? knownGeneration, + CancellationToken cancellationToken) + { + await snapshots.WaitAsync(cancellationToken).ConfigureAwait(false); + HashSet changes; + bool reload; + long epoch; + try + { + if (solution is not null) + { + ReconcileSourceFiles(); + } + lock (eventGate) + { + if (!dirty && generation is not null) + { + return knownGeneration == generation.Generation + ? GraphProtocol.Unchanged(generation) + : GraphProtocol.Replay(generation, knownGeneration); + } + changes = new HashSet(changedFiles, StringComparer.OrdinalIgnoreCase); + changedFiles.Clear(); + reload = buildDirty || solution is null; + buildDirty = false; + dirty = false; + epoch = eventEpoch; + } + if (!reload && changes.Any(file => + Path.GetExtension(file).Equals(".cs", StringComparison.OrdinalIgnoreCase) + && File.Exists(file) + && !solution!.GetDocumentIdsWithFilePath(Path.GetFullPath(file)).Any())) + { + // Only MSBuild evaluation can decide whether a newly observed + // source belongs to a project. Directory ancestry would invent + // ownership for Compile Remove, explicit include, and nested + // project layouts. + reload = true; + } + + try + { + if (reload) + { + await LoadWorkspaceAsync(cancellationToken).ConfigureAwait(false); + } + else + { + solution = await ApplySourceChangesAsync( + solution!, + changes, + cancellationToken).ConfigureAwait(false); + } + + var draft = await GraphExtractor.ExtractAsync( + solution!, + root, + WorkspaceDiagnostics(), + generation?.Draft, + changes, + reload, + cancellationToken).ConfigureAwait(false); + ReconcileSourceFiles(); + lock (eventGate) + { + if (eventEpoch != epoch) + { + dirty = true; + throw new SnapshotInvalidatedException( + "C# workspace inputs changed while the immutable Solution was being exported; retry"); + } + } + if (draft.HasErrors) + { + throw new InvalidOperationException( + $"C# workspace graph retained its prior generation after compiler errors: {draft.ErrorSummary}"); + } + var protocolTiming = Stopwatch.StartNew(); + var envelope = GraphProtocol.Commit( + generation, + draft, + typeof(Microsoft.CodeAnalysis.CSharp.CSharpCompilation) + .Assembly.GetName().Version?.ToString() ?? "unknown"); + Trace("protocol-commit", protocolTiming.ElapsedMilliseconds); + if (envelope["mode"]!.GetValue() != "unchanged") + { + protocolTiming.Restart(); + generation = GraphProtocol.GenerationFrom( + envelope, + draft, + typeof(Microsoft.CodeAnalysis.CSharp.CSharpCompilation) + .Assembly.GetName().Version?.ToString() ?? "unknown"); + Trace("protocol-generation", protocolTiming.ElapsedMilliseconds); + } + return envelope["mode"]!.GetValue() == "unchanged" + && knownGeneration != generation!.Generation + ? GraphProtocol.Replay(generation, knownGeneration) + : envelope; + } + catch + { + lock (eventGate) + { + dirty = true; + buildDirty |= reload; + foreach (var file in changes) + { + changedFiles.Add(file); + } + } + throw; + } + } + finally + { + snapshots.Release(); + } + } + + public async ValueTask DisposeAsync() + { + watcher.EnableRaisingEvents = false; + watcher.Dispose(); + foreach (var inputWatcher in compilerInputWatchers) + { + inputWatcher.Dispose(); + } + snapshots.Dispose(); + if (workspace is not null) + { + workspace.Dispose(); + } + await Task.CompletedTask.ConfigureAwait(false); + } + + private async Task LoadWorkspaceAsync(CancellationToken cancellationToken) + { + var sourceBaseline = CaptureSourceFiles(); + if (!MSBuildLocator.IsRegistered) + { + MSBuildLocator.RegisterDefaults(); + } + var candidate = MSBuildWorkspace.Create(); + candidate.SkipUnrecognizedProjects = false; + try + { + var entry = SelectEntryPoint(root); + var candidateSolution = Path.GetExtension(entry).Equals(".csproj", StringComparison.OrdinalIgnoreCase) + ? (await candidate.OpenProjectAsync(entry, cancellationToken: cancellationToken) + .ConfigureAwait(false)).Solution + : await candidate.OpenSolutionAsync(entry, cancellationToken: cancellationToken) + .ConfigureAwait(false); + if (!candidateSolution.Projects.Any(project => project.Language == LanguageNames.CSharp)) + { + throw new InvalidOperationException($"C# workspace contains no C# projects: {entry}"); + } + ReplaceCompilerInputWatchers(candidateSolution); + var previous = workspace; + workspace = candidate; + solution = candidateSolution; + observedSources = sourceBaseline; + previous?.Dispose(); + } + catch + { + candidate.Dispose(); + throw; + } + } + + private void ReplaceCompilerInputWatchers(Solution candidate) + { + var inputs = candidate.Projects + .Where(project => project.Language == LanguageNames.CSharp) + .SelectMany(project => project.AnalyzerReferences + .Select(reference => reference.FullPath) + .Concat(project.AdditionalDocuments + .Concat(project.AnalyzerConfigDocuments) + .Select(document => document.FilePath)) + .Append(project.FilePath is null + ? null + : Path.Combine( + Path.GetDirectoryName(project.FilePath)!, + "obj", + "project.assets.json"))) + .Where(file => file is { Length: > 0 }) + .Select(file => Path.GetFullPath(file!)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var replacements = new List(); + try + { + foreach (var file in inputs + .Where(file => !IsWithin(root, file) || Ignored(file)) + .Where(file => Directory.Exists(Path.GetDirectoryName(file))) + .Order(StringComparer.Ordinal)) + { + var inputWatcher = new FileSystemWatcher( + Path.GetDirectoryName(file)!, + Path.GetFileName(file)) + { + NotifyFilter = NotifyFilters.FileName + | NotifyFilters.LastWrite + | NotifyFilters.CreationTime + | NotifyFilters.Size, + }; + inputWatcher.Changed += OnChanged; + inputWatcher.Created += OnChanged; + inputWatcher.Deleted += OnChanged; + inputWatcher.Renamed += OnRenamed; + inputWatcher.Error += OnWatcherError; + replacements.Add(inputWatcher); + } + } + catch + { + foreach (var replacement in replacements) + { + replacement.Dispose(); + } + throw; + } + List previous; + lock (eventGate) + { + previous = compilerInputWatchers; + compilerInputWatchers = replacements; + compilerInputs = inputs; + foreach (var replacement in replacements) + { + replacement.EnableRaisingEvents = true; + } + } + foreach (var prior in previous) + { + prior.Dispose(); + } + } + + private static string SelectEntryPoint(string root) + { + foreach (var extension in new[] { ".slnx", ".sln" }) + { + var solutions = Directory.EnumerateFiles(root, $"*{extension}", SearchOption.TopDirectoryOnly) + .Order(StringComparer.Ordinal) + .ToArray(); + if (solutions.Length == 1) + { + return solutions[0]; + } + if (solutions.Length > 1) + { + throw new InvalidOperationException( + $"C# workspace root has multiple {extension} entry points; select one explicitly"); + } + } + var projects = EnumerateProjectFiles(root).ToArray(); + return projects.Length switch + { + 1 => projects[0], + 0 => throw new InvalidOperationException("C# workspace has no .sln, .slnx, or .csproj entry point"), + _ => throw new InvalidOperationException( + "C# workspace has multiple projects and no solution entry point"), + }; + } + + private static IEnumerable EnumerateProjectFiles(string root) + { + var pending = new Stack(); + pending.Push(root); + while (pending.Count != 0) + { + var directory = pending.Pop(); + foreach (var entry in Directory.EnumerateFileSystemEntries(directory) + .Order(StringComparer.Ordinal)) + { + if (Directory.Exists(entry)) + { + if (!Ignored(root, entry)) + { + pending.Push(entry); + } + } + else if (Path.GetExtension(entry).Equals( + ".csproj", + StringComparison.OrdinalIgnoreCase)) + { + yield return entry; + } + } + } + } + + private async Task ApplySourceChangesAsync( + Solution current, + IReadOnlySet changes, + CancellationToken cancellationToken) + { + var next = current; + foreach (var file in changes + .Where(file => Path.GetExtension(file).Equals(".cs", StringComparison.OrdinalIgnoreCase)) + .Order(StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + var absolute = Path.GetFullPath(file); + var documents = next.GetDocumentIdsWithFilePath(absolute).ToArray(); + if (!File.Exists(absolute)) + { + foreach (var document in documents) + { + next = next.RemoveDocument(document); + } + continue; + } + var bytes = await File.ReadAllBytesAsync(absolute, cancellationToken).ConfigureAwait(false); + using var stream = new MemoryStream(bytes, writable: false); + var text = SourceText.From( + stream, + encoding: null, + checksumAlgorithm: SourceHashAlgorithm.Sha256, + throwIfBinaryDetected: true, + canBeEmbedded: true); + if (documents.Length != 0) + { + foreach (var document in documents) + { + next = next.WithDocumentText(document, text, PreservationMode.PreserveIdentity); + } + continue; + } + throw new SnapshotInvalidatedException( + $"C# source membership moved after refresh preparation: {absolute}"); + } + return next; + } + + private IReadOnlyList WorkspaceDiagnostics() => workspace is null + ? [] + : workspace.Diagnostics + .Select(diagnostic => diagnostic.Message) + .Order(StringComparer.Ordinal) + .ToArray(); + + private void ReconcileSourceFiles() + { + var current = CaptureSourceFiles(); + var changes = observedSources.Keys + .Concat(current.Keys) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Where(file => !observedSources.TryGetValue(file, out var prior) + || !current.TryGetValue(file, out var next) + || prior != next) + .ToArray(); + observedSources = current; + if (changes.Length == 0) + { + return; + } + lock (eventGate) + { + dirty = true; + foreach (var file in changes) + { + changedFiles.Add(file); + } + eventEpoch++; + } + } + + private Dictionary CaptureSourceFiles() + { + var sources = new Dictionary(StringComparer.OrdinalIgnoreCase); + var pending = new Stack(); + pending.Push(root); + while (pending.Count != 0) + { + var directory = pending.Pop(); + foreach (var entry in Directory.EnumerateFileSystemEntries(directory)) + { + if (Directory.Exists(entry)) + { + if (!Ignored(root, entry)) + { + pending.Push(entry); + } + continue; + } + if (!Path.GetExtension(entry).Equals(".cs", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + try + { + var file = new FileInfo(entry); + sources[file.FullName] = new SourceStamp( + file.Length, + file.LastWriteTimeUtc.Ticks); + } + catch (FileNotFoundException) + { + // A concurrent deletion is represented by the absent entry. + } + } + } + return sources; + } + + private void OnChanged(object sender, FileSystemEventArgs args) => MarkChanged(args.FullPath); + + private void OnRenamed(object sender, RenamedEventArgs args) + { + MarkChanged(args.OldFullPath); + MarkChanged(args.FullPath); + } + + private void OnWatcherError(object sender, ErrorEventArgs args) + { + lock (eventGate) + { + dirty = true; + buildDirty = true; + eventEpoch++; + } + } + + private void MarkChanged(string file) + { + var absolute = Path.GetFullPath(file); + bool compilerInput; + lock (eventGate) + { + compilerInput = compilerInputs.Contains(absolute); + } + if ((!IsWithin(root, absolute) && !compilerInput) + || (Ignored(absolute) && !compilerInput)) + { + return; + } + var extension = Path.GetExtension(absolute); + if (!extension.Equals(".cs", StringComparison.OrdinalIgnoreCase) + && !IsBuildInput(absolute) + && !compilerInput) + { + return; + } + if (extension.Equals(".cs", StringComparison.OrdinalIgnoreCase)) + { + SourceStamp? current = null; + try + { + var source = new FileInfo(absolute); + if (source.Exists) + { + current = new SourceStamp(source.Length, source.LastWriteTimeUtc.Ticks); + } + } + catch (FileNotFoundException) + { + // Treat a file that disappeared during inspection as deleted. + } + if (current is { } stamp + && observedSources.TryGetValue(absolute, out var observed) + && stamp == observed + || current is null && !observedSources.ContainsKey(absolute)) + { + return; + } + } + lock (eventGate) + { + dirty = true; + buildDirty |= IsBuildInput(absolute) || compilerInput; + changedFiles.Add(absolute); + eventEpoch++; + } + } + + private static bool IsBuildInput(string file) + { + var name = Path.GetFileName(file); + return Path.GetExtension(file).ToLowerInvariant() is ".sln" or ".slnx" or ".csproj" or ".props" or ".targets" + || name.Equals("global.json", StringComparison.OrdinalIgnoreCase) + || name.Equals("packages.lock.json", StringComparison.OrdinalIgnoreCase) + || name.Equals("nuget.config", StringComparison.OrdinalIgnoreCase); + } + + private bool Ignored(string file) + => Ignored(root, file); + + private static bool Ignored(string root, string file) + { + var parts = Path.GetRelativePath(root, file) + .Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return parts.Any(part => part is ".git" or ".wiki" or "bin" or "node_modules" or "obj"); + } + + private static bool IsWithin(string parent, string child) + { + var relative = Path.GetRelativePath(Path.GetFullPath(parent), Path.GetFullPath(child)); + return relative != ".." + && !relative.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) + && !Path.IsPathRooted(relative); + } + + private readonly record struct SourceStamp(long Length, long LastWriteTicks); + + private static void Trace(string phase, long elapsedMs) + { + if (Environment.GetEnvironmentVariable("SAMCHON_GRAPH_ROSLYN_TRACE") == "1") + { + Console.Error.WriteLine( + $"{{\"phase\":\"roslyn-{phase}\",\"elapsedMs\":{elapsedMs}}}"); + } + } +} diff --git a/sidecars/csharp/packages.lock.json b/sidecars/csharp/packages.lock.json new file mode 100644 index 00000000..cdc090d6 --- /dev/null +++ b/sidecars/csharp/packages.lock.json @@ -0,0 +1,188 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Microsoft.Build.Framework": { + "type": "Direct", + "requested": "[17.11.48, )", + "resolved": "17.11.48", + "contentHash": "C3WIMt2wBl4++NX3jSEpTq5KXBhvAV154R4JrYHkfy9JSBcXWiL0mkgpspk5xSdOj+fS/uz7zluIy6bMM1fkkQ==" + }, + "Microsoft.Build.Locator": { + "type": "Direct", + "requested": "[1.11.2, )", + "resolved": "1.11.2", + "contentHash": "tY+/S54G29CGsbL3slVu4vqtpciwVnb3fKOmrhgzEQmu/VziFaWmD/E1e/2KH7cDucuycGSkWsSXndBs5Uawow==" + }, + "Microsoft.CodeAnalysis.CSharp.Workspaces": { + "type": "Direct", + "requested": "[5.9.0, )", + "resolved": "5.9.0", + "contentHash": "D2zqK/k16fto0yMz0hcXMTkzOxEwMDJyA1mu/KXF9Befwz4zub3MpHQD8FeRxJtVSSsC3dQFYBw7zu7r/pfO7g==", + "dependencies": { + "Humanizer.Core": "2.14.1", + "Microsoft.CodeAnalysis.Analyzers": "5.9.0-1.26328.17", + "Microsoft.CodeAnalysis.CSharp": "[5.9.0]", + "Microsoft.CodeAnalysis.Common": "[5.9.0]", + "Microsoft.CodeAnalysis.Workspaces.Common": "[5.9.0]", + "System.Composition": "10.0.1" + } + }, + "Microsoft.CodeAnalysis.Workspaces.MSBuild": { + "type": "Direct", + "requested": "[5.9.0, )", + "resolved": "5.9.0", + "contentHash": "BBux6hhD4wXt4lYN2oaY+jV4PnIj5plyqohijgSZtxJQqsvsldPEdii2q9lGrJbNjKEALnldDHz1rmGKG0PeeA==", + "dependencies": { + "Humanizer.Core": "2.14.1", + "Microsoft.Build.Framework": "17.11.48", + "Microsoft.CodeAnalysis.Analyzers": "5.9.0-1.26328.17", + "Microsoft.CodeAnalysis.Workspaces.Common": "[5.9.0]", + "Microsoft.Extensions.DependencyInjection": "10.0.1", + "Microsoft.Extensions.Logging": "10.0.1", + "Microsoft.Extensions.Logging.Abstractions": "10.0.1", + "Microsoft.Extensions.Options": "10.0.1", + "Microsoft.Extensions.Primitives": "10.0.1", + "Microsoft.VisualStudio.SolutionPersistence": "1.0.52", + "System.Composition": "10.0.1" + } + }, + "Humanizer.Core": { + "type": "Transitive", + "resolved": "2.14.1", + "contentHash": "lQKvtaTDOXnoVJ20ibTuSIOf2i0uO0MPbDhd1jm238I+U/2ZnRENj0cktKZhtchBMtCUSRQ5v4xBCUbKNmyVMw==" + }, + "Microsoft.CodeAnalysis.Analyzers": { + "type": "Transitive", + "resolved": "5.9.0-1.26328.17", + "contentHash": "HP9NNk8ZjOSI2hgOyXnQg+kv7/X837Vr2nAlXiGAtqtYnYKjRRa1UmQFr8KFs5ynGYKqfbb8zB9APoWjiAGdMg==" + }, + "Microsoft.CodeAnalysis.Common": { + "type": "Transitive", + "resolved": "5.9.0", + "contentHash": "IYaIaUWdIx539AReKZOBEqTskFusZfCh/wFSPilDvCn5Say8MegLw2LONcSIcVy+v3Gzv53qYBspgvBGSErfbQ==", + "dependencies": { + "Microsoft.CodeAnalysis.Analyzers": "5.9.0-1.26328.17" + } + }, + "Microsoft.CodeAnalysis.CSharp": { + "type": "Transitive", + "resolved": "5.9.0", + "contentHash": "7JGDA0UT1+h7k9ZcA3rF4eFC8+QPq1xyYaXxag4p8r/zzPurEJxvdi7aM+MRL/SfP7XADXpWF/pl/eUYXOq/ww==", + "dependencies": { + "Microsoft.CodeAnalysis.Analyzers": "5.9.0-1.26328.17", + "Microsoft.CodeAnalysis.Common": "[5.9.0]" + } + }, + "Microsoft.CodeAnalysis.Workspaces.Common": { + "type": "Transitive", + "resolved": "5.9.0", + "contentHash": "1A6jz50NG4nOEW8tX5+h+MyHqjWL0mPGwrUdwu+OlTfyknLo0GfxSqj4zEks8uVUdHdo9v8Ir9dHxACf8iYNEA==", + "dependencies": { + "Humanizer.Core": "2.14.1", + "Microsoft.CodeAnalysis.Analyzers": "5.9.0-1.26328.17", + "Microsoft.CodeAnalysis.Common": "[5.9.0]", + "System.Composition": "10.0.1" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "zerXV0GAR9LCSXoSIApbWn+Dq1/T+6vbXMHGduq1LoVQRHT0BXsGQEau0jeLUBUcsoF/NaUT8ADPu8b+eNcIyg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.1" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "oIy8fQxxbUsSrrOvgBqlVgOeCtDmrcynnTG+FQufcUWBrwyPfwlUkCDB2vaiBeYPyT+20u9/HeuHeBf+H4F/8g==" + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "9ItMpMLFZFJFqCuHLLbR3LiA4ahA8dMtYuXpXl2YamSDWZhYS9BruPprkftY0tYi2bQ0slNrixdFm+4kpz1g5w==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.1", + "Microsoft.Extensions.Logging.Abstractions": "10.0.1", + "Microsoft.Extensions.Options": "10.0.1" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "YkmyiPIWAXVb+lPIrM0LE5bbtLOJkCiRTFiHpkVOvhI7uTvCfoOHLEN0LcsY56GpSD7NqX3gJNpsaDe87/B3zg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.1" + } + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "G6VVwywpJI4XIobetGHwg7wDOYC2L2XBYdtskxLaKF/Ynb5QBwLl7Q//wxAR2aVCLkMpoQrjSP9VoORkyddsNQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.1", + "Microsoft.Extensions.Primitives": "10.0.1" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "DO8XrJkp5x4PddDuc/CH37yDBCs9BYN6ijlKyR3vMb55BP1Vwh90vOX8bNfnKxr5B2qEI3D8bvbY1fFbDveDHQ==" + }, + "Microsoft.VisualStudio.SolutionPersistence": { + "type": "Transitive", + "resolved": "1.0.52", + "contentHash": "oNv2JtYXhpdJrX63nibx1JT3uCESOBQ1LAk7Dtz/sr0+laW0KRM6eKp4CZ3MHDR2siIkKsY8MmUkeP5DKkQQ5w==" + }, + "System.Composition": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "DJkqaWQfho/ReTzKcZD3zJJ6K4GcS154k+T0UCPMBNIOZ2U/lNpyiiWZ6Etw0onWyTH1K+yhICsdmwA5xy2aPQ==", + "dependencies": { + "System.Composition.AttributedModel": "10.0.1", + "System.Composition.Convention": "10.0.1", + "System.Composition.Hosting": "10.0.1", + "System.Composition.Runtime": "10.0.1", + "System.Composition.TypedParts": "10.0.1" + } + }, + "System.Composition.AttributedModel": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "mRxYvpCVPAeuLEk0c0kxWJVjbW1/HUoxCgYotOj9eDeQiYcTDOMdCQApsTrHYMN3pHBA8WoF00KGolG632Etaw==" + }, + "System.Composition.Convention": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "dYynUByfVBzYDheNPGxS8UN8AvG/4tXf/coSs1odHOyoh4etv1kad/FrLWLMq4f8NO49NV20Xu+0/y613woTUA==", + "dependencies": { + "System.Composition.AttributedModel": "10.0.1" + } + }, + "System.Composition.Hosting": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "4UGmyBdKWEN1nkqspJlji/nV7XIVm6KGlOC2So0mtM/gKvaNgLz+tUkcbY+6Zpr7dr6ohX1S5yl0RLID5otRHw==", + "dependencies": { + "System.Composition.Runtime": "10.0.1" + } + }, + "System.Composition.Runtime": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "TVfys1gnUIhmXuYfFzyez0fOkDyELe9UwlxYeVlq6FmqmWmt1ouF0OQJ+6ozkHbkaop7uBUaXw7Qb+/o0m+nMg==" + }, + "System.Composition.TypedParts": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "koSfjkdQZfgQr3SyiSIBboDn+GiR0vZ3x9Uek9FJbXK0w5AiATV8KrnMEP8B8OAlO+Y3zQf0CPCNzwH+VIYDKg==", + "dependencies": { + "System.Composition.AttributedModel": "10.0.1", + "System.Composition.Hosting": "10.0.1", + "System.Composition.Runtime": "10.0.1" + } + } + } + } +} \ No newline at end of file diff --git a/sidecars/scala/README.md b/sidecars/scala/README.md new file mode 100644 index 00000000..73e7215a --- /dev/null +++ b/sidecars/scala/README.md @@ -0,0 +1,31 @@ +# Scala graph producer + +This Maven reactor builds the `samchon-scala-graph` BSP client and the paired +Scala 2.13.18 and Scala 3.9.0 compiler plugins shipped with `@samchon/graph`. +Build all three artifacts with JDK 17 or newer: + +```bash +mvn --batch-mode --file sidecars/scala/pom.xml verify +``` + +The build writes these runnable artifacts: + +- `scala2-plugin/target/scala-graph-plugin_2.13.18-0.1.0-SNAPSHOT.jar` +- `scala3-plugin/target/scala-graph-plugin_3.9.0-0.1.0-SNAPSHOT.jar` +- `server/target/samchon-scala-graph-0.1.0-SNAPSHOT.jar` + +Expose the server jar through a `samchon-scala-graph` launcher that executes +`java -jar`, or point `SAMCHON_GRAPH_SCALA_GRAPH` at such a launcher. The +indexed repository must contain exactly one usable `.bsp/*.json` connection. + +Each non-empty Scala BSP target must load the matching typed plugin with +`-Xplugin`, pass the plugin's `root`, `output`, `target`, and `version` options, +and emit SemanticDB during the same compile. Scala 2 also needs SemanticDB's +source root, target root, build target, md5, symbols, and diagnostics options; +Scala 3 needs `-Xsemanticdb` and `-sourceroot`. The plugin `target` value and +SemanticDB build target must equal the target URI returned by BSP. + +The pinned [Scala experiment fixture](https://github.com/samchon/graph-benchmark-scala) +contains a complete sbt configuration for both compiler lines. Generate its +BSP connection with `sbt bspConfig`; graph refreshes then ask that BSP server +for ordinary incremental compilation and never run `clean`. diff --git a/sidecars/scala/common/pom.xml b/sidecars/scala/common/pom.xml new file mode 100644 index 00000000..5709e6bf --- /dev/null +++ b/sidecars/scala/common/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + + org.samchon.graph + scala-graph-parent + 0.1.0-SNAPSHOT + + scala-graph-common + + + + com.fasterxml.jackson.core + jackson-databind + + + diff --git a/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/Evidence.java b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/Evidence.java new file mode 100644 index 00000000..52b1ee49 --- /dev/null +++ b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/Evidence.java @@ -0,0 +1,8 @@ +package org.samchon.graph.scala.model; + +public record Evidence( + String file, + int startLine, + int startColumn, + int endLine, + int endColumn) {} diff --git a/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/GraphEdge.java b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/GraphEdge.java new file mode 100644 index 00000000..ff140b60 --- /dev/null +++ b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/GraphEdge.java @@ -0,0 +1,12 @@ +package org.samchon.graph.scala.model; + +public record GraphEdge( + String from, + String to, + String kind, + String access, + String provenance, + String targetKind, + String targetName, + String targetQualifiedName, + Evidence evidence) {} diff --git a/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/GraphNode.java b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/GraphNode.java new file mode 100644 index 00000000..10059492 --- /dev/null +++ b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/GraphNode.java @@ -0,0 +1,15 @@ +package org.samchon.graph.scala.model; + +import java.util.List; + +public record GraphNode( + String symbol, + String kind, + String name, + String qualifiedName, + String file, + boolean exported, + List modifiers, + String signature, + String origin, + Evidence evidence) {} diff --git a/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/TypedShard.java b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/TypedShard.java new file mode 100644 index 00000000..59d5ed2e --- /dev/null +++ b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/TypedShard.java @@ -0,0 +1,17 @@ +package org.samchon.graph.scala.model; + +import java.util.List; + +public record TypedShard( + int schemaVersion, + String language, + String source, + String checkerDigest, + String diskDigest, + String target, + String compilerVersion, + String compilerPlugin, + String compilerPluginVersion, + List nodes, + List edges, + List unresolved) {} diff --git a/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/UnresolvedSite.java b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/UnresolvedSite.java new file mode 100644 index 00000000..f1ff4581 --- /dev/null +++ b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/model/UnresolvedSite.java @@ -0,0 +1,9 @@ +package org.samchon.graph.scala.model; + +import java.util.List; + +public record UnresolvedSite( + String family, + String reason, + Evidence evidence, + List candidates) {} diff --git a/sidecars/scala/common/src/main/java/org/samchon/graph/scala/plugin/GraphShardWriter.java b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/plugin/GraphShardWriter.java new file mode 100644 index 00000000..1528ec8f --- /dev/null +++ b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/plugin/GraphShardWriter.java @@ -0,0 +1,87 @@ +package org.samchon.graph.scala.plugin; + +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.FileSystemException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.List; +import org.samchon.graph.scala.model.GraphEdge; +import org.samchon.graph.scala.model.GraphNode; +import org.samchon.graph.scala.model.TypedShard; +import org.samchon.graph.scala.model.UnresolvedSite; + +/** Content-addressed, atomic per-source output shared by both typed plugins. */ +public final class GraphShardWriter { + private static final ObjectMapper JSON = new ObjectMapper() + .enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY) + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + + private GraphShardWriter() {} + + public static void write( + PluginOptions options, + Path source, + String compilerText, + String compilerVersion, + String compilerPlugin, + List nodes, + List edges, + List unresolved) throws IOException { + Path absolute = source.toAbsolutePath().normalize(); + if (!absolute.startsWith(options.projectRoot())) { + throw new IOException("samchon-graph: source escapes project root: " + absolute); + } + String relative = options.projectRoot().relativize(absolute).toString().replace('\\', '/'); + byte[] disk = Files.readAllBytes(absolute); + TypedShard shard = new TypedShard( + 1, + "scala", + relative, + digest(compilerText.getBytes(StandardCharsets.UTF_8)), + digest(disk), + options.target(), + compilerVersion, + compilerPlugin, + options.pluginVersion(), + List.copyOf(nodes), + List.copyOf(edges), + List.copyOf(unresolved)); + byte[] body = JSON.writeValueAsBytes(shard); + String targetKey = digest(options.target().getBytes(StandardCharsets.UTF_8)); + String sourceKey = digest(relative.getBytes(StandardCharsets.UTF_8)); + Path directory = options.output().resolve("typed").resolve(targetKey); + Files.createDirectories(directory); + Path destination = directory.resolve(sourceKey + ".json"); + Path temporary = Files.createTempFile(directory, sourceKey + ".", ".tmp"); + try { + Files.write(temporary, body); + try { + Files.move( + temporary, + destination, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (FileSystemException ignored) { + Files.move(temporary, destination, StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporary); + } + } + + public static String digest(byte[] bytes) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } +} diff --git a/sidecars/scala/common/src/main/java/org/samchon/graph/scala/plugin/PluginOptions.java b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/plugin/PluginOptions.java new file mode 100644 index 00000000..ab1bdb33 --- /dev/null +++ b/sidecars/scala/common/src/main/java/org/samchon/graph/scala/plugin/PluginOptions.java @@ -0,0 +1,53 @@ +package org.samchon.graph.scala.plugin; + +import java.nio.file.Path; +import java.util.List; + +/** Strict, shared option parsing for both compiler-plugin generations. */ +public record PluginOptions( + Path projectRoot, + Path output, + String target, + String pluginVersion) { + public static PluginOptions parse(List values) { + String root = null; + String output = null; + String target = null; + String version = null; + for (String value : values) { + int split = value.indexOf('='); + if (split <= 0 || split == value.length() - 1) { + throw new IllegalArgumentException("samchon-graph: malformed option " + value); + } + String key = value.substring(0, split); + String body = value.substring(split + 1); + switch (key) { + case "root" -> root = unique(key, root, body); + case "output" -> output = unique(key, output, body); + case "target" -> target = unique(key, target, body); + case "version" -> version = unique(key, version, body); + default -> throw new IllegalArgumentException("samchon-graph: unknown option " + key); + } + } + if (root == null || output == null || target == null || version == null) { + throw new IllegalArgumentException( + "samchon-graph: root, output, target and version are required"); + } + Path projectRoot = Path.of(root).toAbsolutePath().normalize(); + Path targetOutput = Path.of(output).toAbsolutePath().normalize(); + if (!targetOutput.startsWith(projectRoot)) { + throw new IllegalArgumentException("samchon-graph: output must stay inside project root"); + } + if (!target.contains(":")) { + throw new IllegalArgumentException("samchon-graph: target must be a BSP URI"); + } + return new PluginOptions(projectRoot, targetOutput, target, version); + } + + private static String unique(String key, String current, String next) { + if (current != null) { + throw new IllegalArgumentException("samchon-graph: duplicate option " + key); + } + return next; + } +} diff --git a/sidecars/scala/pom.xml b/sidecars/scala/pom.xml new file mode 100644 index 00000000..0ce2bf58 --- /dev/null +++ b/sidecars/scala/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + org.samchon.graph + scala-graph-parent + 0.1.0-SNAPSHOT + pom + + + common + scala2-plugin + scala3-plugin + server + + + + 17 + UTF-8 + 2.13.18 + 3.9.0 + 4.17.2 + 2.1.1 + 2.20.0 + + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.module + jackson-module-scala_2.13 + ${jackson.version} + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.14.1 + + + net.alchim31.maven + scala-maven-plugin + 4.9.10 + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.1 + + + + + diff --git a/sidecars/scala/scala2-plugin/pom.xml b/sidecars/scala/scala2-plugin/pom.xml new file mode 100644 index 00000000..1c5c5f6d --- /dev/null +++ b/sidecars/scala/scala2-plugin/pom.xml @@ -0,0 +1,79 @@ + + + 4.0.0 + + org.samchon.graph + scala-graph-parent + 0.1.0-SNAPSHOT + + scala-graph-plugin_2.13.18 + + + + org.samchon.graph + scala-graph-common + ${project.version} + + + org.scala-lang + scala-compiler + ${scala2.version} + provided + + + org.scala-lang + scala-library + ${scala2.version} + provided + + + + + + + net.alchim31.maven + scala-maven-plugin + + + compile + + + + ${scala2.version} + -deprecation-feature + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + shade + + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.scala-lang:scala-compiler + org.scala-lang:scala-library + + + + + + + + + diff --git a/sidecars/scala/scala2-plugin/src/main/resources/scalac-plugin.xml b/sidecars/scala/scala2-plugin/src/main/resources/scalac-plugin.xml new file mode 100644 index 00000000..665ff258 --- /dev/null +++ b/sidecars/scala/scala2-plugin/src/main/resources/scalac-plugin.xml @@ -0,0 +1,4 @@ + + samchon-graph + org.samchon.graph.scala2.Scala2GraphPlugin + diff --git a/sidecars/scala/scala2-plugin/src/main/scala/org/samchon/graph/scala2/Scala2GraphPlugin.scala b/sidecars/scala/scala2-plugin/src/main/scala/org/samchon/graph/scala2/Scala2GraphPlugin.scala new file mode 100644 index 00000000..bae7a4ae --- /dev/null +++ b/sidecars/scala/scala2-plugin/src/main/scala/org/samchon/graph/scala2/Scala2GraphPlugin.scala @@ -0,0 +1,271 @@ +package org.samchon.graph.scala2 + +import java.nio.file.Path +import java.util.{ArrayList, Collections, List => JList} + +import org.samchon.graph.scala.model.{Evidence, GraphEdge, GraphNode, UnresolvedSite} +import org.samchon.graph.scala.plugin.{GraphShardWriter, PluginOptions} +import scala.tools.nsc.Global +import scala.tools.nsc.Phase +import scala.tools.nsc.plugins.{Plugin, PluginComponent} + +/** Scala 2 typed-tree exporter. Its phase runs immediately after typer. */ +final class Scala2GraphPlugin(val global: Global) extends Plugin { + import global._ + + override val name = "samchon-graph" + override val description = "Emit compiler-owned graph shards after typer" + override val components: List[PluginComponent] = List(GraphComponent) + private var configured: Option[PluginOptions] = None + + override def processOptions(options: List[String], error: String => Unit): Unit = + try configured = Some(PluginOptions.parse(java.util.List.copyOf(options.asJava))) + catch { case exception: IllegalArgumentException => error(exception.getMessage) } + + override val optionsHelp: Option[String] = Some( + "-P:samchon-graph:root=:output=:target=:version=") + + private object GraphComponent extends PluginComponent { + override val global: Scala2GraphPlugin.this.global.type = Scala2GraphPlugin.this.global + override val phaseName = "samchon-graph" + override val runsAfter: List[String] = List("typer") + override val runsBefore: List[String] = List("patmat") + + override def newPhase(previous: Phase): StdPhase = new StdPhase(previous) { + override def apply(unit: CompilationUnit): Unit = configured match { + case Some(options) => new Collector(unit, options).write() + case None => reporter.error(unit.position(0), "samchon-graph plugin options are missing") + } + } + } + + private final class Collector(unit: CompilationUnit, options: PluginOptions) + extends Traverser { + private val nodes = new ArrayList[GraphNode]() + private val edges = new ArrayList[GraphEdge]() + private val unresolved = new ArrayList[UnresolvedSite]() + private val declared = scala.collection.mutable.HashSet.empty[String] + private val sourcePath = unit.source.file.file.toPath.toAbsolutePath.normalize + private val source = options.projectRoot.relativize(sourcePath).toString.replace('\\', '/') + private var owner: String = source + + def write(): Unit = { + traverse(unit.body) + GraphShardWriter.write( + options, + sourcePath, + new String(unit.source.content), + scala.util.Properties.versionNumberString, + "scala2", + nodes, + edges, + unresolved) + } + + override def traverse(tree: Tree): Unit = tree match { + case definition: PackageDef => within(definition.symbol, definition) { super.traverse(tree) } + case definition: ClassDef => within(definition.symbol, definition) { super.traverse(tree) } + case definition: ModuleDef => within(definition.symbol, definition) { super.traverse(tree) } + case definition: DefDef => within(definition.symbol, definition) { super.traverse(tree) } + case definition: ValDef => + declare(definition.symbol, definition) + typeReference(definition.symbol, definition) + super.traverse(tree) + case definition: TypeDef => + declare(definition.symbol, definition) + super.traverse(tree) + case function: Function => within(function.symbol, function) { super.traverse(tree) } + case application: Apply => + call(application.fun.symbol, application) + application.fun match { + case Select(New(target), _) => instantiate(target.tpe.typeSymbol, application) + case _ => () + } + super.traverse(tree) + case assignment: Assign => + reference(assignment.lhs.symbol, "write", assignment.lhs) + super.traverse(tree) + case selection: Select => + reference(selection.symbol, "read", selection) + super.traverse(tree) + case identifier: Ident if identifier.symbol != null && identifier.symbol != NoSymbol => + reference(identifier.symbol, "read", identifier) + super.traverse(tree) + case importing: Import => + val target = importing.expr.symbol + if (usable(target, importing)) addEdge(owner, target, "imports", importing, "semanticdb") + super.traverse(tree) + case _ => super.traverse(tree) + } + + private def within(symbol: Symbol, tree: Tree)(body: => Unit): Unit = { + declare(symbol, tree) + if (tree.isInstanceOf[ClassDef]) parents(symbol, tree.asInstanceOf[ClassDef]) + val previous = owner + if (symbol != null && symbol != NoSymbol) owner = symbolKey(symbol) + try body finally owner = previous + } + + private def declare(symbol: Symbol, tree: Tree): Unit = { + if (!usable(symbol, tree)) return + val key = symbolKey(symbol) + if (!declared.add(key)) return + val evidence = position(tree) + val modifiers = new ArrayList[String]() + if (symbol.isPublic) modifiers.add("public") + if (symbol.isPrivate) modifiers.add("private") + if (symbol.isProtected) modifiers.add("protected") + if (symbol.isAbstract) modifiers.add("abstract") + if (symbol.isFinal) modifiers.add("readonly") + if (symbol.isImplicit) modifiers.add("declare") + nodes.add(new GraphNode( + key, + kind(symbol), + symbol.name.decodedName.toString.trim, + qualified(symbol), + source, + symbol.isPublic, + modifiers, + signature(symbol), + if (symbol.isSynthetic || symbol.isAccessor) "Synthetic" else "Source", + evidence)) + edges.add(edge(owner, symbol, "contains", tree, "typed-plugin")) + if (symbol.isPublic) edges.add(edge(source, symbol, "exports", tree, "semanticdb")) + symbol.allOverriddenSymbols.foreach(overridden => + edges.add(edge(key, overridden, "overrides", tree, "semanticdb"))) + symbol.annotations.foreach(annotation => { + val target = annotation.tree.tpe.typeSymbol + if (target != NoSymbol) edges.add(edge(key, target, "decorates", tree, "semanticdb")) + }) + } + + private def parents(symbol: Symbol, definition: ClassDef): Unit = + definition.impl.parents.foreach(parent => { + val target = parent.tpe.typeSymbol + if (target != null && target != NoSymbol) { + val family = if (target.isTrait) "implements" else "extends" + edges.add(edge(symbolKey(symbol), target, family, parent, "semanticdb")) + } + }) + + private def call(target: Symbol, tree: Tree): Unit = { + if (!usable(target, tree)) return + edges.add(edge(owner, target, "calls", tree, "typed-plugin")) + edges.add(edge(owner, target, "references", tree, "semanticdb")) + if (target.isMethod && !target.isFinal && !target.owner.isFinal) { + unresolved.add(new UnresolvedSite( + "dispatches", "dynamic", position(tree), Collections.singletonList(symbolKey(target)))) + } + if (target.isMacro) { + unresolved.add(new UnresolvedSite( + "calls", "macro-or-generated", position(tree), Collections.singletonList(symbolKey(target)))) + } + } + + private def instantiate(target: Symbol, tree: Tree): Unit = + if (usable(target, tree)) edges.add(edge(owner, target, "instantiates", tree, "typed-plugin")) + + private def reference(target: Symbol, access: String, tree: Tree): Unit = { + if (!usable(target, tree) || target.isPackage) return + edges.add(new GraphEdge( + owner, + symbolKey(target), + "accesses", + access, + "typed-plugin", + kind(target), + target.name.decodedName.toString.trim, + qualified(target), + position(tree))) + if (target.isImplicit) { + unresolved.add(new UnresolvedSite( + "references", "macro-or-generated", position(tree), Collections.singletonList(symbolKey(target)))) + } + } + + private def typeReference(symbol: Symbol, tree: Tree): Unit = { + if (symbol == null || symbol == NoSymbol || symbol.info == null) return + val target = symbol.info.finalResultType.typeSymbol + if (target != null && target != NoSymbol && target != symbol) { + edges.add(edge(symbolKey(symbol), target, "type_ref", tree, "semanticdb")) + } + } + + private def addEdge(from: String, target: Symbol, family: String, tree: Tree, provenance: String): Unit = + edges.add(edge(from, target, family, tree, provenance)) + + private def edge(from: String, target: Symbol, family: String, tree: Tree, provenance: String) = + new GraphEdge( + from, + symbolKey(target), + family, + null, + provenance, + kind(target), + target.name.decodedName.toString.trim, + qualified(target), + position(tree)) + + private def usable(symbol: Symbol, tree: Tree): Boolean = + symbol != null && symbol != NoSymbol && tree.pos != null && tree.pos.isDefined + + private def symbolKey(symbol: Symbol): String = + // A package declaration is source syntax repeated in every compilation + // unit, unlike the single package symbol the compiler interns globally. + if (symbol.isPackage) s"scala-package $source|${symbol.fullName}" + else s"scala-structural ${ownerIdentity(symbol.owner)}|${kind(symbol)}|${symbol.name.decodedName}|${signature(symbol)}${lexical(symbol)}" + + private def ownerIdentity(symbol: Symbol): String = + if (symbol == null || symbol == NoSymbol) "" + else if (symbol.isPackage) symbol.fullName + else s"${ownerIdentity(symbol.owner)}|${kind(symbol)}|${symbol.name.decodedName}|${signature(symbol)}${lexical(symbol)}" + + private def lexical(symbol: Symbol): String = + if (symbol.owner != null && symbol.owner != NoSymbol && symbol.owner.isMethod && + !symbol.isParameter && symbol.pos != null && symbol.pos.isDefined) + s"|lexical=${symbol.pos.point}" + else "" + + private def signature(symbol: Symbol): String = + if (symbol.info == null) "" + else if (symbol.isClass) symbol.typeParams.map(_.name.decodedName.toString).mkString("[", ",", "]") + else symbol.info.dealias.toString.replaceAll("\\s+", " ").trim + + private def qualified(symbol: Symbol): String = + try symbol.fullName catch { case _: Throwable => symbol.name.decodedName.toString } + + private def kind(symbol: Symbol): String = + if (symbol.isPackage) "package" + else if (symbol.isModule || symbol.isModuleClass) "module" + else if (symbol.isTrait) "interface" + else if (symbol.isClass) "class" + else if (symbol.isConstructor) "constructor" + else if (symbol.isMethod) "method" + else if (symbol.isType) "type" + else if (symbol.isParameter) "parameter" + else if (symbol.owner != null && symbol.owner.isClass) "field" + else "variable" + + private def position(tree: Tree): Evidence = { + val pos = tree.pos + val end = if (pos.isRange) pos.end else pos.point + math.max(1, tree.toString.length) + val endPosition = + if (unit.source.length == 0) pos + else unit.source.position(math.max(0, math.min(end, unit.source.length - 1))) + new Evidence( + source, + math.max(1, pos.line), + math.max(1, pos.column + 1), + math.max(1, endPosition.line), + math.max(1, endPosition.column + 1)) + } + } + + private implicit final class JavaListOps[A](private val values: List[A]) { + def asJava: JList[A] = { + val out = new ArrayList[A](values.size) + values.foreach(out.add) + out + } + } +} diff --git a/sidecars/scala/scala3-plugin/pom.xml b/sidecars/scala/scala3-plugin/pom.xml new file mode 100644 index 00000000..00950408 --- /dev/null +++ b/sidecars/scala/scala3-plugin/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + + org.samchon.graph + scala-graph-parent + 0.1.0-SNAPSHOT + + scala-graph-plugin_3.9.0 + + + + org.samchon.graph + scala-graph-common + ${project.version} + + + org.scala-lang + scala3-compiler_3 + ${scala3.version} + provided + + + org.scala-lang + scala3-library_3 + ${scala3.version} + provided + + + + + + + net.alchim31.maven + scala-maven-plugin + + compile + + + ${scala3.version} + -deprecation-feature + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + shade + + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.scala-lang:scala3-compiler_3 + org.scala-lang:scala3-library_3 + org.scala-lang:scala-library + + + + + + + + + diff --git a/sidecars/scala/scala3-plugin/src/main/resources/plugin.properties b/sidecars/scala/scala3-plugin/src/main/resources/plugin.properties new file mode 100644 index 00000000..96df4fa7 --- /dev/null +++ b/sidecars/scala/scala3-plugin/src/main/resources/plugin.properties @@ -0,0 +1 @@ +pluginClass=org.samchon.graph.scala3.Scala3GraphPlugin diff --git a/sidecars/scala/scala3-plugin/src/main/scala/org/samchon/graph/scala3/Scala3GraphPlugin.scala b/sidecars/scala/scala3-plugin/src/main/scala/org/samchon/graph/scala3/Scala3GraphPlugin.scala new file mode 100644 index 00000000..c69d831b --- /dev/null +++ b/sidecars/scala/scala3-plugin/src/main/scala/org/samchon/graph/scala3/Scala3GraphPlugin.scala @@ -0,0 +1,280 @@ +package org.samchon.graph.scala3 + +import java.nio.file.Path +import java.util.{ArrayList, Collections} + +import dotty.tools.dotc.ast.tpd +import dotty.tools.dotc.ast.tpd.* +import dotty.tools.dotc.core.Contexts.Context +import dotty.tools.dotc.core.Flags +import dotty.tools.dotc.core.Symbols.Symbol +import dotty.tools.dotc.plugins.{PluginPhase, StandardPlugin} +import dotty.tools.dotc.transform.{FirstTransform, PickleQuotes} +import org.samchon.graph.scala.model.{Evidence, GraphEdge, GraphNode, UnresolvedSite} +import org.samchon.graph.scala.plugin.{GraphShardWriter, PluginOptions} +import scala.jdk.CollectionConverters.* + +/** Scala 3 typed-tree exporter. The phase observes trees after typer and before lowering. */ +final class Scala3GraphPlugin extends StandardPlugin: + override val name = "samchon-graph" + override val description = "Emit compiler-owned graph shards after typer" + override val optionsHelp = Some( + "-P:samchon-graph:root=:output=:target=:version=") + + override def initialize(options: List[String])(using Context): List[PluginPhase] = + val parsed = PluginOptions.parse(java.util.List.copyOf(options.asJava)) + List(new Scala3GraphPhase(parsed)) + +private final class Scala3GraphPhase(options: PluginOptions) extends PluginPhase: + override val phaseName = "samchon-graph" + override val runsAfter = Set("typer") + override val runsBefore = Set(PickleQuotes.name, FirstTransform.name) + + private var collector: Collector | Null = null + + override def prepareForUnit(tree: Tree)(using Context): Context = + collector = new Collector(options) + summon[Context] + + override def transformUnit(tree: Tree)(using Context): Tree = + current.write() + collector = null + tree + + override def transformTypeDef(tree: TypeDef)(using Context): Tree = + current.declare(tree.symbol, tree) + tree.rhs match + case template: Template => current.parents(tree.symbol, template) + case _ => () + tree + + override def transformPackageDef(tree: PackageDef)(using Context): Tree = + current.declare(tree.symbol, tree) + tree + + override def transformDefDef(tree: DefDef)(using Context): Tree = + current.declare(tree.symbol, tree) + current.typeReference(tree.symbol, tree) + tree + + override def transformValDef(tree: ValDef)(using Context): Tree = + current.declare(tree.symbol, tree) + current.typeReference(tree.symbol, tree) + tree + + override def transformApply(tree: Apply)(using Context): Tree = + current.call(tree.fun.symbol, tree) + tree + + override def transformNew(tree: New)(using Context): Tree = + current.instantiate(tree.tpe.typeSymbol, tree) + tree + + override def transformAssign(tree: Assign)(using Context): Tree = + current.reference(tree.lhs.symbol, "write", tree.lhs) + tree + + override def transformSelect(tree: Select)(using Context): Tree = + current.reference(tree.symbol, "read", tree) + tree + + override def transformIdent(tree: Ident)(using Context): Tree = + current.reference(tree.symbol, "read", tree) + tree + + override def transformTypeTree(tree: TypeTree)(using Context): Tree = + current.typeReference(summon[Context].owner, tree, tree.tpe.typeSymbol) + tree + + override def transformOther(tree: Tree)(using Context): Tree = + tree match + case importing: Import => current.importReference(importing.expr.symbol, importing) + case _ => () + tree + + private def current: Collector = + if collector == null then throw new IllegalStateException("samchon-graph: compilation unit is missing") + collector.nn + +private final class Collector(options: PluginOptions)(using initialContext: Context): + private val nodes = new ArrayList[GraphNode]() + private val edges = new ArrayList[GraphEdge]() + private val unresolved = new ArrayList[UnresolvedSite]() + private val declared = scala.collection.mutable.HashSet.empty[String] + private val sourcePath = Path.of(initialContext.source.path).toAbsolutePath.normalize + private val source = options.projectRoot.relativize(sourcePath).toString.replace('\\', '/') + + def write()(using Context): Unit = + GraphShardWriter.write( + options, + sourcePath, + initialContext.source.content.mkString, + dotty.tools.dotc.config.Properties.versionNumberString, + "scala3", + nodes, + edges, + unresolved) + + def declare(symbol: Symbol, tree: Tree)(using Context): Unit = + if !usable(symbol, tree) then return + val key = symbolKey(symbol) + if !declared.add(key) then return + val modifiers = new ArrayList[String]() + if exported(symbol) then modifiers.add("public") + if symbol.is(Flags.Private) then modifiers.add("private") + if symbol.is(Flags.Protected) then modifiers.add("protected") + if symbol.is(Flags.Deferred) then modifiers.add("abstract") + if symbol.is(Flags.Final) then modifiers.add("readonly") + if symbol.isOneOf(Flags.GivenOrImplicit) then modifiers.add("declare") + nodes.add(new GraphNode( + key, + kind(symbol), + symbol.name.show, + qualified(symbol), + source, + exported(symbol), + modifiers, + signature(symbol), + if generated(symbol) then "Synthetic" else "Source", + position(tree))) + edges.add(edge(ownerKey(symbol.owner), symbol, "contains", tree, "typed-plugin")) + if exported(symbol) then edges.add(edge(source, symbol, "exports", tree, "semanticdb")) + symbol.allOverriddenSymbols.foreach(overridden => + edges.add(edge(key, overridden, "overrides", tree, "semanticdb"))) + symbol.annotations.foreach(annotation => + val target = annotation.symbol + if target.exists && !target.fullName.show.startsWith("scala.annotation.internal.") then + edges.add(edge(key, target, "decorates", tree, "semanticdb"))) + + def parents(symbol: Symbol, template: Template)(using Context): Unit = + template.parents.foreach(parent => + val target = parent.tpe.typeSymbol + if target.exists then + val family = if target.is(Flags.Trait) then "implements" else "extends" + edges.add(edge(symbolKey(symbol), target, family, parent, "semanticdb"))) + + def call(target: Symbol, tree: Tree)(using Context): Unit = + if !usable(target, tree) then return + val owner = ownerKey(summon[Context].owner) + edges.add(edge(owner, target, "calls", tree, "typed-plugin")) + edges.add(edge(owner, target, "references", tree, "semanticdb")) + if target.is(Flags.Method) && !target.is(Flags.Final) && !target.owner.is(Flags.Final) then + unresolved.add(new UnresolvedSite( + "dispatches", "dynamic", position(tree), Collections.singletonList(symbolKey(target)))) + if target.is(Flags.Inline) then + unresolved.add(new UnresolvedSite( + "calls", "macro-or-generated", position(tree), Collections.singletonList(symbolKey(target)))) + + def instantiate(target: Symbol, tree: Tree)(using Context): Unit = + if usable(target, tree) then + edges.add(edge(ownerKey(summon[Context].owner), target, "instantiates", tree, "typed-plugin")) + + def reference(target: Symbol, access: String, tree: Tree)(using Context): Unit = + if !usable(target, tree) || target.is(Flags.Package) then return + edges.add(new GraphEdge( + ownerKey(summon[Context].owner), + symbolKey(target), + "accesses", + access, + "typed-plugin", + kind(target), + target.name.show, + qualified(target), + position(tree))) + if target.isOneOf(Flags.GivenOrImplicit) then + unresolved.add(new UnresolvedSite( + "references", "macro-or-generated", position(tree), Collections.singletonList(symbolKey(target)))) + + def importReference(target: Symbol, tree: Tree)(using Context): Unit = + if usable(target, tree) then + edges.add(edge(ownerKey(summon[Context].owner), target, "imports", tree, "semanticdb")) + + def typeReference(symbol: Symbol, tree: Tree)(using Context): Unit = + if symbol.exists then typeReference(symbol, tree, symbol.info.finalResultType.typeSymbol) + + def typeReference(symbol: Symbol, tree: Tree, target: Symbol)(using Context): Unit = + if symbol.exists && target.exists && target != symbol then + edges.add(edge(symbolKey(symbol), target, "type_ref", tree, "semanticdb")) + + private def edge(from: String, target: Symbol, family: String, tree: Tree, provenance: String)(using Context) = + new GraphEdge( + from, + symbolKey(target), + family, + null, + provenance, + kind(target), + target.name.show, + qualified(target), + position(tree)) + + private def usable(symbol: Symbol, tree: Tree)(using Context): Boolean = + symbol.exists && tree.sourcePos.exists + + private def exported(symbol: Symbol): Boolean = + !symbol.isOneOf(Flags.Private | Flags.Protected) + + private def generated(symbol: Symbol): Boolean = + symbol.isOneOf(Flags.Synthetic | Flags.Artifact | Flags.ModuleVal) || + symbol.name.show.contains("$anon") || + symbol.name.show.contains("$proxy") || + symbol.name.show.endsWith("$package") || + symbol.name.show == "MirroredMonoType" || + symbol.isOneOf(Flags.Module | Flags.ModuleClass) && + symbol.companionClass.exists && + symbol.companionClass.isOneOf(Flags.Case | Flags.Enum) || + symbol.isConstructor && symbol.owner.isOneOf(Flags.Synthetic | Flags.Artifact) || + symbol.is(Flags.Param) && ( + symbol.owner.isOneOf(Flags.Synthetic | Flags.Artifact) || + symbol.owner.isConstructor && + symbol.owner.owner.isOneOf(Flags.Synthetic | Flags.Artifact)) + + private def ownerKey(symbol: Symbol)(using Context): String = + if !symbol.exists || symbol.is(Flags.Package) then source else symbolKey(symbol) + + private def symbolKey(symbol: Symbol)(using Context): String = + // A package declaration is source syntax repeated in every compilation + // unit, unlike the single package symbol the compiler interns globally. + if symbol.is(Flags.Package) then s"scala-package $source|${symbol.fullName.show}" + else s"scala-structural ${ownerIdentity(symbol.owner)}|${kind(symbol)}|${symbol.name.show}|${signature(symbol)}${lexical(symbol)}" + + private def ownerIdentity(symbol: Symbol)(using Context): String = + if !symbol.exists then "" + else if symbol.is(Flags.Package) then symbol.fullName.show + else s"${ownerIdentity(symbol.owner)}|${kind(symbol)}|${symbol.name.show}|${signature(symbol)}${lexical(symbol)}" + + private def lexical(symbol: Symbol): String = + if symbol.owner.exists && symbol.owner.is(Flags.Method) && !symbol.is(Flags.Param) && + symbol.sourcePos.exists + then s"|lexical=${symbol.sourcePos.start}" + else "" + + private def signature(symbol: Symbol)(using Context): String = + val value = + if symbol.isClass then symbol.typeParams.map(_.name.show).mkString("[", ",", "]") + else symbol.info.show + value.replaceAll("\\u001b\\[[;\\d]*m", "").replaceAll("\\s+", " ").trim + + private def qualified(symbol: Symbol)(using Context): String = + symbol.fullName.show + + private def kind(symbol: Symbol): String = + if symbol.is(Flags.Package) then "package" + else if symbol.isOneOf(Flags.Module | Flags.ModuleClass) then "module" + else if symbol.is(Flags.Trait) then "interface" + else if symbol.isClass then "class" + else if symbol.isConstructor then "constructor" + else if symbol.is(Flags.Method) then "method" + else if symbol.isType then "type" + else if symbol.is(Flags.Param) then "parameter" + else if symbol.owner.exists && symbol.owner.isClass then "field" + else "variable" + + private def position(tree: Tree)(using Context): Evidence = + val pos = tree.sourcePos + new Evidence( + source, + math.max(1, pos.startLine + 1), + math.max(1, pos.startColumn + 1), + math.max(1, pos.endLine + 1), + math.max(1, pos.endColumn + 1)) diff --git a/sidecars/scala/server/pom.xml b/sidecars/scala/server/pom.xml new file mode 100644 index 00000000..b2cdb19c --- /dev/null +++ b/sidecars/scala/server/pom.xml @@ -0,0 +1,84 @@ + + + 4.0.0 + + org.samchon.graph + scala-graph-parent + 0.1.0-SNAPSHOT + + samchon-scala-graph + + + + org.samchon.graph + scala-graph-common + ${project.version} + + + org.scala-lang + scala3-library_3 + ${scala3.version} + + + ch.epfl.scala + bsp4j + ${bsp4j.version} + + + org.scalameta + semanticdb-shared_2.13 + ${semanticdb.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.module + jackson-module-scala_2.13 + + + + + + + net.alchim31.maven + scala-maven-plugin + + compile + + ${scala3.version} + + + org.apache.maven.plugins + maven-shade-plugin + + + package + shade + + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.samchon.graph.scala.server.Main + + + + + + + + + diff --git a/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/DiagnosticFact.java b/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/DiagnosticFact.java new file mode 100644 index 00000000..8c9450f2 --- /dev/null +++ b/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/DiagnosticFact.java @@ -0,0 +1,5 @@ +package org.samchon.graph.scala.server.model; + +import org.samchon.graph.scala.model.Evidence; + +public record DiagnosticFact(String severity, String message, Evidence evidence) {} diff --git a/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/SemanticShard.java b/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/SemanticShard.java new file mode 100644 index 00000000..6fccce36 --- /dev/null +++ b/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/SemanticShard.java @@ -0,0 +1,25 @@ +package org.samchon.graph.scala.server.model; + +import java.util.List; +import org.samchon.graph.scala.model.GraphEdge; +import org.samchon.graph.scala.model.GraphNode; +import org.samchon.graph.scala.model.UnresolvedSite; + +public record SemanticShard( + int schemaVersion, + String language, + String source, + String checkerDigest, + String diskDigest, + String target, + String compilerVersion, + String compilerPlugin, + String compilerPluginVersion, + int semanticdbSchema, + String semanticdbUri, + String semanticdbMd5, + String semanticdbBuildTarget, + List nodes, + List edges, + List unresolved, + List diagnostics) {} diff --git a/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/SnapshotArtifact.java b/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/SnapshotArtifact.java new file mode 100644 index 00000000..577f25a3 --- /dev/null +++ b/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/SnapshotArtifact.java @@ -0,0 +1,24 @@ +package org.samchon.graph.scala.server.model; + +import java.util.List; + +public record SnapshotArtifact( + int schemaVersion, + String projectRoot, + Producer producer, + List targets) { + public record Producer( + String name, + String version, + int protocolVersion, + Capabilities capabilities) {} + + public record Capabilities( + boolean atomicGenerations, + boolean incremental, + boolean diagnostics, + boolean bsp, + boolean semanticdb, + boolean typedPlugins, + boolean zinc) {} +} diff --git a/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/TargetSnapshot.java b/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/TargetSnapshot.java new file mode 100644 index 00000000..f24b0522 --- /dev/null +++ b/sidecars/scala/server/src/main/java/org/samchon/graph/scala/server/model/TargetSnapshot.java @@ -0,0 +1,23 @@ +package org.samchon.graph.scala.server.model; + +import java.util.List; +import java.util.Map; + +public record TargetSnapshot( + String name, + String generation, + String universe, + String bspUri, + String scalaVersion, + String scalaBinaryVersion, + String platform, + String sourceEncoding, + String scalacOptionsDigest, + String classpathDigest, + String sourceRootsDigest, + String semanticdbOptionsDigest, + String compilerPluginsDigest, + String zincAnalysisDigest, + String generatedSourcesDigest, + Map coverage, + List shards) {} diff --git a/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/AtomicJson.scala b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/AtomicJson.scala new file mode 100644 index 00000000..2661e4c3 --- /dev/null +++ b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/AtomicJson.scala @@ -0,0 +1,25 @@ +package org.samchon.graph.scala.server + +import com.fasterxml.jackson.databind.{MapperFeature, ObjectMapper, SerializationFeature} +import java.nio.file.{FileSystemException, Files, Path, StandardCopyOption} + +private[server] object AtomicJson: + val mapper: ObjectMapper = new ObjectMapper() + .enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY) + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) + + def write(path: Path, value: AnyRef): Unit = + val parent = path.toAbsolutePath.normalize.getParent + Files.createDirectories(parent) + val temporary = Files.createTempFile(parent, path.getFileName.toString + ".", ".tmp") + try + Files.write(temporary, mapper.writeValueAsBytes(value)) + try Files.move( + temporary, + path, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING) + catch + case _: FileSystemException => + Files.move(temporary, path, StandardCopyOption.REPLACE_EXISTING) + finally Files.deleteIfExists(temporary) diff --git a/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/BspSession.scala b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/BspSession.scala new file mode 100644 index 00000000..241114e5 --- /dev/null +++ b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/BspSession.scala @@ -0,0 +1,307 @@ +package org.samchon.graph.scala.server + +import ch.epfl.scala.bsp4j.* +import com.fasterxml.jackson.databind.JsonNode +import java.net.URI +import java.nio.file.{Files, Path} +import java.util.concurrent.{ConcurrentHashMap, Executors, TimeUnit} +import org.eclipse.lsp4j.jsonrpc.Launcher +import scala.collection.concurrent.TrieMap +import scala.jdk.CollectionConverters.* + +private[server] final case class SourceDescriptor(uri: String, path: Path, generated: Boolean) + +private[server] final case class TargetDescriptor( + target: BuildTarget, + scalaVersion: String, + scalaBinaryVersion: String, + platform: String, + options: List[String], + classpath: List[String], + classDirectory: Path, + sources: List[SourceDescriptor], + sourceRoots: List[String], + graphOutput: Path +): + val uri: String = target.getId.getUri + val expectedPlugin: String = if scalaVersion.startsWith("2.") then "scala2" else "scala3" + +private[server] trait RemoteScalaBuildServer extends BuildServer with ScalaBuildServer + +/** One resident, initialized BSP connection. It never requests a clean. */ +private[server] final class BspSession private ( + root: Path, + details: ConnectionDetails, + process: Process, + launcher: Launcher[RemoteScalaBuildServer], + client: DiagnosticClient, + executor: java.util.concurrent.ExecutorService +) extends AutoCloseable: + private val server = launcher.getRemoteProxy + private var reloadSupported = false + + def reload(): Unit = + ensureAlive() + if reloadSupported then await(server.workspaceReload(), 120) + + def describeTargets(): List[TargetDescriptor] = + ensureAlive() + val rawTargets = await(server.workspaceBuildTargets()).getTargets.asScala.toList + .filter(target => + Option(target.getLanguageIds).exists(_.asScala.contains("scala")) && + Option(target.getCapabilities).exists(capabilities => + java.lang.Boolean.TRUE == capabilities.getCanCompile)) + .sortBy(_.getId.getUri) + if rawTargets.isEmpty then fail("BSP workspace has no Scala build targets") + val identifiers = rawTargets.map(_.getId) + val sourcesByTarget = await(server.buildTargetSources(new SourcesParams(identifiers.asJava))) + .getItems.asScala.map(item => item.getTarget.getUri -> item).toMap + val optionsByTarget = await(server.buildTargetScalacOptions(new ScalacOptionsParams(identifiers.asJava))) + .getItems.asScala.map(item => item.getTarget.getUri -> item).toMap + + val descriptors = rawTargets.flatMap { target => + val uri = target.getId.getUri + val sourceItem = sourcesByTarget.getOrElse(uri, fail(s"BSP omitted sources for $uri")) + val sources = expandSources(sourceItem) + if sources.isEmpty then None + else + if target.getDataKind != BuildTargetDataKind.SCALA then + fail(s"Scala BSP target $uri has no Scala target data") + val scala = scalaTarget(target) + val scalac = optionsByTarget.getOrElse(uri, fail(s"BSP omitted scalac options for $uri")) + val options = scalac.getOptions.asScala.toList + val graphOutput = graphOption(options, "output").map(pathValue).getOrElse( + fail(s"BSP target $uri does not configure the samchon-graph output")) + val expectedTarget = graphOption(options, "target").getOrElse( + fail(s"BSP target $uri does not configure the samchon-graph target")) + val expectedRoot = graphOption(options, "root").map(pathValue).getOrElse( + fail(s"BSP target $uri does not configure the samchon-graph root")) + if expectedTarget != uri then + fail(s"BSP target $uri configures a different samchon-graph target: $expectedTarget") + if expectedRoot != root then fail(s"BSP target $uri configures a different project root") + if !graphOutput.startsWith(root) then fail(s"BSP target $uri graph output escapes the project root") + requireSemanticdb(uri, scala._1, options) + Some(TargetDescriptor( + target, + scala._1, + scala._2, + scala._3, + options, + scalac.getClasspath.asScala.toList, + confinedPath(scalac.getClassDirectory, s"class directory for $uri"), + sources, + Option(sourceItem.getRoots).fold(List.empty[String])(_.asScala.toList.sorted), + graphOutput)) + } + if descriptors.isEmpty then fail("BSP workspace has no non-empty Scala build targets") + descriptors + + def compile(targets: List[TargetDescriptor]): Unit = + targets.foreach(target => client.clear(target.uri)) + val params = new CompileParams(targets.map(_.target.getId).asJava) + params.setOriginId("samchon-scala-graph") + val result = await(server.buildTargetCompile(params)) + result.getStatusCode match + case StatusCode.OK => () + case StatusCode.CANCELLED => fail("BSP compile was cancelled") + case _ => fail("BSP compile failed") + + def diagnostics(target: String, source: Path): List[ch.epfl.scala.bsp4j.Diagnostic] = + client.diagnostics(target, source.toUri.toString) + + private def requireSemanticdb(uri: String, version: String, options: List[String]): Unit = + val hasGraphPlugin = options.exists(_.startsWith("-P:samchon-graph:")) && + options.exists(option => option.startsWith("-Xplugin:") || option.startsWith("-Xplugin-require:samchon-graph")) + val hasSemanticdb = + if version.startsWith("2.") then + options.exists(_.startsWith("-P:semanticdb:sourceroot:")) && + options.exists(_.startsWith("-P:semanticdb:buildtarget:")) && + options.exists(_.startsWith("-P:semanticdb:targetroot:")) + else options.contains("-Xsemanticdb") || options.exists(_.startsWith("-Xsemanticdb:")) + if !hasGraphPlugin then fail(s"BSP target $uri does not load the samchon-graph compiler plugin") + if !hasSemanticdb then fail(s"BSP target $uri does not emit SemanticDB") + + private def graphOption(options: List[String], name: String): Option[String] = + val prefix = s"-P:samchon-graph:$name=" + val values = options.filter(_.startsWith(prefix)).map(_.substring(prefix.length)).distinct + values match + case value :: Nil if value.nonEmpty => Some(value) + case Nil => None + case _ => fail(s"duplicate samchon-graph option $name") + + private def scalaTarget(target: BuildTarget): (String, String, String) = + val node = AtomicJson.mapper.readTree(String.valueOf(target.getData)) + val version = requiredText(node, "scalaVersion", target) + val binary = requiredText(node, "scalaBinaryVersion", target) + if !(version.startsWith("2.12.") || version.startsWith("2.13.") || version.startsWith("3.")) then + fail(s"unsupported Scala version $version in ${target.getId.getUri}") + val expectedBinary = if version.startsWith("3.") then "3" else version.split('.').take(2).mkString(".") + if binary != expectedBinary then fail(s"invalid Scala binary version $binary in ${target.getId.getUri}") + val platformNode = node.get("platform") + val platform = + if platformNode == null then fail(s"missing Scala platform in ${target.getId.getUri}") + else if platformNode.isNumber then platformNode.intValue match + case 1 => "jvm" + case 2 => "js" + case 3 => "native" + case value => fail(s"unsupported Scala platform $value in ${target.getId.getUri}") + else platformNode.asText.toLowerCase + (version, binary, platform) + + private def requiredText(node: JsonNode, key: String, target: BuildTarget): String = + val value = if node == null then null else node.get(key) + if value == null || !value.isTextual || value.asText.isEmpty then + fail(s"missing $key in Scala BSP target ${target.getId.getUri}") + value.asText + + private def expandSources(item: SourcesItem): List[SourceDescriptor] = + val expanded = item.getSources.asScala.toList.flatMap { source => + val path = confinedPath(source.getUri, s"source in ${item.getTarget.getUri}") + if Files.isDirectory(path) then + val stream = Files.walk(path) + try stream.iterator.asScala + .filter(file => Files.isRegularFile(file) && file.getFileName.toString.endsWith(".scala")) + .map(file => SourceDescriptor(file.toUri.toString, file, source.getGenerated.booleanValue)) + .toList + finally stream.close() + else if path.getFileName.toString.endsWith(".scala") then + List(SourceDescriptor(source.getUri, path, source.getGenerated.booleanValue)) + else Nil + } + expanded.groupBy(_.path).values.map(_.head).toList.sortBy(_.path.toString) + + private def confinedPath(value: String, label: String): Path = + val path = pathValue(value) + if !path.startsWith(root) then fail(s"BSP $label escapes the project root: $path") + path + + private def pathValue(value: String): Path = + val path = + if value.matches("^[A-Za-z]:[\\\\/].*") then Path.of(value) + else + val parsed = URI.create(value) + if parsed.getScheme == null then Path.of(value) + else if parsed.getScheme == "file" then Path.of(parsed) + else fail(s"unsupported non-file BSP path: $value") + (if path.isAbsolute then path else root.resolve(path)).toAbsolutePath.normalize + + private def ensureAlive(): Unit = + if !process.isAlive then fail(s"BSP server ${details.name} exited") + + override def close(): Unit = + try + if process.isAlive then + try await(server.buildShutdown(), 10) + finally server.onBuildExit() + catch case _: Throwable => () + finally + if process.isAlive then + process.destroy() + if !process.waitFor(5, TimeUnit.SECONDS) then process.destroyForcibly() + executor.shutdownNow() + + private def await[A](future: java.util.concurrent.CompletableFuture[A], seconds: Long = 300): A = + future.get(seconds, TimeUnit.SECONDS) + + private def fail(message: String): Nothing = throw new IllegalStateException(s"samchon-scala-graph: $message") + +private[server] object BspSession: + def open(rootValue: Path): BspSession = + val root = rootValue.toAbsolutePath.normalize + val details = readConnection(root) + val builder = new ProcessBuilder(details.argv.asJava) + .directory(root.toFile) + .redirectError(ProcessBuilder.Redirect.INHERIT) + val process = builder.start() + val client = new DiagnosticClient(root) + val executor = Executors.newCachedThreadPool((task: Runnable) => + val thread = new Thread(task, "samchon-scala-bsp") + thread.setDaemon(true) + thread) + val launcher = new Launcher.Builder[RemoteScalaBuildServer]() + .setLocalService(client) + .setRemoteInterface(classOf[RemoteScalaBuildServer]) + .setInput(process.getInputStream) + .setOutput(process.getOutputStream) + .setExecutorService(executor) + .create() + launcher.startListening() + val session = new BspSession(root, details, process, launcher, client, executor) + try + val params = new InitializeBuildParams( + "samchon-scala-graph", + Main.Version, + details.bspVersion, + root.toUri.toString, + new BuildClientCapabilities(List("scala").asJava)) + val initialized = session.await(launcher.getRemoteProxy.buildInitialize(params), 120) + val compile = initialized.getCapabilities.getCompileProvider + if compile == null || !compile.getLanguageIds.asScala.contains("scala") then + session.fail("BSP server does not advertise Scala compilation") + launcher.getRemoteProxy.onBuildInitialized() + session.reloadSupported = java.lang.Boolean.TRUE == initialized.getCapabilities.getCanReload + session + catch + case error: Throwable => + session.close() + throw error + + private def readConnection(root: Path): ConnectionDetails = + val directory = root.resolve(".bsp") + if !Files.isDirectory(directory) then fail("the project has no .bsp directory") + val stream = Files.list(directory) + val files = try stream.iterator.asScala + .filter(path => Files.isRegularFile(path) && path.getFileName.toString.endsWith(".json")) + .toList.sortBy(_.getFileName.toString) + finally stream.close() + val connections = files.flatMap { file => + val node = AtomicJson.mapper.readTree(Files.readAllBytes(file)) + val languages = Option(node.get("languages")).filter(_.isArray) + .fold(List.empty[String])(_.elements.asScala.map(_.asText).toList) + if !languages.contains("scala") then None + else + val argvNode = node.get("argv") + if argvNode == null || !argvNode.isArray then fail(s"invalid BSP argv in $file") + val argv = argvNode.elements.asScala.map(_.asText).filter(_.nonEmpty).toList + if argv.isEmpty then fail(s"empty BSP argv in $file") + Some(ConnectionDetails( + required(node, "name", file), + argv, + required(node, "bspVersion", file))) + } + connections match + case connection :: Nil => connection + case Nil => fail("the project has no Scala BSP connection") + case _ => fail("the project has more than one Scala BSP connection") + + private def required(node: JsonNode, key: String, file: Path): String = + val value = node.get(key) + if value == null || !value.isTextual || value.asText.isEmpty then fail(s"missing $key in $file") + value.asText + + private def fail(message: String): Nothing = throw new IllegalStateException(s"samchon-scala-graph: $message") + +private[server] final case class ConnectionDetails(name: String, argv: List[String], bspVersion: String) + +private[server] final class DiagnosticClient(root: Path) extends BuildClient: + private val values = TrieMap.empty[String, TrieMap[String, Vector[ch.epfl.scala.bsp4j.Diagnostic]]] + + def clear(target: String): Unit = values.remove(target) + + def diagnostics(target: String, source: String): List[ch.epfl.scala.bsp4j.Diagnostic] = + values.get(target).flatMap(_.get(source)).fold(List.empty[ch.epfl.scala.bsp4j.Diagnostic])(_.toList) + + override def onBuildPublishDiagnostics(params: PublishDiagnosticsParams): Unit = + val target = params.getBuildTarget.getUri + val source = params.getTextDocument.getUri + val targetValues = values.getOrElseUpdate(target, TrieMap.empty) + val incoming = params.getDiagnostics.asScala.toVector + if java.lang.Boolean.TRUE == params.getReset then targetValues.put(source, incoming) + else targetValues.updateWith(source)(previous => Some(previous.getOrElse(Vector.empty) ++ incoming)) + + override def onBuildShowMessage(params: ShowMessageParams): Unit = () + override def onBuildLogMessage(params: LogMessageParams): Unit = () + override def onBuildTargetDidChange(params: DidChangeBuildTarget): Unit = () + override def onBuildTaskStart(params: TaskStartParams): Unit = () + override def onBuildTaskProgress(params: TaskProgressParams): Unit = () + override def onBuildTaskFinish(params: TaskFinishParams): Unit = () diff --git a/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/Main.scala b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/Main.scala new file mode 100644 index 00000000..2cf72c5d --- /dev/null +++ b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/Main.scala @@ -0,0 +1,76 @@ +package org.samchon.graph.scala.server + +import com.fasterxml.jackson.databind.node.ObjectNode +import java.io.{BufferedReader, InputStreamReader} +import java.nio.file.Path + +object Main: + val Version = "0.1.0-SNAPSHOT" + private val Capability = "Serve BSP-driven Scala compiler graph generations over NDJSON." + + def main(arguments: Array[String]): Unit = + try run(arguments.toList) + catch + case error: Throwable => + System.err.println(message(error)) + System.exit(1) + + private def run(arguments: List[String]): Unit = arguments match + case "--version" :: Nil => println(s"samchon-scala-graph $Version") + case "graph-server" :: rest if rest.contains("--help") => println(Capability) + case "supports" :: rest => + val producer = new SnapshotProducer(cwd(rest)) + try producer.supports() finally producer.close() + case "snapshot" :: rest => + val producer = new SnapshotProducer(cwd(rest)) + try producer.produce(requiredPath(rest, "--output")) finally producer.close() + case "graph-server" :: rest => serve(cwd(rest)) + case _ => throw new IllegalArgumentException( + "usage: samchon-scala-graph --version | supports --cwd ROOT | snapshot --cwd ROOT --output FILE | graph-server --cwd ROOT") + + private def serve(root: Path): Unit = + val producer = new SnapshotProducer(root) + val reader = new BufferedReader(new InputStreamReader(System.in, java.nio.charset.StandardCharsets.UTF_8)) + try + Iterator.continually(reader.readLine()).takeWhile(_ != null).foreach { line => + if line.trim.nonEmpty then respond(producer, line) + } + finally producer.close() + + private def respond(producer: SnapshotProducer, line: String): Unit = + var id = -1L + try + val request = AtomicJson.mapper.readTree(line) + if request == null || !request.isObject then throw new IllegalArgumentException("request must be an object") + val idNode = request.get("id") + if idNode == null || !idNode.canConvertToLong || idNode.longValue < 0 then + throw new IllegalArgumentException("request id must be a non-negative integer") + id = idNode.longValue + val protocol = request.get("protocolVersion") + if protocol == null || !protocol.isInt || protocol.intValue != 1 then + throw new IllegalArgumentException("unsupported protocol version") + val output = request.get("output") + if output == null || !output.isTextual || output.asText.isEmpty then + throw new IllegalArgumentException("request output must be a path") + producer.produce(Path.of(output.asText)) + writeResponse(id, true, null) + catch case error: Throwable => writeResponse(id, false, message(error)) + + private def writeResponse(id: Long, ok: Boolean, error: String | Null): Unit = + val response = AtomicJson.mapper.createObjectNode() + response.put("id", id) + response.put("protocolVersion", 1) + response.put("ok", ok) + if error != null then response.put("error", error) + System.out.println(AtomicJson.mapper.writeValueAsString(response)) + System.out.flush() + + private def cwd(arguments: List[String]): Path = requiredPath(arguments, "--cwd") + + private def requiredPath(arguments: List[String], option: String): Path = + arguments.sliding(2).collectFirst { case List(`option`, value) if value.nonEmpty => Path.of(value) } + .getOrElse(throw new IllegalArgumentException(s"$option is required")) + + private def message(error: Throwable): String = + Iterator.iterate(error)(_.getCause).takeWhile(_ != null).map(current => + Option(current.getMessage).filter(_.nonEmpty).getOrElse(current.getClass.getSimpleName)).toList.last diff --git a/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/SemanticDbReader.scala b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/SemanticDbReader.scala new file mode 100644 index 00000000..8d1370a8 --- /dev/null +++ b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/SemanticDbReader.scala @@ -0,0 +1,219 @@ +package org.samchon.graph.scala.server + +import java.nio.file.{Files, Path} +import java.security.MessageDigest +import java.util.HexFormat +import org.samchon.graph.scala.model.{Evidence, GraphEdge, GraphNode, TypedShard} +import org.samchon.graph.scala.plugin.GraphShardWriter +import org.samchon.graph.scala.server.model.{DiagnosticFact, SemanticShard} +import scala.jdk.CollectionConverters.* +import scala.meta.internal.semanticdb.{Diagnostic as SemanticDiagnostic, Range as SemanticRange, SymbolInformation, SymbolOccurrence, TextDocument, TextDocuments} + +private[server] final class SemanticDbReader(root: Path, target: TargetDescriptor, bsp: BspSession): + private val documents = loadDocuments() + + def shard(source: SourceDescriptor): SemanticShard = + val relative = relativeSource(source.path) + val typed = loadTyped(relative) + val document = documents.getOrElse(relative, + fail(s"SemanticDB omitted $relative in ${target.uri}")) + validateTyped(typed, source.path, relative) + validateSemanticdb(document, source.path, relative, typed) + val diagnostics = ( + document.diagnostics.toList.map(semanticDiagnostic(relative, _)) ++ + bsp.diagnostics(target.uri, source.path).map(bspDiagnostic(relative, _)) + ).distinct.sortBy(value => ( + value.evidence.startLine, + value.evidence.startColumn, + value.severity, + value.message)) + new SemanticShard( + typed.schemaVersion, + typed.language, + typed.source, + typed.checkerDigest, + typed.diskDigest, + target.uri, + typed.compilerVersion, + typed.compilerPlugin, + typed.compilerPluginVersion, + document.schema.value, + relative, + document.md5.toLowerCase, + target.uri, + typed.nodes, + typed.edges, + typed.unresolved, + diagnostics.asJava) + + private def loadDocuments(): Map[String, TextDocument] = + val directory = target.classDirectory.resolve("META-INF").resolve("semanticdb") + if !Files.isDirectory(directory) then + fail(s"SemanticDB output is missing for ${target.uri}: $directory") + val stream = Files.walk(directory) + val entries = try stream.iterator.asScala + .filter(path => Files.isRegularFile(path) && path.getFileName.toString.endsWith(".semanticdb")) + .toList.sortBy(_.toString) + .flatMap(path => TextDocuments.parseFrom(Files.readAllBytes(path)).documents.toList) + finally stream.close() + val grouped = entries.groupBy(document => normalize(document.uri)) + grouped.map { case (uri, values) => + val source = root.resolve(uri).normalize + val current = + if source.startsWith(root) && Files.isRegularFile(source) then + val expected = md5(Files.readAllBytes(source)) + values.filter(_.md5.equalsIgnoreCase(expected)).distinct + else Nil + val document = current match + case value :: Nil => value + case Nil if values.size == 1 => values.head + case Nil => fail(s"duplicate stale SemanticDB document $uri in ${target.uri}") + case _ => fail(s"conflicting current SemanticDB documents $uri in ${target.uri}") + uri -> document + } + + private def loadTyped(relative: String): TypedShard = + val targetKey = GraphShardWriter.digest(target.uri.getBytes(java.nio.charset.StandardCharsets.UTF_8)) + val sourceKey = GraphShardWriter.digest(relative.getBytes(java.nio.charset.StandardCharsets.UTF_8)) + val path = target.graphOutput.resolve("typed").resolve(targetKey).resolve(sourceKey + ".json") + if !Files.isRegularFile(path) then fail(s"typed compiler shard is missing for ${target.uri} $relative") + AtomicJson.mapper.readValue(Files.readAllBytes(path), classOf[TypedShard]) + + private def validateTyped(typed: TypedShard, source: Path, relative: String): Unit = + val bytes = Files.readAllBytes(source) + val diskDigest = GraphShardWriter.digest(bytes) + if typed.schemaVersion != 1 || typed.language != "scala" || typed.source != relative then + fail(s"malformed typed compiler shard for ${target.uri} $relative") + if typed.target != target.uri || typed.compilerVersion != target.scalaVersion || + typed.compilerPlugin != target.expectedPlugin || typed.compilerPluginVersion.isEmpty then + fail(s"typed compiler identity does not match ${target.uri} $relative") + if typed.diskDigest != diskDigest || typed.checkerDigest != diskDigest then + fail(s"typed compiler source digest does not match ${target.uri} $relative") + if typed.nodes.isEmpty then fail(s"typed compiler shard has no declarations for ${target.uri} $relative") + + private def validateSemanticdb( + document: TextDocument, + source: Path, + relative: String, + typed: TypedShard + ): Unit = + if !document.schema.isSemanticdb4 then fail(s"unsupported SemanticDB schema for ${target.uri} $relative") + if normalize(document.uri) != relative then fail(s"SemanticDB URI does not match $relative") + val expectedMd5 = md5(Files.readAllBytes(source)) + if !document.md5.equalsIgnoreCase(expectedMd5) then fail(s"SemanticDB md5 does not match $relative") + if document.buildTarget.nonEmpty && document.buildTarget != target.uri then + fail(s"SemanticDB build target does not match ${target.uri} $relative") + + val definitions = document.occurrences.toList + .filter(_.role == SymbolOccurrence.Role.DEFINITION) + val information = document.symbols.toList.groupBy(_.displayName) + typed.nodes.asScala.filter(_.origin != "Synthetic").foreach { node => + val semanticNames = + if node.kind == "constructor" then + List(node.qualifiedName.split('.').dropRight(1).lastOption.getOrElse(node.name).stripSuffix("$")) + else List(node.name) + val named = semanticNames.flatMap(name => information.getOrElse(name, Nil)) + val namedSymbols = named.map(_.symbol).toSet + val positionedSymbols = definitions.iterator.filter(occurrence => + namedSymbols.contains(occurrence.symbol) && + (node.kind == "constructor" || occurrence.range.exists(range => overlaps(node.evidence, range)))) + .map(_.symbol).toSet + val anonymousContextual = node.modifiers.contains("declare") && named.nonEmpty + val matched = + if positionedSymbols.nonEmpty then named.filter(info => positionedSymbols.contains(info.symbol)) + else if anonymousContextual then named + else Nil + // SemanticDB records package definitions as occurrences but commonly + // omits their SymbolInformation row. The typed plugin still owns the + // declaration; cross-check it against the canonical package symbol. + val packageMatched = node.kind == "package" && definitions.exists(occurrence => + occurrence.symbol == node.qualifiedName.replace('.', '/') + "/" && + occurrence.range.exists(range => overlaps(node.evidence, range))) + if matched.isEmpty && !packageMatched then + fail(s"typed declaration ${node.qualifiedName} has no SemanticDB definition in $relative") + // Case-class constructor parameters have SymbolInformation entries, but + // SemanticDB positions the generated accessor method at their source + // token instead of the parameter symbol itself. + val kindMatched = + if node.kind == "parameter" then named.filter(info => compatibleKind(node.kind, info.kind)) + else matched.filter(info => compatibleKind(node.kind, info.kind)) + if kindMatched.isEmpty && !packageMatched then + fail(s"typed declaration ${node.qualifiedName} disagrees with SemanticDB kind in $relative") + if !Set("package", "type", "parameter").contains(node.kind) && + (node.signature.isEmpty || matched.forall(_.signature.isEmpty)) then + fail(s"typed declaration ${node.qualifiedName} has no SemanticDB signature in $relative") + val outgoing = typed.edges.asScala.filter(_.from == node.symbol) + if outgoing.exists(_.kind == "overrides") && matched.forall(_.overriddenSymbols.isEmpty) then + fail(s"typed override ${node.qualifiedName} is absent from SemanticDB in $relative") + if outgoing.exists(_.kind == "decorates") && matched.forall(_.annotations.isEmpty) then + fail(s"typed annotation ${node.qualifiedName} is absent from SemanticDB in $relative") + } + val semanticEdges = typed.edges.asScala.filter(edge => + Set("imports", "references", "type_ref").contains(edge.kind)) + if semanticEdges.nonEmpty && !document.occurrences.exists(_.role == SymbolOccurrence.Role.REFERENCE) then + fail(s"typed semantic references are absent from SemanticDB in $relative") + + private def compatibleKind(typed: String, semantic: SymbolInformation.Kind): Boolean = + typed match + case "package" => semantic.isPackage || semantic.isPackageObject + case "class" => semantic.isClass + case "interface" => semantic.isTrait || semantic.isInterface + case "module" => semantic.isObject || semantic.isPackageObject + case "constructor" => + semantic.isClass || semantic.isTrait || semantic.isInterface || semantic.isObject + case "method" | "function" => semantic.isMethod || semantic.isMacro + case "field" | "property" | "variable" => + semantic.isField || semantic.isMethod || semantic.isLocal + case "parameter" => + semantic.isParameter || semantic.isSelfParameter || semantic.isTypeParameter + case "type" => semantic.isType || semantic.isTypeParameter + case _ => false + + private def semanticDiagnostic(relative: String, value: SemanticDiagnostic): DiagnosticFact = + val evidence = value.range.fold(new Evidence(relative, 1, 1, 1, 1))(semanticEvidence(relative, _)) + new DiagnosticFact(diagnosticSeverity(value.severity.toString), value.message, evidence) + + private def bspDiagnostic(relative: String, value: ch.epfl.scala.bsp4j.Diagnostic): DiagnosticFact = + val range = value.getRange + val evidence = new Evidence( + relative, + range.getStart.getLine + 1, + range.getStart.getCharacter + 1, + range.getEnd.getLine + 1, + range.getEnd.getCharacter + 1) + val severity = Option(value.getSeverity).fold("info")(value => diagnosticSeverity(value.toString)) + new DiagnosticFact(severity, value.getMessage, evidence) + + private def diagnosticSeverity(value: String): String = + value.toLowerCase match + case "information" => "info" + case severity => severity + + private def semanticEvidence(relative: String, range: SemanticRange): Evidence = + new Evidence( + relative, + range.startLine + 1, + range.startCharacter + 1, + range.endLine + 1, + range.endCharacter + 1) + + private def overlaps(evidence: Evidence, range: SemanticRange): Boolean = + val start = (range.startLine + 1, range.startCharacter + 1) + val end = (range.endLine + 1, range.endCharacter + 1) + val evidenceStart = (evidence.startLine, evidence.startColumn) + val evidenceEnd = (evidence.endLine, evidence.endColumn) + beforeOrEqual(start, evidenceEnd) && beforeOrEqual(evidenceStart, end) + + private def beforeOrEqual(left: (Int, Int), right: (Int, Int)): Boolean = + left._1 < right._1 || left._1 == right._1 && left._2 <= right._2 + + private def relativeSource(path: Path): String = + root.relativize(path.toAbsolutePath.normalize).toString.replace('\\', '/') + + private def normalize(value: String): String = + value.replace('\\', '/').stripPrefix("./") + + private def md5(bytes: Array[Byte]): String = + HexFormat.of.formatHex(MessageDigest.getInstance("MD5").digest(bytes)) + + private def fail(message: String): Nothing = throw new IllegalStateException(s"samchon-scala-graph: $message") diff --git a/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/SnapshotProducer.scala b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/SnapshotProducer.scala new file mode 100644 index 00000000..b0389a37 --- /dev/null +++ b/sidecars/scala/server/src/main/scala/org/samchon/graph/scala/server/SnapshotProducer.scala @@ -0,0 +1,210 @@ +package org.samchon.graph.scala.server + +import java.nio.charset.StandardCharsets +import java.net.URI +import java.nio.file.attribute.{BasicFileAttributes, FileTime} +import java.nio.file.{Files, Path} +import java.util.LinkedHashMap +import org.samchon.graph.scala.plugin.GraphShardWriter +import org.samchon.graph.scala.server.model.{SnapshotArtifact, TargetSnapshot} +import scala.collection.mutable +import scala.jdk.CollectionConverters.* + +/** Builds and publishes one all-target generation from the repository's own BSP compile. */ +private[server] final class SnapshotProducer(rootValue: Path) extends AutoCloseable: + private val root = rootValue.toAbsolutePath.normalize + private var bsp: BspSession | Null = null + private val contentDigests = mutable.HashMap.empty[Path, CachedDigest] + + def supports(): Unit = withSession { session => + session.reload() + session.describeTargets() + } + + def produce(outputValue: Path): Unit = + val output = outputValue.toAbsolutePath.normalize + withSession { session => + session.reload() + val before = session.describeTargets() + session.compile(before) + val targets = session.describeTargets() + if before.map(_.uri) != targets.map(_.uri) then fail("BSP target set changed during compile") + val snapshots = targets.map(targetSnapshot(session, _)) + snapshots.foreach { target => + val descriptor = targets.find(_.uri == target.bspUri).get + val manifest = descriptor.graphOutput.resolve("manifests") + .resolve(digest(target.bspUri) + ".json") + AtomicJson.write(manifest, target) + } + val artifact = new SnapshotArtifact( + 1, + root.toString, + new SnapshotArtifact.Producer( + "samchon-scala-graph", + Main.Version, + 1, + new SnapshotArtifact.Capabilities(true, true, true, true, true, true, true)), + snapshots.asJava) + AtomicJson.write(output, artifact) + } + + private def targetSnapshot(session: BspSession, target: TargetDescriptor): TargetSnapshot = + val reader = new SemanticDbReader(root, target, session) + val sources = target.sources.filter(source => Files.isRegularFile(source.path)) + if sources.isEmpty then fail(s"BSP target ${target.uri} has no Scala source files") + val shards = sources.map(reader.shard) + val scalacOptionsDigest = digestRows(target.options) + // BSP commonly includes this target's own output directory in its + // classpath. It is derived output, not a build input, and retaining it + // makes a restored source tree inherit stale Zinc history. Other target + // outputs remain real inputs and are still fenced by their bytes. + val classpathDigest = digestRows(target.classpath + .filterNot(value => classpathPath(value) == target.classDirectory) + .map(classpathCoordinate)) + val sourceRootsDigest = digestSet( + target.sourceRoots ++ sources.flatMap(source => Option(relative(source.path).getParent).map(_.toString))) + val semanticdbOptionsDigest = digestRows(target.options.filter(_.toLowerCase.contains("semanticdb"))) + val compilerPluginsDigest = digestRows(target.options.filter(option => + option.startsWith("-Xplugin:") || option.startsWith("-P:samchon-graph:"))) + val zincAnalysisDigest = digestSet(zincCoordinates(target.classDirectory)) + val generatedSourcesDigest = digestSet(sources.filter(_.generated).map(source => relative(source.path).toString)) + val universe = digestRows(List( + target.uri, + target.scalaVersion, + target.scalaBinaryVersion, + target.platform, + sourceEncoding(target.options), + scalacOptionsDigest, + classpathDigest, + sourceRootsDigest, + semanticdbOptionsDigest, + compilerPluginsDigest, + zincAnalysisDigest, + generatedSourcesDigest)) + val generation = digestRows(universe :: shards.flatMap(shard => List( + shard.source, + shard.diskDigest, + shard.checkerDigest, + shard.semanticdbMd5, + digestBytes(AtomicJson.mapper.writeValueAsBytes(shard))))) + new TargetSnapshot( + target.uri, + generation, + universe, + target.uri, + target.scalaVersion, + target.scalaBinaryVersion, + target.platform, + sourceEncoding(target.options), + scalacOptionsDigest, + classpathDigest, + sourceRootsDigest, + semanticdbOptionsDigest, + compilerPluginsDigest, + zincAnalysisDigest, + generatedSourcesDigest, + coverage, + shards.asJava) + + private def zincCoordinates(classDirectory: Path): List[String] = + val parent = Option(classDirectory.getParent).getOrElse(classDirectory) + if !Files.isDirectory(parent) then List(parent.toString) + else + val stream = Files.walk(parent, 2) + try stream.iterator.asScala + .filter(path => Files.isRegularFile(path) && path.getFileName.toString.toLowerCase.contains("inc_compile")) + .map(path => root.relativize(path.toAbsolutePath.normalize).toString.replace('\\', '/')) + .toList.sorted match + case Nil => List(root.relativize(classDirectory).toString.replace('\\', '/')) + case values => values + finally stream.close() + + /** Preserve classpath order while fencing every file by its actual bytes. */ + private def classpathCoordinate(value: String): String = + val path = classpathPath(value) + if Files.isRegularFile(path) then s"$value\u0000${contentDigest(path)}" + else if Files.isDirectory(path) then + val stream = Files.walk(path) + val files = try stream.iterator.asScala + .filter(Files.isRegularFile(_)) + .toList.sortBy(_.toString) + finally stream.close() + val rows = files.map(file => + s"${path.relativize(file).toString.replace('\\', '/')}\u0000${contentDigest(file)}") + s"$value\u0000${digestRows(rows)}" + else fail(s"classpath entry is missing: $value") + + private def classpathPath(value: String): Path = + val path = + if value.matches("^[A-Za-z]:[\\\\/].*") then Path.of(value) + else + val parsed = URI.create(value) + if parsed.getScheme == null then Path.of(value) + else if parsed.getScheme == "file" then Path.of(parsed) + else fail(s"unsupported non-file classpath entry: $value") + (if path.isAbsolute then path else root.resolve(path)).toAbsolutePath.normalize + + private def contentDigest(path: Path): String = + val attributes = Files.readAttributes(path, classOf[BasicFileAttributes]) + val stamp = FileStamp( + attributes.size, + attributes.lastModifiedTime, + Option(attributes.fileKey).fold("")(_.toString)) + contentDigests.get(path) match + case Some(cached) if cached.stamp == stamp => cached.digest + case _ => + val value = digestBytes(Files.readAllBytes(path)) + contentDigests.put(path, CachedDigest(stamp, value)) + value + + private def coverage: java.util.Map[String, String] = + val values = new LinkedHashMap[String, String]() + List( + "contains", "exports", "imports", "calls", "accesses", "instantiates", + "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", + "renders", "tests", "references" + ).foreach(family => values.put( + family, + if Set("renders", "tests").contains(family) then "unsupported" else "partial")) + values.put("contains", "complete") + values.put("calls", "partial") + values + + private def withSession[A](body: BspSession => A): A = + if bsp == null then bsp = BspSession.open(root) + try body(bsp.nn) + catch + case error: Throwable => + bsp.nn.close() + bsp = null + throw error + + override def close(): Unit = + if bsp != null then + bsp.nn.close() + bsp = null + + private def relative(path: Path): Path = root.relativize(path.toAbsolutePath.normalize) + private def sourceEncoding(options: List[String]): String = + val separated = options.sliding(2).collect { + case List("-encoding", value) if value.nonEmpty => value + }.toList + val attached = options.flatMap { option => + List("-encoding:", "-encoding=").collectFirst { + case prefix if option.startsWith(prefix) && option.length > prefix.length => + option.substring(prefix.length) + } + } + (separated ++ attached).distinct match + case Nil => "UTF-8" + case value :: Nil => value + case _ => fail("scalac configures more than one source encoding") + private def digest(value: String): String = digestBytes(value.getBytes(StandardCharsets.UTF_8)) + private def digestRows(values: Iterable[String]): String = + digestBytes(AtomicJson.mapper.writeValueAsBytes(values.toList.asJava)) + private def digestSet(values: Iterable[String]): String = digestRows(values.toList.distinct.sorted) + private def digestBytes(bytes: Array[Byte]): String = GraphShardWriter.digest(bytes) + private def fail(message: String): Nothing = throw new IllegalStateException(s"samchon-scala-graph: $message") + +private final case class FileStamp(size: Long, modified: FileTime, fileKey: String) +private final case class CachedDigest(stamp: FileStamp, digest: String) diff --git a/sidecars/swift/Package.resolved b/sidecars/swift/Package.resolved new file mode 100644 index 00000000..5e261111 --- /dev/null +++ b/sidecars/swift/Package.resolved @@ -0,0 +1,13 @@ +{ + "pins" : [ + { + "identity" : "indexstore-db", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/indexstore-db.git", + "state" : { + "revision" : "54212fce1aecb199070808bdb265e7f17e396015" + } + } + ], + "version" : 2 +} diff --git a/sidecars/swift/Package.swift b/sidecars/swift/Package.swift new file mode 100644 index 00000000..e50604a9 --- /dev/null +++ b/sidecars/swift/Package.swift @@ -0,0 +1,26 @@ +// swift-tools-version: 5.10 + +import PackageDescription + +let package = Package( + name: "SamchonSwiftGraph", + platforms: [.macOS(.v13)], + products: [ + .executable(name: "samchon-swift-graph", targets: ["SamchonSwiftGraph"]), + ], + dependencies: [ + .package( + url: "https://github.com/swiftlang/indexstore-db.git", + revision: "54212fce1aecb199070808bdb265e7f17e396015" + ), + ], + targets: [ + .executableTarget( + name: "SamchonSwiftGraph", + dependencies: [ + .product(name: "IndexStoreDB", package: "indexstore-db"), + ] + ), + ], + cxxLanguageStandard: .cxx17 +) diff --git a/sidecars/swift/README.md b/sidecars/swift/README.md new file mode 100644 index 00000000..0510d57d --- /dev/null +++ b/sidecars/swift/README.md @@ -0,0 +1,39 @@ +# Swift IndexStoreDB sidecar + +This SwiftPM package builds `samchon-swift-graph`, the standalone strict Swift +producer shipped as source with `@samchon/graph`. It runs the package's native +incremental +`swift build --enable-index-store --build-tests -Xswiftc -index-include-locals`, +takes only the current build description's source and object paths, opens the +completed store with IndexStoreDB's explicit-output-unit mode, and commits one +atomic graph artifact. + +IndexStoreDB is pinned to the Swift 6.1 release commit +`54212fce1aecb199070808bdb265e7f17e396015`. That release retains Swift 6.0 source +compatibility and fixes the 64-bit canonical-role declaration for Clang 19 and +newer. The binary must run with a compatible Swift toolchain and +`libIndexStore`. macOS and Linux are supported; Windows declines the strict +route. + +Build and expose the executable on `PATH`, or set +`SAMCHON_GRAPH_SWIFT_GRAPH` to its absolute path: + +```bash +swift build --package-path sidecars/swift -c release +``` + +On Linux, IndexStoreDB's C++ targets also need the Swift toolchain's dispatch +headers, as documented upstream: + +```bash +SWIFT_ROOT="$(dirname "$(dirname "$(command -v swift)")")" +swift build --package-path sidecars/swift -c release \ + -Xcxx "-I${SWIFT_ROOT}/lib/swift" \ + -Xcxx "-I${SWIFT_ROOT}/lib/swift/Block" +``` + +The sidecar keeps its process resident, but it does not claim ownership of +SourceKit-LSP's scheduler or caches. Each changed generation is a completed +SwiftPM build followed by a frozen IndexStoreDB query. The ordinary +SourceKit-LSP/static route remains the fallback when the sidecar or matching +toolchain is unavailable. diff --git a/sidecars/swift/Sources/SamchonSwiftGraph/GraphModel.swift b/sidecars/swift/Sources/SamchonSwiftGraph/GraphModel.swift new file mode 100644 index 00000000..f8400d3c --- /dev/null +++ b/sidecars/swift/Sources/SamchonSwiftGraph/GraphModel.swift @@ -0,0 +1,114 @@ +import Foundation + +struct GraphArtifact: Codable { + let schemaVersion: Int + let projectRoot: String + let producer: Producer + let targets: [TargetArtifact] +} + +struct Producer: Codable { + let name: String + let version: String + let protocolVersion: Int + let capabilities: Capabilities +} + +struct Capabilities: Codable { + let atomicGenerations: Bool + let incremental: Bool + let diagnostics: Bool + let explicitOutputUnits: Bool + let indexStoreDB: Bool + let sourceEnrichment: Bool + let swiftpm: Bool + let sourceKitResident: Bool +} + +struct TargetArtifact: Codable { + let name: String + let generation: String + let universe: String + let moduleName: String + let targetTriple: String + let sdk: String + let configuration: String + let swiftLanguageVersion: String + let compilerFlagsDigest: String + let moduleDependenciesDigest: String + let packageResolutionDigest: String + let pluginsDigest: String + let generatedSourcesDigest: String + let indexStoreDBCommit: String + let outputUnits: [OutputUnit] + let coverage: [String: String] + let shards: [SourceShard] +} + +struct OutputUnit: Codable, Equatable { + let path: String + let digest: String +} + +struct SourceShard: Codable { + let schemaVersion: Int + let language: String + let source: String + let checkerDigest: String + let diskDigest: String + let target: String + let compilerVersion: String + let moduleName: String + let targetTriple: String + let sourceEnrichmentPasses: Int + let nodes: [GraphNode] + let edges: [GraphEdge] + let unresolved: [UnresolvedSite] + let diagnostics: [DiagnosticFact] +} + +struct Evidence: Codable, Hashable { + let file: String + let startLine: Int + let startColumn: Int + let endLine: Int + let endColumn: Int +} + +struct GraphNode: Codable { + let symbol: String + let kind: String + let name: String + let qualifiedName: String + let file: String + let exported: Bool + let modifiers: [String] + let signature: String + let origin: String + let evidence: Evidence +} + +struct GraphEdge: Codable { + let from: String + let to: String + let kind: String + let access: String? + let provenance: String? + let targetKind: String? + let targetName: String? + let targetQualifiedName: String? + let evidence: Evidence +} + +struct UnresolvedSite: Codable { + let family: String + let reason: String + let evidence: Evidence + let candidates: [String] +} + +struct DiagnosticFact: Codable { + let severity: String + let message: String + let evidence: Evidence +} diff --git a/sidecars/swift/Sources/SamchonSwiftGraph/SHA256.swift b/sidecars/swift/Sources/SamchonSwiftGraph/SHA256.swift new file mode 100644 index 00000000..75104f6d --- /dev/null +++ b/sidecars/swift/Sources/SamchonSwiftGraph/SHA256.swift @@ -0,0 +1,94 @@ +import Foundation + +enum SHA256 { + static func hash(_ data: Data) -> String { + var message = [UInt8](data) + let bitLength = UInt64(message.count) * 8 + message.append(0x80) + while message.count % 64 != 56 { message.append(0) } + message.append(contentsOf: withUnsafeBytes(of: bitLength.bigEndian, Array.init)) + + var state: [UInt32] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, + ] + for offset in stride(from: 0, to: message.count, by: 64) { + var words = [UInt32](repeating: 0, count: 64) + for index in 0..<16 { + let at = offset + index * 4 + words[index] = + UInt32(message[at]) << 24 | + UInt32(message[at + 1]) << 16 | + UInt32(message[at + 2]) << 8 | + UInt32(message[at + 3]) + } + for index in 16..<64 { + let s0 = rotate(words[index - 15], by: 7) ^ + rotate(words[index - 15], by: 18) ^ (words[index - 15] >> 3) + let s1 = rotate(words[index - 2], by: 17) ^ + rotate(words[index - 2], by: 19) ^ (words[index - 2] >> 10) + words[index] = words[index - 16] &+ s0 &+ words[index - 7] &+ s1 + } + var a = state[0] + var b = state[1] + var c = state[2] + var d = state[3] + var e = state[4] + var f = state[5] + var g = state[6] + var h = state[7] + for index in 0..<64 { + let sum1 = rotate(e, by: 6) ^ rotate(e, by: 11) ^ rotate(e, by: 25) + let choose = (e & f) ^ ((~e) & g) + let temporary1 = h &+ sum1 &+ choose &+ constants[index] &+ words[index] + let sum0 = rotate(a, by: 2) ^ rotate(a, by: 13) ^ rotate(a, by: 22) + let majority = (a & b) ^ (a & c) ^ (b & c) + let temporary2 = sum0 &+ majority + h = g + g = f + f = e + e = d &+ temporary1 + d = c + c = b + b = a + a = temporary1 &+ temporary2 + } + state[0] &+= a + state[1] &+= b + state[2] &+= c + state[3] &+= d + state[4] &+= e + state[5] &+= f + state[6] &+= g + state[7] &+= h + } + return state.map { String(format: "%08x", $0) }.joined() + } + + static func hash(_ text: String) -> String { + hash(Data(text.utf8)) + } + + private static func rotate(_ value: UInt32, by count: UInt32) -> UInt32 { + (value >> count) | (value << (32 - count)) + } + + private static let constants: [UInt32] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, + ] +} diff --git a/sidecars/swift/Sources/SamchonSwiftGraph/SourceEnrichment.swift b/sidecars/swift/Sources/SamchonSwiftGraph/SourceEnrichment.swift new file mode 100644 index 00000000..3a45a5c7 --- /dev/null +++ b/sidecars/swift/Sources/SamchonSwiftGraph/SourceEnrichment.swift @@ -0,0 +1,168 @@ +import Foundation + +struct ImportFact { + let module: String + let evidence: Evidence +} + +struct AttributeFact { + let name: String + let evidence: Evidence +} + +struct SourceEnrichment { + let relativePath: String + let data: Data + private let lines: [String] + + init(url: URL, relativePath: String) throws { + self.relativePath = relativePath + data = try Data(contentsOf: url) + lines = String(decoding: data, as: UTF8.self) + .split(separator: "\n", omittingEmptySubsequences: false) + .map(String.init) + } + + var digest: String { SHA256.hash(data) } + + func evidence(line: Int, utf8Column: Int, name: String) -> Evidence { + let lineNumber = min(max(line, 1), max(lines.count, 1)) + let text = lines.indices.contains(lineNumber - 1) ? lines[lineNumber - 1] : "" + let bytes = Array(text.utf8) + let requested = min(max(utf8Column - 1, 0), bytes.count) + let needle = Array(name.utf8) + let found = find(needle, in: bytes, atOrAfter: requested) + ?? find(needle, in: bytes, atOrAfter: 0) + ?? requested + return Evidence( + file: relativePath, + startLine: lineNumber, + startColumn: found + 1, + endLine: lineNumber, + endColumn: found + max(needle.count, 1) + 1 + ) + } + + func signature(line: Int) -> String { + guard lines.indices.contains(line - 1) else { return "" } + var value = lines[line - 1].trimmingCharacters(in: .whitespaces) + if let body = value.firstIndex(of: "{") { + value = String(value[.. Bool { + let head = signature(line: line) + return word("public", in: head) || word("open", in: head) || word("package", in: head) + } + + func isStaticallyClosed(line: Int) -> Bool { + let head = signature(line: line) + return word("final", in: head) || word("static", in: head) || word("private", in: head) + } + + func imports() -> [ImportFact] { + let expression = try! NSRegularExpression( + pattern: #"^\s*(?:@testable\s+)?import\s+(?:(?:struct|class|enum|protocol|typealias|func|var|let)\s+)?([A-Za-z_][A-Za-z0-9_]*)"# + ) + return lines.enumerated().compactMap { index, line in + let range = NSRange(line.startIndex.. [AttributeFact] { + let expression = try! NSRegularExpression( + pattern: #"@([A-Za-z_][A-Za-z0-9_.]*)"# + ) + var rows: [(Int, String)] = [] + var index = min(max(line - 1, 0), max(lines.count - 1, 0)) + while lines.indices.contains(index) { + let text = lines[index].trimmingCharacters(in: .whitespaces) + if index != line - 1 && !text.hasPrefix("@") { break } + rows.append((index, lines[index])) + if index == 0 { break } + index -= 1 + } + return rows.reversed().flatMap { row, text in + let range = NSRange(text.startIndex.. AttributeFact? in + guard let nameRange = Range(match.range(at: 1), in: text) else { return nil } + let name = String(text[nameRange]) + let column = text.utf8.distance(from: text.utf8.startIndex, to: nameRange.lowerBound.samePosition(in: text.utf8)!) + 1 + return AttributeFact( + name: name, + evidence: evidence(line: row + 1, utf8Column: column, name: name) + ) + } + } + } + + func unresolvedSyntax() -> [UnresolvedSite] { + lines.enumerated().flatMap { index, line -> [UnresolvedSite] in + var sites: [UnresolvedSite] = [] + if let range = line.range(of: "#if") { + let column = line.utf8.distance(from: line.utf8.startIndex, to: range.lowerBound.samePosition(in: line.utf8)!) + 1 + sites.append(UnresolvedSite( + family: "references", + reason: "conditional-build", + evidence: evidence(line: index + 1, utf8Column: column, name: "#if"), + candidates: [] + )) + } + if let range = line.range(of: "#externalMacro") ?? line.range(of: "#") { + let token = line[range.lowerBound...].prefix { $0 == "#" || $0.isLetter || $0.isNumber || $0 == "_" } + let name = String(token) + if name != "#if" && name != "#else" && name != "#endif" { + let column = line.utf8.distance(from: line.utf8.startIndex, to: range.lowerBound.samePosition(in: line.utf8)!) + 1 + sites.append(UnresolvedSite( + family: "references", + reason: "macro-or-generated", + evidence: evidence(line: index + 1, utf8Column: column, name: name), + candidates: [] + )) + } + } + return sites + } + } + + private func word(_ value: String, in text: String) -> Bool { + text.split { !$0.isLetter && !$0.isNumber && $0 != "_" }.contains(Substring(value)) + } + + private func find(_ needle: [UInt8], in haystack: [UInt8], atOrAfter start: Int) -> Int? { + guard !needle.isEmpty, needle.count <= haystack.count else { return nil } + let lower = min(max(start, 0), haystack.count - needle.count) + for index in lower...(haystack.count - needle.count) { + if Array(haystack[index..<(index + needle.count)]) == needle { return index } + } + return nil + } +} diff --git a/sidecars/swift/Sources/SamchonSwiftGraph/SwiftGraphProducer.swift b/sidecars/swift/Sources/SamchonSwiftGraph/SwiftGraphProducer.swift new file mode 100644 index 00000000..3fb7c2db --- /dev/null +++ b/sidecars/swift/Sources/SamchonSwiftGraph/SwiftGraphProducer.swift @@ -0,0 +1,804 @@ +import Foundation +import IndexStoreDB + +private let indexStoreDBCommit = "54212fce1aecb199070808bdb265e7f17e396015" +private let factFamilies = [ + "contains", "exports", "imports", "calls", "accesses", "instantiates", + "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", + "renders", "tests", "references", +] + +struct SwiftGraphProducer { + let root: URL + + static func supports(root: URL) -> Bool { + guard FileManager.default.fileExists(atPath: root.appendingPathComponent("Package.swift").path) else { + return false + } + return (try? toolchain()) != nil + } + + func write(to output: URL) throws { + let artifact = try snapshot() + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + var data = try encoder.encode(artifact) + data.append(0x0a) + try FileManager.default.createDirectory( + at: output.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try data.write(to: output, options: .atomic) + } + + func snapshot() throws -> GraphArtifact { + let build = try run( + [ + "swift", "build", "--enable-index-store", "--configuration", "debug", + "--build-tests", + "-Xswiftc", "-index-include-locals", + ], + cwd: root + ) + guard build.status == 0 else { + throw ProducerError.message("swift build failed:\n\(build.output)") + } + let bin = try successful(["swift", "build", "--show-bin-path"], cwd: root) + .trimmingCharacters(in: .whitespacesAndNewlines) + let binURL = URL(fileURLWithPath: bin).standardizedFileURL + let store = binURL.appendingPathComponent("index/store", isDirectory: true) + guard FileManager.default.fileExists(atPath: store.path) else { + throw ProducerError.message("swift build produced no index store at \(store.path)") + } + let toolchain = try Self.toolchain() + let buildDescription = binURL.appendingPathComponent("description.json") + let buildDescriptionData = try Data(contentsOf: buildDescription) + let plan = try buildPlan(from: buildDescriptionData) + let sources = plan.sources + let outputUnits = plan.outputUnits + guard !sources.isEmpty else { throw ProducerError.message("the build plan has no project Swift sources") } + guard !outputUnits.isEmpty else { + throw ProducerError.message("swift build produced no object output units beneath \(binURL.path)") + } + let packageDump = try successful(["swift", "package", "dump-package"], cwd: root) + let buildBefore = try buildIdentity( + descriptionDigest: SHA256.hash(buildDescriptionData), + packageDump: packageDump, + targetTriple: toolchain.targetTriple + ) + let enrichments = try Dictionary(uniqueKeysWithValues: sources.map { source in + let relative = relativePath(source) + return (relative, try SourceEnrichment(url: source, relativePath: relative)) + }) + let diagnostics = diagnosticsBySource(build.output, sources: Set(enrichments.keys)) + + let queried = try query( + store: store, + library: toolchain.library, + outputUnits: outputUnits, + sources: sources, + enrichments: enrichments, + diagnostics: diagnostics, + compilerVersion: toolchain.compilerVersion, + targetTriple: toolchain.targetTriple + ) + try fence( + outputUnits: outputUnits, + enrichments: enrichments, + buildDescription: buildDescription, + buildIdentity: buildBefore, + targetTriple: toolchain.targetTriple + ) + + let encodedUnits = outputUnits.map { + OutputUnit(path: relativePath($0.url), digest: $0.digest) + }.sorted { $0.path < $1.path } + let coverage = Dictionary(uniqueKeysWithValues: factFamilies.map { + ($0, $0 == "renders" ? "unsupported" : "partial") + }) + let modules = Set(queried.map(\.moduleName)).sorted() + let targets = try modules.map { module -> TargetArtifact in + let name = "\(module)@\(toolchain.targetTriple)/debug" + let shards = queried.filter { $0.moduleName == module }.map { value in + SourceShard( + schemaVersion: 1, + language: "swift", + source: value.source, + checkerDigest: value.checkerDigest, + diskDigest: value.checkerDigest, + target: name, + compilerVersion: toolchain.compilerVersion, + moduleName: module, + targetTriple: toolchain.targetTriple, + sourceEnrichmentPasses: 1, + nodes: value.nodes, + edges: value.edges, + unresolved: value.unresolved, + diagnostics: value.diagnostics + ) + }.sorted { $0.source < $1.source } + let universeSeed = [ + module, toolchain.targetTriple, toolchain.sdk, "debug", + toolchain.compilerVersion, buildBefore.compilerFlagsDigest, + buildBefore.moduleDependenciesDigest, + buildBefore.packageResolutionDigest, buildBefore.pluginsDigest, + buildBefore.generatedSourcesDigest, + indexStoreDBCommit, + encodedUnits.map { "\($0.path)=\($0.digest)" }.joined(separator: "\n"), + ].joined(separator: "\0") + let universe = SHA256.hash(universeSeed) + let shardData = try JSONEncoder.sorted.encode(shards) + let generation = SHA256.hash(Data(universe.utf8) + shardData) + return TargetArtifact( + name: name, + generation: generation, + universe: universe, + moduleName: module, + targetTriple: toolchain.targetTriple, + sdk: toolchain.sdk, + configuration: "debug", + swiftLanguageVersion: toolchain.compilerVersion, + compilerFlagsDigest: buildBefore.compilerFlagsDigest, + moduleDependenciesDigest: buildBefore.moduleDependenciesDigest, + packageResolutionDigest: buildBefore.packageResolutionDigest, + pluginsDigest: buildBefore.pluginsDigest, + generatedSourcesDigest: buildBefore.generatedSourcesDigest, + indexStoreDBCommit: indexStoreDBCommit, + outputUnits: encodedUnits, + coverage: coverage, + shards: shards + ) + } + guard !targets.isEmpty else { + throw ProducerError.message("IndexStoreDB returned no Swift modules from the explicit output units") + } + return GraphArtifact( + schemaVersion: 1, + projectRoot: root.path, + producer: Producer( + name: "samchon-swift-graph", + version: "0.1.0", + protocolVersion: 1, + capabilities: Capabilities( + atomicGenerations: true, + incremental: true, + diagnostics: true, + explicitOutputUnits: true, + indexStoreDB: true, + sourceEnrichment: true, + swiftpm: true, + sourceKitResident: false + ) + ), + targets: targets + ) + } + + private func query( + store: URL, + library: URL, + outputUnits: [UnitFile], + sources: [URL], + enrichments: [String: SourceEnrichment], + diagnostics: [String: [DiagnosticFact]], + compilerVersion: String, + targetTriple: String + ) throws -> [QueriedShard] { + let database = root.appendingPathComponent( + ".build/samchon-graph/indexstoredb-\(UUID().uuidString)", + isDirectory: true + ) + try FileManager.default.createDirectory( + at: database.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + defer { try? FileManager.default.removeItem(at: database) } + return try { () throws -> [QueriedShard] in + let indexLibrary = try IndexStoreLibrary(dylibPath: library.path) + let databaseView = try IndexStoreDB( + storePath: store.path, + databasePath: database.path, + library: indexLibrary, + useExplicitOutputUnits: true, + waitUntilDoneInitializing: true, + readonly: false, + enableOutOfDateFileWatching: false, + listenToUnitEvents: false + ) + databaseView.addUnitOutFilePaths(outputUnits.map(\.url.path), waitForProcessing: true) + + var symbols = Set() + for source in sources { + for symbol in databaseView.symbols(inFilePath: source.path) { symbols.insert(symbol.usr) } + } + var occurrences: [SymbolOccurrence] = [] + var occurrenceKeys = Set() + for usr in symbols.sorted() { + for occurrence in databaseView.occurrences(ofUSR: usr, roles: .all) { + guard confined(occurrence.location.path), occurrence.location.path.hasSuffix(".swift") else { continue } + let key = [ + occurrence.symbol.usr, occurrence.location.path, + String(occurrence.location.line), String(occurrence.location.utf8Column), + String(occurrence.roles.rawValue), + ].joined(separator: "\0") + if occurrenceKeys.insert(key).inserted { occurrences.append(occurrence) } + } + } + occurrences.sort() + return buildShards( + occurrences: occurrences, + enrichments: enrichments, + diagnostics: diagnostics, + compilerVersion: compilerVersion, + targetTriple: targetTriple + ) + }() + } + + private func buildShards( + occurrences: [SymbolOccurrence], + enrichments: [String: SourceEnrichment], + diagnostics: [String: [DiagnosticFact]], + compilerVersion: String, + targetTriple: String + ) -> [QueriedShard] { + var sourceModules: [String: String] = [:] + for occurrence in occurrences { + let source = relativePath(URL(fileURLWithPath: occurrence.location.path)) + guard enrichments[source] != nil else { continue } + if !occurrence.location.moduleName.isEmpty { sourceModules[source] = occurrence.location.moduleName } + } + for source in enrichments.keys where sourceModules[source] == nil { + sourceModules[source] = inferredModule(source) + } + + let declarationOccurrences = occurrences.filter { + !$0.roles.intersection([.declaration, .definition]).isEmpty + } + var declarations: [String: Declaration] = [:] + for occurrence in declarationOccurrences { + let source = relativePath(URL(fileURLWithPath: occurrence.location.path)) + guard let enrichment = enrichments[source], declarations[occurrence.symbol.usr] == nil else { continue } + let module = sourceModules[source] ?? inferredModule(source) + let parent = occurrence.relations.first { + !$0.roles.intersection([.childOf, .containedBy, .accessorOf]).isEmpty + }?.symbol + let qualified = [module, parent?.name, occurrence.symbol.name] + .compactMap { $0 }.filter { !$0.isEmpty }.joined(separator: ".") + let node = GraphNode( + symbol: occurrence.symbol.usr, + kind: graphKind(occurrence.symbol.kind), + name: occurrence.symbol.name, + qualifiedName: qualified, + file: source, + exported: enrichment.isExported(line: occurrence.location.line), + modifiers: enrichment.modifiers( + line: occurrence.location.line, + properties: occurrence.symbol.properties.rawValue + ), + signature: enrichment.signature(line: occurrence.location.line), + origin: "IndexStoreDB+source-enrichment", + evidence: enrichment.evidence( + line: occurrence.location.line, + utf8Column: occurrence.location.utf8Column, + name: occurrence.symbol.name + ) + ) + declarations[occurrence.symbol.usr] = Declaration( + node: node, + module: module, + staticallyClosed: enrichment.isStaticallyClosed(line: occurrence.location.line), + unitTest: occurrence.symbol.properties.contains(.unitTest) + ) + } + + var nodes: [String: [GraphNode]] = [:] + var edges: [String: [GraphEdge]] = [:] + var unresolved: [String: [UnresolvedSite]] = [:] + for declaration in declarations.values { + nodes[declaration.node.file, default: []].append(declaration.node) + } + for occurrence in occurrences { + let source = relativePath(URL(fileURLWithPath: occurrence.location.path)) + guard let enrichment = enrichments[source] else { continue } + let evidence = enrichment.evidence( + line: occurrence.location.line, + utf8Column: occurrence.location.utf8Column, + name: occurrence.symbol.name + ) + let owner = occurrence.relations.first { + !$0.roles.intersection([.calledBy, .containedBy]).isEmpty + }?.symbol.usr ?? source + if !occurrence.roles.intersection([.declaration, .definition]).isEmpty, + let declaration = declarations[occurrence.symbol.usr] { + let parent = occurrence.relations.first { + !$0.roles.intersection([.childOf, .containedBy, .accessorOf]).isEmpty + }?.symbol.usr ?? source + append(edge( + from: parent, to: occurrence.symbol, kind: "contains", + evidence: evidence, provenance: "IndexStoreDB" + ), to: &edges[source, default: []]) + if declaration.node.exported { + append(edge( + from: source, to: occurrence.symbol, kind: "exports", + evidence: evidence, provenance: "source-enrichment" + ), to: &edges[source, default: []]) + } + for attribute in enrichment.attributes(at: occurrence.location.line) { + append(GraphEdge( + from: occurrence.symbol.usr, + to: "swift-attribute:\(attribute.name)", + kind: "decorates", + access: nil, + provenance: "source-enrichment", + targetKind: "type", + targetName: attribute.name, + targetQualifiedName: attribute.name, + evidence: attribute.evidence + ), to: &edges[source, default: []]) + } + } + if occurrence.roles.contains(.reference) { + append(edge( + from: owner, to: occurrence.symbol, kind: "references", + evidence: evidence, provenance: "IndexStoreDB" + ), to: &edges[source, default: []]) + } + if !occurrence.roles.intersection([.read, .write]).isEmpty { + let access = occurrence.roles.contains(.read) && occurrence.roles.contains(.write) + ? "read-write" : (occurrence.roles.contains(.write) ? "write" : "read") + append(edge( + from: owner, to: occurrence.symbol, kind: "accesses", + evidence: evidence, access: access, provenance: "IndexStoreDB" + ), to: &edges[source, default: []]) + } + if occurrence.roles.contains(.call) { + append(edge( + from: owner, to: occurrence.symbol, kind: "calls", + evidence: evidence, provenance: "IndexStoreDB" + ), to: &edges[source, default: []]) + if occurrence.symbol.kind == .constructor { + append(edge( + from: owner, to: occurrence.symbol, kind: "instantiates", + evidence: evidence, provenance: "IndexStoreDB" + ), to: &edges[source, default: []]) + } + if declarations[owner]?.unitTest == true { + append(edge( + from: owner, to: occurrence.symbol, kind: "tests", + evidence: evidence, provenance: "IndexStoreDB-call-graph" + ), to: &edges[source, default: []]) + } + if occurrence.roles.contains(.dynamic) { + unresolved[source, default: []].append(UnresolvedSite( + family: "dispatches", + reason: "dynamic", + evidence: evidence, + candidates: occurrence.relations.map(\.symbol.usr).sorted() + )) + } else if declarations[occurrence.symbol.usr]?.staticallyClosed == true { + append(edge( + from: owner, to: occurrence.symbol, kind: "dispatches", + evidence: evidence, provenance: "IndexStoreDB+source-enrichment" + ), to: &edges[source, default: []]) + } + } + if typeKind(occurrence.symbol.kind) && occurrence.roles.contains(.reference) { + append(edge( + from: owner, to: occurrence.symbol, kind: "type_ref", + evidence: evidence, provenance: "IndexStoreDB" + ), to: &edges[source, default: []]) + } + for relation in occurrence.relations { + if relation.roles.contains(.overrideOf) { + append(edge( + from: occurrence.symbol.usr, to: relation.symbol, kind: "overrides", + evidence: evidence, provenance: "IndexStoreDB" + ), to: &edges[source, default: []]) + } + if relation.roles.contains(.baseOf) { + let kind = occurrence.symbol.kind == .protocol ? "implements" : "extends" + append(edge( + from: relation.symbol.usr, to: occurrence.symbol, kind: kind, + evidence: evidence, provenance: "IndexStoreDB" + ), to: &edges[source, default: []]) + } + } + } + for (source, enrichment) in enrichments { + for imported in enrichment.imports() { + append(GraphEdge( + from: source, + to: "swift-module:\(imported.module)", + kind: "imports", + access: nil, + provenance: "source-enrichment", + targetKind: "module", + targetName: imported.module, + targetQualifiedName: imported.module, + evidence: imported.evidence + ), to: &edges[source, default: []]) + } + unresolved[source, default: []].append(contentsOf: enrichment.unresolvedSyntax()) + } + return enrichments.keys.sorted().map { source in + let module = sourceModules[source] ?? inferredModule(source) + return QueriedShard( + source: source, + moduleName: module, + checkerDigest: enrichments[source]!.digest, + nodes: (nodes[source] ?? []).sorted { $0.symbol < $1.symbol }, + edges: (edges[source] ?? []).sorted(by: edgeOrder), + unresolved: (unresolved[source] ?? []).sorted(by: unresolvedOrder), + diagnostics: diagnostics[source] ?? [] + ) + } + } + + private func fence( + outputUnits: [UnitFile], + enrichments: [String: SourceEnrichment], + buildDescription: URL, + buildIdentity: BuildIdentity, + targetTriple: String + ) throws { + for unit in outputUnits where try digest(unit.url) != unit.digest { + throw ProducerError.message("an explicit output unit moved while the generation was queried") + } + for (source, enrichment) in enrichments { + let current = try Data(contentsOf: root.appendingPathComponent(source)) + if SHA256.hash(current) != enrichment.digest { + throw ProducerError.message("\(source) moved while the generation was queried") + } + } + let packageDump = try successful(["swift", "package", "dump-package"], cwd: root) + if try self.buildIdentity( + descriptionDigest: SHA256.hash(try Data(contentsOf: buildDescription)), + packageDump: packageDump, + targetTriple: targetTriple + ) != buildIdentity { + throw ProducerError.message("SwiftPM build settings moved while the generation was queried") + } + } + + private func buildIdentity( + descriptionDigest: String, + packageDump: String, + targetTriple: String + ) throws -> BuildIdentity { + let packageResolutionDigest = try digestFile("Package.resolved") + return BuildIdentity( + compilerFlagsDigest: SHA256.hash( + targetTriple + "\0debug\0" + descriptionDigest + "\0" + + (try digestFile("Package.swift")) + "\0-index-include-locals" + ), + moduleDependenciesDigest: SHA256.hash(packageDump + "\0" + packageResolutionDigest), + packageResolutionDigest: packageResolutionDigest, + pluginsDigest: try digestTree(root.appendingPathComponent(".build/plugins")), + generatedSourcesDigest: try digestTree(root.appendingPathComponent(".build/plugins/outputs")) + ) + } + + private func buildPlan(from description: Data) throws -> BuildPlan { + guard let document = try JSONSerialization.jsonObject(with: description) as? [String: Any], + let commands = document["swiftCommands"] as? [String: Any], + !commands.isEmpty else { + throw ProducerError.message("SwiftPM build description has no Swift compiler commands") + } + var objectPaths = Set() + var sourcePaths = Set() + for (name, raw) in commands { + guard let command = raw as? [String: Any], + let objects = command["objects"] as? [String], + let sources = command["sources"] as? [String], + !objects.isEmpty, !sources.isEmpty else { + throw ProducerError.message("SwiftPM compiler command \(name) has no sources or object outputs") + } + objectPaths.formUnion(objects) + sourcePaths.formUnion(sources) + } + let units = try objectPaths.sorted().map { path in + let file = absoluteBuildPath(path) + guard file.pathExtension == "o", confined(file.path), + try file.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile == true else { + throw ProducerError.message("SwiftPM named an invalid output unit at \(file.path)") + } + return UnitFile(url: file, digest: try digest(file)) + } + let sources = try sourcePaths.sorted().compactMap { path -> URL? in + let file = absoluteBuildPath(path) + guard file.pathExtension == "swift", confined(file.path) else { return nil } + let relative = relativePath(file) + guard !relative.hasPrefix(".build/checkouts/"), + !relative.hasPrefix(".build/repositories/"), + !relative.hasPrefix(".build/artifacts/") else { return nil } + guard try file.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile == true else { + throw ProducerError.message("SwiftPM named a missing project source at \(file.path)") + } + return file + } + return BuildPlan(outputUnits: units, sources: sources) + } + + private func digestFile(_ relative: String) throws -> String { + let file = root.appendingPathComponent(relative) + return FileManager.default.fileExists(atPath: file.path) + ? try digest(file) + : SHA256.hash("absent:\(relative)") + } + + private func digestTree(_ directory: URL) throws -> String { + guard FileManager.default.fileExists(atPath: directory.path), + let enumerator = FileManager.default.enumerator( + at: directory, + includingPropertiesForKeys: [.isRegularFileKey], + options: [.skipsHiddenFiles] + ) else { return SHA256.hash("absent:\(directory.lastPathComponent)") } + var rows: [String] = [] + for case let file as URL in enumerator { + if (try? file.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile) == true { + rows.append("\(file.path.replacingOccurrences(of: directory.path, with: ""))=\(try digest(file))") + } + } + return SHA256.hash(rows.sorted().joined(separator: "\n")) + } + + private func relativePath(_ file: URL) -> String { + String(file.standardizedFileURL.path.dropFirst(root.standardizedFileURL.path.count)) + .trimmingCharacters(in: CharacterSet(charactersIn: "/")) + .replacingOccurrences(of: "\\", with: "/") + } + + private func absoluteBuildPath(_ path: String) -> URL { + path.hasPrefix("/") + ? URL(fileURLWithPath: path).standardizedFileURL + : root.appendingPathComponent(path).standardizedFileURL + } + + private func confined(_ file: String) -> Bool { + let base = root.standardizedFileURL.path + let candidate = URL(fileURLWithPath: file).standardizedFileURL.path + return candidate.hasPrefix(base + "/") + } + + private func inferredModule(_ source: String) -> String { + let parts = source.split(separator: "/").map(String.init) + if let marker = parts.firstIndex(where: { $0 == "Sources" || $0 == "Tests" }), + parts.indices.contains(marker + 1) { + return parts[marker + 1] + } + return root.lastPathComponent.replacingOccurrences(of: "-", with: "_") + } + + private static func toolchain() throws -> Toolchain { + let targetOutput = try successful(["swiftc", "-print-target-info"], cwd: nil) + guard let json = try JSONSerialization.jsonObject(with: Data(targetOutput.utf8)) as? [String: Any], + let target = json["target"] as? [String: Any], + let triple = target["triple"] as? String, + let compilerVersion = json["compilerVersion"] as? String, + let paths = json["paths"] as? [String: Any] else { + throw ProducerError.message("swiftc -print-target-info returned an unknown shape") + } + let runtimePaths = paths["runtimeLibraryPaths"] as? [String] ?? [] + let sdk = paths["sdkPath"] as? String ?? "" + let swiftc = try successful(["which", "swiftc"], cwd: nil) + .trimmingCharacters(in: .whitespacesAndNewlines) + let resolved = URL(fileURLWithPath: swiftc).resolvingSymlinksInPath() + #if os(macOS) + let libraryName = "libIndexStore.dylib" + #elseif os(Linux) + let libraryName = "libIndexStore.so" + #else + throw ProducerError.message("samchon-swift-graph supports macOS and Linux only") + #endif + var candidates = [ + resolved.deletingLastPathComponent().deletingLastPathComponent() + .appendingPathComponent("lib/\(libraryName)"), + ] + for path in runtimePaths { + let runtime = URL(fileURLWithPath: path) + candidates.append(runtime.appendingPathComponent(libraryName)) + candidates.append(runtime.deletingLastPathComponent().appendingPathComponent(libraryName)) + candidates.append(runtime.deletingLastPathComponent().deletingLastPathComponent().appendingPathComponent(libraryName)) + } + guard let library = candidates.first(where: { FileManager.default.fileExists(atPath: $0.path) }) else { + throw ProducerError.message("the Swift toolchain has no loadable \(libraryName)") + } + _ = try IndexStoreLibrary(dylibPath: library.path) + return Toolchain( + compilerVersion: compilerVersion, + targetTriple: triple, + sdk: sdk, + library: library + ) + } +} + +private struct Toolchain { + let compilerVersion: String + let targetTriple: String + let sdk: String + let library: URL +} + +private struct UnitFile { + let url: URL + let digest: String +} + +private struct BuildPlan { + let outputUnits: [UnitFile] + let sources: [URL] +} + +private struct BuildIdentity: Equatable { + let compilerFlagsDigest: String + let moduleDependenciesDigest: String + let packageResolutionDigest: String + let pluginsDigest: String + let generatedSourcesDigest: String +} + +private struct Declaration { + let node: GraphNode + let module: String + let staticallyClosed: Bool + let unitTest: Bool +} + +private struct QueriedShard { + let source: String + let moduleName: String + let checkerDigest: String + let nodes: [GraphNode] + let edges: [GraphEdge] + let unresolved: [UnresolvedSite] + let diagnostics: [DiagnosticFact] +} + +private struct ProcessResult { + let status: Int32 + let output: String +} + +private enum ProducerError: Error, CustomStringConvertible { + case message(String) + var description: String { + switch self { case .message(let text): return text } + } +} + +private func run(_ arguments: [String], cwd: URL?) throws -> ProcessResult { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = arguments + process.currentDirectoryURL = cwd + let output = Pipe() + process.standardOutput = output + process.standardError = output + try process.run() + let data = output.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + return ProcessResult(status: process.terminationStatus, output: String(decoding: data, as: UTF8.self)) +} + +private func successful(_ arguments: [String], cwd: URL?) throws -> String { + let result = try run(arguments, cwd: cwd) + guard result.status == 0 else { + throw ProducerError.message("\(arguments.joined(separator: " ")) failed:\n\(result.output)") + } + return result.output +} + +private func digest(_ file: URL) throws -> String { + SHA256.hash(try Data(contentsOf: file)) +} + +private func graphKind(_ kind: IndexSymbolKind) -> String { + switch kind { + case .module: return "module" + case .namespace, .namespaceAlias: return "namespace" + case .enum: return "enum" + case .class: return "class" + case .protocol: return "interface" + case .struct, .extension, .union, .typealias, .concept, .macro: return "type" + case .function, .conversionFunction, .destructor: return "function" + case .variable: return "variable" + case .field, .enumConstant: return "field" + case .instanceMethod, .classMethod, .staticMethod: return "method" + case .instanceProperty, .classProperty, .staticProperty: return "property" + case .constructor: return "constructor" + case .parameter: return "parameter" + case .using, .commentTag, .unknown: return "type" + } +} + +private func typeKind(_ kind: IndexSymbolKind) -> Bool { + switch kind { + case .module, .namespace, .namespaceAlias, .enum, .struct, .class, .protocol, + .extension, .union, .typealias, .concept: + return true + default: + return false + } +} + +private func edge( + from: String, + to symbol: Symbol, + kind: String, + evidence: Evidence, + access: String? = nil, + provenance: String +) -> GraphEdge { + GraphEdge( + from: from, + to: symbol.usr, + kind: kind, + access: access, + provenance: provenance, + targetKind: graphKind(symbol.kind), + targetName: symbol.name, + targetQualifiedName: symbol.name, + evidence: evidence + ) +} + +private func append(_ edge: GraphEdge, to edges: inout [GraphEdge]) { + if !edges.contains(where: { $0.kind == edge.kind && $0.from == edge.from && $0.to == edge.to }) { + edges.append(edge) + } +} + +private func edgeOrder(_ left: GraphEdge, _ right: GraphEdge) -> Bool { + [left.kind, left.from, left.to, String(left.evidence.startLine), String(left.evidence.startColumn)] + .joined(separator: "\0") < + [right.kind, right.from, right.to, String(right.evidence.startLine), String(right.evidence.startColumn)] + .joined(separator: "\0") +} + +private func unresolvedOrder(_ left: UnresolvedSite, _ right: UnresolvedSite) -> Bool { + [left.family, left.reason, String(left.evidence.startLine), String(left.evidence.startColumn)] + .joined(separator: "\0") < + [right.family, right.reason, String(right.evidence.startLine), String(right.evidence.startColumn)] + .joined(separator: "\0") +} + +private func diagnosticsBySource(_ output: String, sources: Set) -> [String: [DiagnosticFact]] { + let expression = try! NSRegularExpression( + pattern: #"^(.+\.swift):(\d+):(\d+):\s*(warning|note):\s*(.+)$"# + ) + var result: [String: [DiagnosticFact]] = [:] + for line in output.split(separator: "\n").map(String.init) { + let range = NSRange(line.startIndex.. URL { + if let index = arguments.firstIndex(of: "--cwd"), arguments.indices.contains(index + 1) { + return URL(fileURLWithPath: arguments[index + 1]).standardizedFileURL + } + return URL(fileURLWithPath: FileManager.default.currentDirectoryPath).standardizedFileURL +} + +private func value(_ name: String, in arguments: [String]) throws -> String { + guard let index = arguments.firstIndex(of: name), arguments.indices.contains(index + 1) else { + throw MainError.message("\(name) requires a value") + } + return arguments[index + 1] +} + +private func absolute(_ path: String, relativeTo root: URL) -> URL { + path.hasPrefix("/") + ? URL(fileURLWithPath: path).standardizedFileURL + : root.appendingPathComponent(path).standardizedFileURL +} diff --git a/tests/benchmark/graph/corpus.mjs b/tests/benchmark/graph/corpus.mjs index b60c8bb2..2cde262d 100644 --- a/tests/benchmark/graph/corpus.mjs +++ b/tests/benchmark/graph/corpus.mjs @@ -23,8 +23,9 @@ import path from "node:path"; // indexing when a server needs it. export const CORPUS = [ { - // A fork of excalidraw/excalidraw with `ttsc` (and its native TS7 - // runtime, @typescript/typescript-win32-x64) pinned as devDependencies + // A fork of excalidraw/excalidraw with protocol-v1 `ttsc@0.25.0` (and its + // native TS7 runtime, @typescript/typescript-win32-x64) pinned as + // devDependencies // (github.com/samchon/ttsc-benchmark-excalidraw) so ttscserver resolves // and runs from the repo's own node_modules instead of depending on a // global install — otherwise a fresh clone falls back to the static @@ -32,7 +33,7 @@ export const CORPUS = [ name: "excalidraw", language: "typescript", url: "https://github.com/samchon/ttsc-benchmark-excalidraw.git", - commit: "e576ee0ecbdec2b3de3b7378f2cc08038026531b", + commit: "b4a1909669c3fe5cbcd9837fbd30c4b35d27c10e", preflight: preflightMinimums(1_000, 5_000, 3_000, 4), }, { diff --git a/tests/benchmark/graph/index-time-cell.mjs b/tests/benchmark/graph/index-time-cell.mjs index 188b3c4d..bc2d9c18 100644 --- a/tests/benchmark/graph/index-time-cell.mjs +++ b/tests/benchmark/graph/index-time-cell.mjs @@ -1,3 +1,5 @@ +import { GRAPH_PROVIDERS } from "@samchon/graph"; + export const TOOL_SAMCHON = "samchon-graph"; export const TOOL_SAMCHON_FALLBACK = "samchon-graph-fallback"; export const TOOL_CODEGRAPH = "codegraph"; @@ -20,8 +22,140 @@ export function strictIntentOfTool(tool) { : undefined; } +/** The one canonical registry owner for a measured source language. */ +export function expectedPrimaryProvider( + language, + registry = GRAPH_PROVIDERS, +) { + const owners = registry.filter((provider) => + provider.languages.includes(language), + ); + if (owners.length !== 1) { + throw new Error( + `index-time: expected one primary provider for ${String(language)}, found ${String(owners.length)}`, + ); + } + return owners[0].name; +} + +/** Separate the route a graph cell asked for from the route that answered. */ +export function indexRoute(language, tool, summary) { + const strict = strictIntentOfTool(tool); + if (strict === undefined) return undefined; + const expected = expectedPrimaryProvider(language); + const indexer = + summary?.indexer === "lsp" || + summary?.indexer === "hybrid" || + summary?.indexer === "static" + ? summary.indexer + : null; + const provenance = Array.isArray(summary?.provenance) + ? summary.provenance + : []; + const primaryServed = provenance.some( + (row) => row?.provider === expected, + ); + const verdict = + summary?.truncated === true || indexer === null + ? "unknown" + : indexer === "static" + ? "static" + : strict && primaryServed + ? "served" + : "fallback"; + return { + schemaVersion: 1, + intent: { + strictProviders: strict ? "enabled" : "stood-down", + expectedPrimaryProvider: strict ? expected : null, + }, + outcome: { + verdict, + indexer, + provenance, + ...(summary?.truncated === true ? { truncated: true } : {}), + }, + }; +} + +/** Parse the bounded dump side channel without guessing missing provenance. */ +export function graphResultFromLog(text) { + const servedPrefix = "@samchon/graph: indexer="; + const routePrefix = "@samchon/graph: route="; + const attemptingPrefix = "@samchon/graph: indexing with "; + const lines = String(text).split(/\r?\n/u); + const outcome = lines.find((line) => line.startsWith(servedPrefix)); + const intent = lines.find((line) => line.startsWith(attemptingPrefix)); + const routeLine = lines.find((line) => line.startsWith(routePrefix)); + let summary; + if (routeLine !== undefined) { + try { + const parsed = JSON.parse(routeLine.slice(routePrefix.length)); + if ( + parsed?.schemaVersion === 1 && + Array.isArray(parsed.provenance) + ) { + summary = parsed; + } + } catch { + summary = undefined; + } + } + return { + servedBy: + outcome !== undefined + ? outcome.slice(servedPrefix.length).trim() + : intent === undefined + ? "unknown" + : `attempted ${intent.slice(attemptingPrefix.length).trim()}`, + summary, + }; +} + +/** Read the structured route, or conservatively classify a historical cell. */ +export function effectiveIndexRoute(cell, language) { + if (cell?.route?.schemaVersion === 1) return cell.route; + const strict = strictIntentOfTool(cell?.tool); + if (strict === undefined) return undefined; + const expected = expectedPrimaryProvider(language); + const servedBy = typeof cell?.servedBy === "string" ? cell.servedBy : ""; + const indexer = servedBy.startsWith("lsp ") + ? "lsp" + : servedBy.startsWith("hybrid ") + ? "hybrid" + : servedBy.startsWith("static ") + ? "static" + : null; + const providers = [ + ...servedBy.matchAll(/(?:^|\s)([A-Za-z0-9_.-]+)\([^)]*\)/gu), + ].map((match) => match[1]); + const primaryServed = providers.includes(expected); + const verdict = + indexer === null + ? "unknown" + : indexer === "static" + ? "static" + : strict && primaryServed + ? "served" + : "fallback"; + return { + schemaVersion: 1, + intent: { + strictProviders: strict ? "enabled" : "stood-down", + expectedPrimaryProvider: strict ? expected : null, + }, + outcome: { + verdict, + indexer, + provenance: [], + historical: true, + }, + }; +} + export function timedOutIndexCell({ project, + language, tool, timedOutMs, servedBy, @@ -29,10 +163,14 @@ export function timedOutIndexCell({ const strict = strictIntentOfTool(tool); return { project, + language, tool, buildMs: null, timedOutMs, ...(strict === undefined ? {} : { strict }), servedBy, + ...(strict === undefined + ? {} + : { route: indexRoute(language, tool, undefined) }), }; } diff --git a/tests/benchmark/graph/index-time-summary.mjs b/tests/benchmark/graph/index-time-summary.mjs index 955398ef..55fcc897 100644 --- a/tests/benchmark/graph/index-time-summary.mjs +++ b/tests/benchmark/graph/index-time-summary.mjs @@ -21,6 +21,7 @@ import { fileURLToPath } from "node:url"; import { PROJECTS } from "./corpus.mjs"; import currentIndex from "./current-index.cjs"; import { assertWebsitePublication } from "./publication-document.mjs"; +import { effectiveIndexRoute } from "./index-time-cell.mjs"; const { selectCurrentIndex } = currentIndex; const SELECTED_FIXTURES = Object.fromEntries( @@ -143,6 +144,7 @@ for (const row of rows) { continue; } for (const [tool, cell] of row.tools) { + const route = effectiveIndexRoute(cell, row.language); // A timed-out cell has no duration and is not a tool without a build step. // Both would print the same words if this only asked whether buildMs is a // number, and "this configuration does not finish inside the limit" is a @@ -163,19 +165,34 @@ for (const row of rows) { // A cell measured with the providers stood down says so, because // "no strict provider served" is also what a failed provider produces and // the two must not read alike. + const actual = route?.outcome.provenance + .map( + (entry) => + `${entry.provider}/${entry.producer.tool}@${entry.producer.version}`, + ) + .join(", "); const via = - cell.strict === false + route?.intent.strictProviders === "stood-down" ? " strict providers stood down" - : typeof cell.servedBy === "string" - ? ` via ${cell.servedBy}` - : ""; + : route === undefined + ? "" + : ` expected ${route.intent.expectedPrimaryProvider}; ${route.outcome.verdict}` + + (actual === "" ? "" : ` via ${actual}`); const caveat = - typeof cell.servedBy === "string" && cell.servedBy.startsWith("static") + route?.outcome.verdict === "static" ? " <- NOT A SEMANTIC INDEX" : ""; process.stdout.write( ` ${pad(row.project, 12)} ${pad(row.language, 11)} ${pad(row.commit, 13)} ${pad(tool, 17)} ${pad(time, 12)} ${scale}${where}${via}${caveat}\n`, ); + if ( + route?.intent.strictProviders === "enabled" && + route.outcome.verdict !== "served" + ) { + process.stdout.write( + `::warning title=index-time ${row.project} primary route::expected ${route.intent.expectedPrimaryProvider}; observed ${route.outcome.verdict}${actual === "" ? "" : ` via ${actual}`}\n`, + ); + } } } @@ -204,38 +221,27 @@ const paired = rows if (paired.length > 0) { process.stdout.write("\nstrict provider vs the same project with none:\n\n"); for (const { row, strict, fallback } of paired) { - const strictProviderServed = - typeof strict.servedBy === "string" && - /^(?:lsp|hybrid) /.test(strict.servedBy) && - !/no strict provider/.test(strict.servedBy); + const strictRoute = effectiveIndexRoute(strict, row.language); + const fallbackRoute = effectiveIndexRoute(fallback, row.language); + const strictProviderServed = strictRoute?.outcome.verdict === "served"; const strictSemantic = - strictProviderServed && strict.servedBy.startsWith("lsp "); - const strictFallbackSemantic = - !strictProviderServed && - typeof strict.servedBy === "string" && - strict.servedBy.startsWith("lsp "); + strictProviderServed && strictRoute.outcome.indexer === "lsp"; const fallbackSemantic = - typeof fallback.servedBy === "string" && - fallback.servedBy.startsWith("lsp "); + fallbackRoute?.outcome.indexer === "lsp"; const sameMeasurement = typeof strict.measurementId === "string" && strict.measurementId === fallback.measurementId && sameHost(strict.host, fallback.host); const verdict = - !strictProviderServed && - (!strictFallbackSemantic || !fallbackSemantic) - ? "strict provider did not serve and at least one cell produced no semantic index; times are not comparable" - : !strictProviderServed && !sameMeasurement - ? "strict provider did not serve and cells were not measured together on the same host; times are not comparable" - : !strictProviderServed - ? "no strict provider served, so both cells measured the same lane (LSP)" - : !strictSemantic - ? "strict provider served a hybrid index; times are not comparable" - : !fallbackSemantic - ? "strict-off cell produced no semantic index; times are not comparable" - : !sameMeasurement - ? "cells were not measured together on the same host; times are not comparable" - : `${(fallback.buildMs / strict.buildMs).toFixed(1)}x`; + !strictProviderServed + ? `expected ${strictRoute?.intent.expectedPrimaryProvider ?? "primary provider"} did not serve (${strictRoute?.outcome.verdict ?? "unknown"}); times are not comparable` + : !strictSemantic + ? "strict provider served a hybrid index; times are not comparable" + : !fallbackSemantic + ? "strict-off cell produced no semantic index; times are not comparable" + : !sameMeasurement + ? "cells were not measured together on the same host; times are not comparable" + : `${(fallback.buildMs / strict.buildMs).toFixed(1)}x`; const strictTime = `${(strict.buildMs / 1000).toFixed(1)} s`; const fallbackTime = `${(fallback.buildMs / 1000).toFixed(1)} s`; process.stdout.write( diff --git a/tests/benchmark/graph/index-time.mjs b/tests/benchmark/graph/index-time.mjs index 717481fc..5adc22fc 100644 --- a/tests/benchmark/graph/index-time.mjs +++ b/tests/benchmark/graph/index-time.mjs @@ -59,6 +59,8 @@ import { TOOL_SAMCHON, TOOL_SAMCHON_FALLBACK, TOOL_SERENA, + graphResultFromLog, + indexRoute, strictIntentOfTool, timedOutIndexCell, } from "./index-time-cell.mjs"; @@ -256,6 +258,7 @@ for (const project of selected) { if (typeof error?.timedOutMs !== "number") throw error; cell = timedOutIndexCell({ project, + language: spec.language, tool, timedOutMs: error.timedOutMs, // The process was killed before it could write its provenance line, @@ -263,7 +266,7 @@ for (const project of selected) { // say what was being attempted, because that is announced before // the first candidate runs, and "timed out running scip-ruby" is a // finding where "timed out" alone is a mystery. - servedBy: servedBy(error.logStem ?? ""), + servedBy: graphResult(error.logStem ?? "").servedBy, }); } assertPinnedCheckout(spec, cellRepoDir); @@ -413,12 +416,15 @@ function runIndexCell({ project, spec, repoDir, tool, env }) { // served" — and so does a strict cell whose provider failed. One was asked // for and the other is a defect, and a table that cannot tell them apart // reports every fallback measurement as a broken provider. + const observed = graphResult(logStem); return { project, + language: spec.language, tool, buildMs: ms, strict, - servedBy: servedBy(logStem), + servedBy: observed.servedBy, + route: indexRoute(spec.language, tool, observed.summary), }; } if (tool === TOOL_CODEGRAPH) { @@ -814,6 +820,19 @@ function assertIncomingReportScope(incoming) { `incoming index-time result cell ${cell.project}/${cell.tool} does not match its strict-provider intent`, ); } + if (cell.language !== PROJECTS[cell.project]?.language) { + throw new TypeError( + `incoming index-time result cell ${cell.project}/${cell.tool} does not match its project's language`, + ); + } + if ( + expectedStrict !== undefined && + cell.route?.schemaVersion !== 1 + ) { + throw new TypeError( + `incoming index-time result cell ${cell.project}/${cell.tool} omits its structured route evidence`, + ); + } } } @@ -1010,29 +1029,13 @@ function runChecked( * reported as unknown rather than guessed at, because "no strict provider * served" and "this dump predates the line" are different facts. */ -function servedBy(logStem) { - const served = "@samchon/graph: indexer="; - // What it set out to run, written before the first candidate starts. A build - // that finishes says what produced it; a build that is killed says nothing at - // all, and the killed ones are the expensive ones. Reading the intent turns a - // timed-out cell from "unknown" into "was running scip-ruby when the hour - // ran out", which is the difference between a mystery and a finding. - const attempting = "@samchon/graph: indexing with "; +function graphResult(logStem) { try { - const lines = fs - .readFileSync(`${logStem}.err.log`, "utf8") - .split(/\r?\n/); - const outcome = lines.find((line) => line.startsWith(served)); - if (outcome !== undefined) return outcome.slice(served.length).trim(); - const intent = lines.find((line) => line.startsWith(attempting)); - // Marked as an attempt rather than presented as a result: it says what was - // selected, not what published, and those differ exactly when a provider - // was chosen and then failed. - return intent === undefined - ? "unknown" - : `attempted ${intent.slice(attempting.length).trim()}`; + return graphResultFromLog( + fs.readFileSync(`${logStem}.err.log`, "utf8"), + ); } catch { - return "unknown"; + return graphResultFromLog(""); } } diff --git a/tests/benchmark/graph/publication-document.mjs b/tests/benchmark/graph/publication-document.mjs index 2de31045..bc4c0151 100644 --- a/tests/benchmark/graph/publication-document.mjs +++ b/tests/benchmark/graph/publication-document.mjs @@ -2,6 +2,10 @@ import { invalidWebsiteCellReason, websiteCellKey, } from "./website-cell.mjs"; +import { + expectedPrimaryProvider, + strictIntentOfTool, +} from "./index-time-cell.mjs"; /** * Start an agent-result merge without dropping a benchmark axis it does not own. @@ -210,6 +214,9 @@ function assertIndexCells(value, label, fixtures) { if (cell.toolchain !== undefined) { assertToolchainEvidence(cell.toolchain, `${cellLabel}.toolchain`); } + if (cell.route !== undefined) { + assertIndexRoute(cell, cellLabel); + } if (cell.quietWait !== undefined && cell.quietWait !== null) { assertRecord(cell.quietWait, `${cellLabel}.quietWait`); } @@ -221,6 +228,204 @@ function assertIndexCells(value, label, fixtures) { } } +function assertIndexRoute(cell, label) { + const strict = strictIntentOfTool(cell.tool); + if (strict === undefined) { + throw new TypeError(`${label}.route belongs only to a samchon-graph cell`); + } + if (cell.strict !== strict) { + throw new TypeError(`${label}.strict reverses its measured tool intent`); + } + if (typeof cell.language !== "string" || cell.language.trim() === "") { + throw new TypeError(`${label}.language must name the routed language`); + } + assertRecord(cell.route, `${label}.route`); + if (cell.route.schemaVersion !== 1) { + throw new TypeError(`${label}.route.schemaVersion must be 1`); + } + const intentLabel = `${label}.route.intent`; + const outcomeLabel = `${label}.route.outcome`; + assertRecord(cell.route.intent, intentLabel); + assertRecord(cell.route.outcome, outcomeLabel); + const expected = expectedPrimaryProvider(cell.language); + const intent = cell.route.intent; + if ( + (strict && intent.strictProviders !== "enabled") || + (!strict && intent.strictProviders !== "stood-down") + ) { + throw new TypeError(`${intentLabel}.strictProviders reverses its tool`); + } + if ( + (strict && intent.expectedPrimaryProvider !== expected) || + (!strict && intent.expectedPrimaryProvider !== null) + ) { + throw new TypeError( + `${intentLabel}.expectedPrimaryProvider disagrees with the canonical registry`, + ); + } + + const outcome = cell.route.outcome; + if (!["served", "fallback", "static", "unknown"].includes(outcome.verdict)) { + throw new TypeError(`${outcomeLabel}.verdict is invalid`); + } + if ( + outcome.indexer !== null && + !["lsp", "hybrid", "static"].includes(outcome.indexer) + ) { + throw new TypeError(`${outcomeLabel}.indexer is invalid`); + } + if (!Array.isArray(outcome.provenance)) { + throw new TypeError(`${outcomeLabel}.provenance must be an array`); + } + if (outcome.truncated !== undefined && outcome.truncated !== true) { + throw new TypeError(`${outcomeLabel}.truncated must be true when present`); + } + const providers = new Set(); + for (const [index, provenance] of outcome.provenance.entries()) { + const provenanceLabel = `${outcomeLabel}.provenance[${String(index)}]`; + assertRecord(provenance, provenanceLabel); + for (const field of ["provider", "authority"]) { + if ( + typeof provenance[field] !== "string" || + provenance[field].trim() === "" + ) { + throw new TypeError(`${provenanceLabel}.${field} must be nonempty`); + } + } + if (providers.has(provenance.provider)) { + throw new TypeError(`${provenanceLabel}.provider is duplicated`); + } + providers.add(provenance.provider); + if ( + !Array.isArray(provenance.languages) || + !provenance.languages.includes(cell.language) || + provenance.languages.some( + (language) => typeof language !== "string" || language.trim() === "", + ) || + new Set(provenance.languages).size !== provenance.languages.length + ) { + throw new TypeError( + `${provenanceLabel}.languages must include the measured language`, + ); + } + assertRecord(provenance.producer, `${provenanceLabel}.producer`); + for (const field of ["tool", "version"]) { + if ( + typeof provenance.producer[field] !== "string" || + provenance.producer[field].trim() === "" + ) { + throw new TypeError( + `${provenanceLabel}.producer.${field} must be nonempty`, + ); + } + } + for (const field of ["schemaVersion", "protocolVersion"]) { + if ( + !Number.isSafeInteger(provenance.producer[field]) || + provenance.producer[field] < 1 + ) { + throw new TypeError( + `${provenanceLabel}.producer.${field} must be a positive safe integer`, + ); + } + } + if ( + cell.toolchain?.status === "recorded" && + !producerDescribedByToolchain( + provenance.producer, + cell.toolchain.tools, + ) + ) { + throw new TypeError( + `${provenanceLabel}.producer is absent from the claimed toolchain`, + ); + } + } + const primaryServed = providers.has(expected); + if ( + outcome.truncated === true && + (outcome.verdict !== "unknown" || outcome.provenance.length !== 0) + ) { + throw new TypeError( + `${outcomeLabel}.truncated can describe only unknown empty provenance`, + ); + } + if ( + outcome.verdict === "served" && + (!strict || + !primaryServed || + (outcome.indexer !== "lsp" && outcome.indexer !== "hybrid")) + ) { + throw new TypeError(`${outcomeLabel} falsely claims the primary served`); + } + if ( + outcome.verdict === "fallback" && + ((outcome.indexer !== "lsp" && outcome.indexer !== "hybrid") || + (strict && primaryServed)) + ) { + throw new TypeError(`${outcomeLabel} is not a fallback result`); + } + if ( + outcome.verdict === "static" && + (outcome.indexer !== "static" || outcome.provenance.length !== 0) + ) { + throw new TypeError(`${outcomeLabel} is not a static result`); + } + if ( + outcome.verdict === "unknown" && + (outcome.provenance.length !== 0 || + (outcome.indexer !== null && outcome.truncated !== true)) + ) { + throw new TypeError(`${outcomeLabel} claims evidence despite being unknown`); + } +} + +/** Bind producer self-identification to the provisioned launcher/build pin. */ +export function producerDescribedByToolchain(producer, tools) { + const alias = PRODUCER_TOOLCHAIN_ALIASES[producer.tool]; + return tools.some((tool) => { + if (tool.version === "unpinned") { + return false; + } + if (tool.tool === producer.tool) { + return ( + tool.version === producer.version || + (isImmutableBuildPin(tool.version) && + containsExactPin(producer.version, tool.version)) + ); + } + return Boolean( + alias?.tool === tool.tool && + alias.producerVersions.includes(producer.version) && + (containsExactPin(tool.source, tool.version) || + containsExactPin(tool.digest, tool.version)), + ); + }); +} + +const PRODUCER_TOOLCHAIN_ALIASES = { + "scip-java-javac-graph": { + tool: "scip-java", + producerVersions: ["0.0.0-SNAPSHOT"], + }, +}; + +function isImmutableBuildPin(version) { + return /^[0-9a-f]{40,64}$/i.test(version); +} + +function containsExactPin(evidence, pin) { + const index = evidence.indexOf(pin); + if (index === -1) return false; + const before = evidence[index - 1]; + const after = evidence[index + pin.length]; + return !isPinCharacter(before) && !isPinCharacter(after); +} + +function isPinCharacter(character) { + return character !== undefined && /[A-Za-z0-9]/.test(character); +} + function assertToolchainEvidence(value, label) { assertRecord(value, label); if (value.status !== "recorded" && value.status !== "unreported") { diff --git a/tests/benchmark/graph/questions/manifest.json b/tests/benchmark/graph/questions/manifest.json index 6984fa04..30d43b12 100644 --- a/tests/benchmark/graph/questions/manifest.json +++ b/tests/benchmark/graph/questions/manifest.json @@ -6,7 +6,7 @@ "repo": "excalidraw", "family": "dedicated", "file": "excalidraw.md", - "fixtureCommit": "e576ee0ecbdec2b3de3b7378f2cc08038026531b", + "fixtureCommit": "b4a1909669c3fe5cbcd9837fbd30c4b35d27c10e", "language": "typescript", "questionSha256": "e4f3b02a5d3a1e86a9136d90a4b8bd9c73774733f9f1c021faee61b3ea6e83b5" }, @@ -15,7 +15,7 @@ "repo": "excalidraw", "family": "common", "file": "common.md", - "fixtureCommit": "e576ee0ecbdec2b3de3b7378f2cc08038026531b", + "fixtureCommit": "b4a1909669c3fe5cbcd9837fbd30c4b35d27c10e", "language": "typescript", "questionSha256": "efe22b2b8f0d37d2e74fc2391ba689b5c455e16a1f261fd821f209bd7ad20517" }, diff --git a/tests/benchmark/package.json b/tests/benchmark/package.json index da078d18..be6d7ff7 100644 --- a/tests/benchmark/package.json +++ b/tests/benchmark/package.json @@ -17,10 +17,10 @@ "index-time": "node graph/index-time.mjs", "publish": "node graph/publish.mjs", "audit": "node graph/audit-codex-traces.mjs", - "test": "node test/run.mjs", + "test": "pnpm --filter @samchon/graph build && node test/run.mjs", "render": "node build/graph-benchmark-svg.cjs", "render:png": "node build/graph-benchmark-svg.cjs --png", - "index-time:summary": "node graph/index-time-summary.mjs" + "index-time:summary": "pnpm --filter @samchon/graph build && node graph/index-time-summary.mjs" }, "dependencies": { "@samchon/graph": "workspace:*" diff --git a/tests/benchmark/test/clang-producer-cache.mjs b/tests/benchmark/test/clang-producer-cache.mjs new file mode 100644 index 00000000..401577be --- /dev/null +++ b/tests/benchmark/test/clang-producer-cache.mjs @@ -0,0 +1,197 @@ +import assert from "node:assert/strict"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + CLANG_PRODUCER_CACHE_INPUTS, + CLANG_PRODUCER_COMMIT, + CLANG_PRODUCER_REPOSITORY, + assertClangProducerAdapterPin, + clangProducerExecutable, + clangProducerProvisionDecision, +} from "../../experiment/src/clang-producer.mjs"; +import { findExperiment } from "../../experiment/src/catalog.mjs"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, "../../.."); + +/** + * The expensive native producer has one source owner and one build owner. + * + * This is structural on purpose: running LLVM to test workflow scheduling + * would take hours and still could not force both cache-hit and cold-miss + * branches deterministically. The assertions bind the executable recipe to + * the exact `hashFiles` inputs and the producer job to every consumer. + */ +export function assertClangProducerCacheOwnership() { + assert.equal(assertClangProducerAdapterPin(), CLANG_PRODUCER_COMMIT); + assert.throws( + () => + assertClangProducerAdapterPin( + 'export const CPP_CLANG_PRODUCER_COMMIT = "0000000000000000000000000000000000000000";', + ), + /adapter pins 0000000000000000000000000000000000000000/, + "a divergent adapter pin must refuse the producer generation", + ); + assert.equal( + clangProducerExecutable("samchon-clangd", "win32"), + "samchon-clangd.exe", + ); + assert.equal(clangProducerExecutable("clangd", "linux"), "clangd"); + assert.equal(clangProducerExecutable("clangd", "darwin"), "clangd"); + for (const language of ["c", "cpp"]) { + const experiment = findExperiment(language); + assert.equal(experiment.producerRepository, CLANG_PRODUCER_REPOSITORY); + assert.equal(experiment.producerCommit, CLANG_PRODUCER_COMMIT); + } + + assert.equal( + clangProducerProvisionDecision({ installed: true, allowBuild: false }), + "reuse", + "a verified exact cache hit must not build", + ); + assert.equal( + clangProducerProvisionDecision({ installed: false, allowBuild: true }), + "build", + "the dedicated owner must build a cold key", + ); + assert.throws( + () => + clangProducerProvisionDecision({ installed: false, allowBuild: false }), + /only the workflow producer job may build/, + "a consumer must refuse a cold or invalid cache", + ); + + const baseline = cacheDigest(); + assert.equal( + cacheDigest( + new Map([ + [ + "tests/experiment/src/catalog.mjs", + Buffer.from("an unrelated language catalog edit"), + ], + [ + "tests/experiment/src/setup-language.mjs", + Buffer.from("an unrelated installer edit"), + ], + ]), + ), + baseline, + "unrelated catalog and installer edits must leave the Clang key unchanged", + ); + for (const input of CLANG_PRODUCER_CACHE_INPUTS) { + assert.notEqual( + cacheDigest(new Map([[input, Buffer.from(`changed ${input}`)]])), + baseline, + `${input} must invalidate the Clang producer cache`, + ); + } + + const setup = read("tests/experiment/src/setup-language.mjs"); + assert.match(setup, /installClangGraphProducer\(\{/); + assert.doesNotMatch( + setup, + /samchon-clangd-source|LLVM_ENABLE_PROJECTS|--target[\s\S]*"clangd"/, + "the generic installer must not retain a second native producer recipe", + ); + + for (const workflowName of ["experiment.yml", "index-time.yml"]) { + assertWorkflowOwnsOneBuild(workflowName); + } +} + +function assertWorkflowOwnsOneBuild(workflowName) { + const workflow = read(`.github/workflows/${workflowName}`); + const owner = workflow.indexOf("\n clang_producer:"); + const consumerName = + workflowName === "experiment.yml" ? "experiment" : "measure"; + const consumer = workflow.indexOf(`\n ${consumerName}:`, owner); + assert.ok( + owner >= 0 && consumer > owner, + `${workflowName} has no producer predecessor`, + ); + + const ownerBlock = workflow.slice(owner, consumer); + const consumerBlock = workflow.slice(consumer); + assert.equal( + occurrences(workflow, "run: node tests/experiment/src/clang-producer.mjs"), + 1, + `${workflowName} must have exactly one executable build owner`, + ); + assert.match( + ownerBlock, + /Restore the pinned Clang producer[\s\S]*Provision the pinned Clang producer[\s\S]*Save the pinned Clang producer[\s\S]*Pack the verified Clang producer[\s\S]*Upload the verified Clang producer/, + ); + assert.match( + consumerBlock, + /needs: \[latest_update, clang_producer\]/, + `${workflowName} consumers must wait for the producer owner`, + ); + assert.equal( + occurrences(workflow, "Save the pinned Clang producer"), + 1, + `${workflowName} must save only in the predecessor job`, + ); + assert.match( + ownerBlock, + /Save the pinned Clang producer[\s\S]*continue-on-error: true/, + `${workflowName} cache saving must not block the guaranteed artifact handoff`, + ); + assert.match( + ownerBlock, + /SAMCHON_GRAPH_CLANG_PRODUCER_ALLOW_BUILD: \$\{\{ steps\.clang_producer\.outputs\.cache-hit != 'true' && '1' \|\| '0' \}\}/, + `${workflowName} must refuse an invalid immutable exact hit`, + ); + assert.match( + consumerBlock, + /Download the verified Clang producer[\s\S]*uses: actions\/download-artifact@v8[\s\S]*name: pinned-clang-producer[\s\S]*Unpack the verified Clang producer[\s\S]*tar -C tests\/experiment\/\.work\/tools -xf tests\/experiment\/\.work\/clang-producer-artifact\/pinned-clang-producer\.tar/, + `${workflowName} consumers must receive the verified same-run artifact`, + ); + assert.match( + ownerBlock, + /tar -C tests\/experiment\/\.work\/tools -cf pinned-clang-producer\.tar \.[\s\S]*uses: actions\/upload-artifact@v7[\s\S]*path: pinned-clang-producer\.tar/, + `${workflowName} must preserve executable bits inside the artifact`, + ); + assert.match( + consumerBlock, + /SAMCHON_GRAPH_CLANG_PRODUCER_ALLOW_BUILD: "0"/, + `${workflowName} consumers must be unable to rebuild`, + ); + + const expectedInputs = [...CLANG_PRODUCER_CACHE_INPUTS].sort(); + const hashCalls = [...workflow.matchAll(/hashFiles\(([^)]*)\)/gu)]; + assert.equal(hashCalls.length, 1, `${workflowName} must have one cache owner`); + for (const call of hashCalls) { + const actual = [...call[1].matchAll(/'([^']+)'/gu)] + .map((match) => match[1]) + .sort(); + assert.deepEqual( + actual, + expectedInputs, + `${workflowName} uses a broad or incomplete Clang key`, + ); + } +} + +function cacheDigest(overrides = new Map()) { + const hash = crypto.createHash("sha256"); + for (const relative of CLANG_PRODUCER_CACHE_INPUTS) { + hash.update(relative); + hash.update("\0"); + hash.update( + overrides.get(relative) ?? fs.readFileSync(path.join(repoRoot, relative)), + ); + hash.update("\0"); + } + return hash.digest("hex"); +} + +function read(relative) { + return fs.readFileSync(path.join(repoRoot, relative), "utf8"); +} + +function occurrences(text, fragment) { + return text.split(fragment).length - 1; +} diff --git a/tests/benchmark/test/run.mjs b/tests/benchmark/test/run.mjs index b5d72f31..8c512bbf 100644 --- a/tests/benchmark/test/run.mjs +++ b/tests/benchmark/test/run.mjs @@ -11,6 +11,10 @@ import { latestWorkflowUpdateDecision, } from "../../../.github/scripts/latest-workflow-update.mjs"; import { CORPUS, PROJECTS, projectDir } from "../graph/corpus.mjs"; +import { + summarizeCoverage, + summarizeUnresolved, +} from "../../experiment/src/evidence-summary.mjs"; import currentIndex from "../graph/current-index.cjs"; import { analyzePreflightDump, @@ -19,6 +23,9 @@ import { } from "../graph/language.mjs"; import { ALL_TOOLS, + expectedPrimaryProvider, + graphResultFromLog, + indexRoute, timedOutIndexCell, } from "../graph/index-time-cell.mjs"; import { javaSystemProperty } from "../graph/java-tool-options.mjs"; @@ -27,7 +34,11 @@ import { summarizeLspRequestTrace, } from "../graph/lsp-request-summary.mjs"; import { assertPublicationCandidates } from "../graph/publication-gate.mjs"; -import { agentPublicationDocument } from "../graph/publication-document.mjs"; +import { + agentPublicationDocument, + assertIndexReport, + producerDescribedByToolchain, +} from "../graph/publication-document.mjs"; import { removeTree } from "../graph/remove-tree.mjs"; import { invalidWebsiteCellReason, @@ -35,6 +46,7 @@ import { } from "../graph/website-cell.mjs"; import ordinal from "../graph/ordinal.cjs"; import { assertDeclarationsPrecedeExecution } from "./declaration-order.mjs"; +import { assertClangProducerCacheOwnership } from "./clang-producer-cache.mjs"; import { assertWorkflowOptionForms } from "./option-form.mjs"; import { assertBothIndexColumnsAreMeasured, @@ -42,7 +54,7 @@ import { } from "./two-columns.mjs"; const { compareNaturalOrdinal } = ordinal; -const { selectCurrentAgentCells } = currentIndex; +const { selectCurrentAgentCells, selectCurrentIndex } = currentIndex; const here = path.dirname(fileURLToPath(import.meta.url)); const benchmarkDir = path.resolve(here, ".."); @@ -90,6 +102,7 @@ testPublicationRequiresMatchingCodexTraceAudit(); testFixtureAndPreflightIntegrity(); testReferenceRenderer(); testLatestWorkflowUpdateClassifier(); +assertClangProducerCacheOwnership(); assertBothIndexColumnsAreMeasured(); assertStrictComparisonArithmetic(); assertDeclarationsPrecedeExecution(graphDir, ["index-time.mjs"]); @@ -100,6 +113,8 @@ assertWorkflowOptionForms( testPublishedIndexCellsNameTheirMachine(); testAgentPublicationPreservesIndexResults(); testTimedOutIndexCellsPreserveToolIntent(); +testIndexRouteEvidenceContract(); +testExperimentEvidenceSummary(); testIndexPublicationRefusesMalformedJson(); testIndexCellIsolationContract(); testLspRequestDiagnosisSummary(); @@ -153,15 +168,25 @@ function testPublishedIndexCellsNameTheirMachine() { "a published index cell carries a build time, a timeout, or says it has no build step", ); if (published.index.schemaVersion === 2) { - const stale = (published.index.cells ?? []).filter( + const selectedFixtures = Object.fromEntries( + Object.entries(PROJECTS).map(([project, spec]) => [ + project, + spec.commit, + ]), + ); + const current = selectCurrentIndex( + published.index, + selectedFixtures, + ).index; + const stale = (current.cells ?? []).filter( (cell) => cell?.fixtureCommit !== PROJECTS[cell?.project]?.commit || - published.index.fixtures?.[cell?.project] !== cell?.fixtureCommit, + current.fixtures?.[cell?.project] !== cell?.fixtureCommit, ); assert.deepEqual( stale.map((cell) => `${String(cell.project)}/${String(cell.tool)}`), [], - "a revision-bound index cell must name the exact currently selected fixture commit", + "a selected revision-bound cell must name the exact current fixture commit", ); } // A fallback cell reports "no strict provider served" and so does a strict @@ -196,6 +221,7 @@ function testAgentPublicationPreservesIndexResults() { cells: [ { project: INDEX_PROJECT, + language: PROJECTS[INDEX_PROJECT].language, tool: "samchon-graph", buildMs: 1, fixtureCommit: FIXTURE_COMMIT, @@ -292,6 +318,7 @@ function testAgentPublicationPreservesIndexResults() { function testTimedOutIndexCellsPreserveToolIntent() { const common = { project: INDEX_PROJECT, + language: PROJECTS[INDEX_PROJECT].language, timedOutMs: 3_600_000, servedBy: "attempted fixture indexer", }; @@ -314,6 +341,9 @@ function testTimedOutIndexCellsPreserveToolIntent() { tool, buildMs: null, ...(strict === undefined ? {} : { strict }), + ...(strict === undefined + ? {} + : { route: indexRoute(common.language, tool, undefined) }), }); assert.equal( Object.hasOwn(cell, "strict"), @@ -323,6 +353,342 @@ function testTimedOutIndexCellsPreserveToolIntent() { } } +function testIndexRouteEvidenceContract() { + assert.equal(expectedPrimaryProvider("typescript"), "ttscgraph"); + assert.equal(expectedPrimaryProvider("java"), "javac-graph"); + assert.equal(expectedPrimaryProvider("c"), "clangd-snapshot"); + assert.throws( + () => expectedPrimaryProvider("unknown"), + /expected one primary provider/, + ); + for (const [producer, toolchain] of [ + [ + { + tool: "samchon-rust-analyzer", + version: "0.0.0 (378f220482c298775910f0fc46e8fda1bc516ecc)", + }, + { + tool: "samchon-rust-analyzer", + version: "378f220482c298775910f0fc46e8fda1bc516ecc", + source: "fixture", + digest: "git:378f220482c298775910f0fc46e8fda1bc516ecc", + }, + ], + [ + { + tool: "scip-java-javac-graph", + version: "0.0.0-SNAPSHOT", + }, + { + tool: "scip-java", + version: "fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a", + source: + "https://github.com/samchon/scip-java@fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a", + digest: "sha256:fixture", + }, + ], + [ + { + tool: "samchon-clangd", + version: + "clang version 22.1.8 (https://github.com/samchon/llvm-project.git d6371c37445998d24776692a27e086bb24f9916a) ((https://github.com/samchon/llvm-project.git d6371c37445998d24776692a27e086bb24f9916a))", + }, + { + tool: "samchon-clangd", + version: "d6371c37445998d24776692a27e086bb24f9916a", + source: "fixture", + digest: "git:d6371c37445998d24776692a27e086bb24f9916a", + }, + ], + ]) { + assert.equal( + producerDescribedByToolchain(producer, [toolchain]), + true, + `${producer.tool} self-identification must bind to its provisioned build pin`, + ); + } + assert.equal( + producerDescribedByToolchain( + { tool: "jdtls", version: "1.0.0" }, + [ + { + tool: "jdtls", + version: "unpinned", + source: "latest", + digest: "unpinned", + }, + ], + ), + false, + ); + for (const [producer, toolchain] of [ + [ + { + tool: "samchon-clangd", + version: "clang version 22.1.8 (wrong-build)", + }, + { + tool: "samchon-clangd", + version: "22.1.8", + source: "fixture", + digest: "git:22.1.8", + }, + ], + [ + { + tool: "scip-java-javac-graph", + version: "totally-unrelated-build", + }, + { + tool: "scip-java", + version: "fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a", + source: + "https://github.com/samchon/scip-java@fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a", + digest: "sha256:fixture", + }, + ], + [ + { + tool: "scip-java-javac-graph", + version: "0.0.0-SNAPSHOT", + }, + { + tool: "scip-java", + version: "fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a", + source: + "https://github.com/samchon/scip-java@0fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a0", + digest: "sha256:fixture", + }, + ], + ]) { + assert.equal( + producerDescribedByToolchain(producer, [toolchain]), + false, + `${producer.tool} must not bind to an unrelated or partial build pin`, + ); + } + + const primary = routeSummary( + "java", + "javac-graph", + "scip-java-javac-graph", + "fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a", + "compiler", + ); + const fallback = routeSummary( + "java", + "scip-java", + "scip-java", + "0.10.6", + "semantic-index", + ); + const served = indexRoute("java", "samchon-graph", primary); + const fellBack = indexRoute("java", "samchon-graph", fallback); + const staticResult = indexRoute("java", "samchon-graph", { + schemaVersion: 1, + indexer: "static", + provenance: [], + }); + const strictOff = indexRoute("java", "samchon-graph-fallback", fallback); + const missing = indexRoute("java", "samchon-graph", undefined); + assert.deepEqual( + [ + served.outcome.verdict, + fellBack.outcome.verdict, + staticResult.outcome.verdict, + strictOff.intent.strictProviders, + strictOff.intent.expectedPrimaryProvider, + missing.outcome.verdict, + ], + ["served", "fallback", "static", "stood-down", null, "unknown"], + ); + + const parsed = graphResultFromLog( + [ + "@samchon/graph: indexing with javac-graph(java)", + "@samchon/graph: indexer=lsp scip-java(java)", + `@samchon/graph: route=${JSON.stringify(fallback)}`, + ].join("\n"), + ); + assert.equal(parsed.servedBy, "lsp scip-java(java)"); + assert.deepEqual(parsed.summary, fallback); + assert.deepEqual(graphResultFromLog("malformed"), { + servedBy: "unknown", + summary: undefined, + }); + + const host = FIXTURE_HOST; + const valid = { + schemaVersion: 2, + host, + fixtures: { gson: PROJECTS.gson.commit }, + scale: { gson: { files: 1, lines: 1 } }, + cells: [ + { + project: "gson", + language: "java", + tool: "samchon-graph", + strict: true, + buildMs: 1, + fixtureCommit: PROJECTS.gson.commit, + host, + toolchain: { + status: "recorded", + tools: [ + { + tool: "scip-java-javac-graph", + version: "fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a", + source: "fixture", + digest: "git:fixture", + }, + { + tool: "scip-java", + version: "0.10.6", + source: "fixture fallback", + digest: "sha256:fixture", + }, + ], + }, + route: served, + }, + ], + }; + assert.doesNotThrow(() => assertIndexReport(valid, "route fixture")); + assert.doesNotThrow(() => + assertIndexReport( + { + ...valid, + cells: [{ ...valid.cells[0], route: missing }], + }, + "missing route provenance fixture", + ), + ); + assert.doesNotThrow(() => + assertIndexReport( + { + ...valid, + cells: [{ ...valid.cells[0], route: undefined }], + }, + "historical route fixture", + ), + ); + for (const [label, mutate, pattern] of [ + [ + "reversed intent", + (candidate) => { + candidate.cells[0].route.intent.strictProviders = "stood-down"; + }, + /reverses its tool/, + ], + [ + "mismatched expected owner", + (candidate) => { + candidate.cells[0].route.intent.expectedPrimaryProvider = "scip-java"; + }, + /canonical registry/, + ], + [ + "false served verdict", + (candidate) => { + candidate.cells[0].route = fellBack; + candidate.cells[0].route.outcome.verdict = "served"; + }, + /falsely claims/, + ], + [ + "mismatched toolchain", + (candidate) => { + candidate.cells[0].toolchain.tools[0].version = "other"; + }, + /absent from the claimed toolchain/, + ], + [ + "impossible truncated result", + (candidate) => { + candidate.cells[0].route.outcome.truncated = true; + }, + /truncated can describe only unknown empty provenance/, + ], + ]) { + const candidate = structuredClone(valid); + mutate(candidate); + assert.throws( + () => assertIndexReport(candidate, label), + pattern, + `${label} must be rejected`, + ); + } +} + +function routeSummary(language, provider, tool, version, authority) { + return { + schemaVersion: 1, + indexer: "lsp", + provenance: [ + { + provider, + languages: [language], + authority, + producer: { + tool, + version, + schemaVersion: 6, + protocolVersion: 1, + }, + }, + ], + }; +} + +function testExperimentEvidenceSummary() { + const dump = { + coverage: [ + { + provider: "primary", + family: "calls", + state: "complete", + }, + { + provider: "primary", + family: "calls", + state: "partial", + }, + { + provider: "other", + family: "calls", + state: "unsupported", + }, + ], + unresolved: [ + { provider: "primary", family: "calls", reason: "dynamic" }, + { provider: "primary", family: "type_ref", reason: "provider-gap" }, + { provider: "other", family: "calls", reason: "reflection" }, + ], + }; + const coverage = summarizeCoverage(dump, "primary"); + const unresolved = summarizeUnresolved(dump, "primary"); + assert.equal(coverage.families.length, 15); + assert.deepEqual( + coverage.families.find((row) => row.family === "calls"), + { family: "calls", complete: 1, partial: 1, unsupported: 0 }, + ); + assert.deepEqual( + coverage.families.find((row) => row.family === "imports"), + { family: "imports", complete: 0, partial: 0, unsupported: 0 }, + ); + assert.equal(unresolved.total, 2); + assert.equal(unresolved.byFamily.length, 15); + assert.equal(unresolved.byReason.length, 9); + assert.equal( + unresolved.byReason.find((row) => row.reason === "dynamic").count, + 1, + ); + assert.equal( + unresolved.byReason.find((row) => row.reason === "reflection").count, + 0, + ); +} + /** * Publication is a preserving merge, never recovery by replacement. * @@ -354,6 +720,12 @@ function testIndexPublicationRefusesMalformedJson() { source: "fixture", digest: "sha256:fixture", }, + { + tool: "ttscgraph", + version: "fixture-route", + source: "fixture route", + digest: "sha256:fixture-route", + }, ], }, projects: [INDEX_PROJECT], @@ -363,6 +735,7 @@ function testIndexPublicationRefusesMalformedJson() { cells: [ { project: INDEX_PROJECT, + language: PROJECTS[INDEX_PROJECT].language, tool: "samchon-graph", buildMs: 1, strict: true, @@ -377,8 +750,25 @@ function testIndexPublicationRefusesMalformedJson() { source: "fixture", digest: "sha256:fixture", }, + { + tool: "ttscgraph", + version: "fixture-route", + source: "fixture route", + digest: "sha256:fixture-route", + }, ], }, + route: indexRoute( + PROJECTS[INDEX_PROJECT].language, + "samchon-graph", + routeSummary( + PROJECTS[INDEX_PROJECT].language, + "ttscgraph", + "ttscgraph", + "fixture-route", + "compiler", + ), + ), host: FIXTURE_HOST, }, ], @@ -689,6 +1079,17 @@ function testIndexPublicationRefusesMalformedJson() { ], }, ], + [ + "cell language mismatch", + { + ...validReportDocument, + cells: validReportDocument.cells.map((cell) => ({ + ...cell, + language: "c", + route: indexRoute("c", "samchon-graph", undefined), + })), + }, + ], [ "empty measurement identity", { ...validReportDocument, measurementId: "" }, @@ -1011,6 +1412,14 @@ function testIndexPublicationRefusesMalformedJson() { ...cell, tool, strict, + route: + strict === undefined + ? undefined + : indexRoute(cell.language, tool, { + schemaVersion: 1, + indexer: "lsp", + provenance: [], + }), measurementId, })), }; @@ -1081,12 +1490,14 @@ function testIndexPublicationRefusesMalformedJson() { cells: [ { project: INDEX_PROJECT, + language: PROJECTS[INDEX_PROJECT].language, tool: "samchon-graph", buildMs: 4, strict: true, measurementId: validReportDocument.measurementId, fixtureCommit: FIXTURE_COMMIT, toolchain: validReportDocument.toolchain, + route: validReportDocument.cells[0].route, host: FIXTURE_HOST, }, ], @@ -1279,17 +1690,16 @@ function testIndexCellIsolationContract() { workflow.includes("--project=${{ matrix.project }}") && workflow.includes('SAMCHON_GRAPH_BENCH_TIMEOUT_MS: "1800000"') && !workflow.includes('SAMCHON_GRAPH_BENCH_TIMEOUT_MS: "3600000"') && - // Two budgets, and which one a row gets is decided by whether it - // provisions a compiler built from source. The literal 120 that used to - // stand here was the whole cap, and it killed the two rows that spend an - // hour and three quarters compiling clangd before they had measured - // anything. Both numbers are asserted, and so is the condition that - // separates them, because a cap that applied to every row again would - // still contain the string "120". - workflow.includes( - "timeout-minutes: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && 210 || 120 }}", + // The compiler build now has its own 150-minute predecessor budget. The + // measurement itself returns to one 120-minute budget, because no row + // can spend it compiling LLVM. Assert each owner rather than merely + // finding both literals somewhere in the file. + /\n clang_producer:[\s\S]*?timeout-minutes: 150[\s\S]*?\n measure:/u.test( + workflow, + ) && + /\n measure:[\s\S]*?timeout-minutes: 120[\s\S]*?\n strategy:/u.test( + workflow, ) && - !workflow.includes("timeout-minutes: 150") && workflow.includes("--timeout-ms=300000") && workflow.includes("timeout-minutes: 10"), "measurement and slow-lane diagnosis must each run inside their evidence-backed bounded budgets", @@ -1323,6 +1733,9 @@ function testIndexCellIsolationContract() { const installRenderer = collect.indexOf( "- name: Install renderer dependencies", ); + const buildRegistry = collect.indexOf( + "- name: Build provider registry dependency", + ); const foldPublication = collect.indexOf( "- name: Fold reports into the publication", ); @@ -1335,11 +1748,13 @@ function testIndexCellIsolationContract() { collectStart >= 0 && collectSetup >= 0 && installRenderer > collectSetup && - foldPublication > installRenderer && + buildRegistry > installRenderer && + foldPublication > buildRegistry && showPublication > foldPublication && renderPublication > showPublication && uploadPublication > renderPublication && collect.includes("run: pnpm install --frozen-lockfile") && + collect.includes("run: pnpm build") && collect.includes( "run: pnpm --filter @samchon/graph-benchmark render:png", ) && diff --git a/tests/benchmark/test/two-columns.mjs b/tests/benchmark/test/two-columns.mjs index c9446551..59a94112 100644 --- a/tests/benchmark/test/two-columns.mjs +++ b/tests/benchmark/test/two-columns.mjs @@ -6,6 +6,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { PROJECTS } from "../graph/corpus.mjs"; +import { indexRoute } from "../graph/index-time-cell.mjs"; const here = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(here, "../../.."); @@ -125,6 +126,7 @@ export function assertStrictComparisonArithmetic() { }; const cell = (project, tool, extra) => ({ project, + language: PROJECTS[project].language, tool, host, measurementId: "fixture-measurement", @@ -138,7 +140,8 @@ export function assertStrictComparisonArithmetic() { const staticProject = "gson"; const absentStaticProject = "lualine"; const differentRunProject = "tokio"; - const differentHostProject = "redis"; + const differentHostProject = "leveldb"; + const redisFallbackProject = "redis"; const unknownProvenanceProject = "serilog"; const hybridProject = "slim"; const projects = [ @@ -149,6 +152,7 @@ export function assertStrictComparisonArithmetic() { absentStaticProject, differentRunProject, differentHostProject, + redisFallbackProject, unknownProvenanceProject, hybridProject, ]; @@ -167,7 +171,7 @@ export function assertStrictComparisonArithmetic() { // Served, both finished: a ratio. cell(servedProject, "samchon-graph", { buildMs: 10_000, - servedBy: "lsp scip-fake(go)", + servedBy: "lsp ttscgraph(typescript)", }), cell(servedProject, "samchon-graph-fallback", { buildMs: 250_000, @@ -187,7 +191,7 @@ export function assertStrictComparisonArithmetic() { // dividing by it would understate the very gap it is meant to show. cell(timeoutProject, "samchon-graph", { buildMs: 5_000, - servedBy: "lsp scip-fake(c)", + servedBy: "lsp samchon-graph-go(go)", }), cell(timeoutProject, "samchon-graph-fallback", { buildMs: null, @@ -199,11 +203,21 @@ export function assertStrictComparisonArithmetic() { // savings because the cells performed materially different work. cell(staticProject, "samchon-graph", { buildMs: 30_000, - servedBy: "lsp scip-fake(java)", + servedBy: "lsp scip-java(java)", + route: indexRoute( + "java", + "samchon-graph", + routeSummary("java", "scip-java", "scip-java", "0.10.6"), + ), }), cell(staticProject, "samchon-graph-fallback", { buildMs: 3_000, servedBy: "static no strict provider served", + route: indexRoute("java", "samchon-graph-fallback", { + schemaVersion: 1, + indexer: "static", + provenance: [], + }), }), // A provider failure does not make a static strict-off cell the same // lane as the strict cell's completed LSP fallback. @@ -220,7 +234,7 @@ export function assertStrictComparisonArithmetic() { // match. cell(differentRunProject, "samchon-graph", { buildMs: 10_000, - servedBy: "lsp scip-fake(rust)", + servedBy: "lsp samchon-rust-analyzer-hir(rust)", }), cell(differentRunProject, "samchon-graph-fallback", { buildMs: 50_000, @@ -229,13 +243,34 @@ export function assertStrictComparisonArithmetic() { }), cell(differentHostProject, "samchon-graph", { buildMs: 20_000, - servedBy: "lsp scip-fake(c)", + servedBy: "lsp clangd-snapshot(cpp)", }), cell(differentHostProject, "samchon-graph-fallback", { buildMs: 60_000, servedBy: "lsp no strict provider served", host: { ...host, cpu: "another fixture" }, }), + // Redis completed through scip-clang while clangd-snapshot was the + // registry owner. Both clocks remain evidence, but never form a + // primary-provider ratio. + cell(redisFallbackProject, "samchon-graph", { + buildMs: 20_000, + servedBy: "lsp scip-clang(c)", + route: indexRoute( + "c", + "samchon-graph", + routeSummary("c", "scip-clang", "scip-clang", "0.3.3"), + ), + }), + cell(redisFallbackProject, "samchon-graph-fallback", { + buildMs: 60_000, + servedBy: "lsp no strict provider served", + route: indexRoute("c", "samchon-graph-fallback", { + schemaVersion: 1, + indexer: "lsp", + provenance: [], + }), + }), // A completed process without the provenance line proves neither that a // strict provider served nor that its output is semantic. Missing // evidence must not become a savings claim merely because both clocks @@ -253,7 +288,7 @@ export function assertStrictComparisonArithmetic() { // wholly semantic index whose duration can headline a speedup. cell(hybridProject, "samchon-graph", { buildMs: 12_000, - servedBy: "hybrid scip-fake(php)", + servedBy: "hybrid scip-php(php)", }), cell(hybridProject, "samchon-graph-fallback", { buildMs: 48_000, @@ -293,14 +328,14 @@ export function assertStrictComparisonArithmetic() { assert.match( out, new RegExp( - `${absentProject}[^\\n]*both cells measured the same lane`, + `${absentProject}[^\\n]*expected scip-python did not serve`, ), "a project whose provider never served must not report a ratio", ); assert.match( out, new RegExp( - `${absentStaticProject}[^\\n]*at least one cell produced no semantic index`, + `${absentStaticProject}[^\\n]*expected samchon-graph-lua did not serve`, ), "a provider failure must not make a static strict-off result the same lane", ); @@ -317,9 +352,9 @@ export function assertStrictComparisonArithmetic() { assert.match( out, new RegExp( - `${staticProject}[^\\n]*no semantic index; times are not comparable`, + `${staticProject}[^\\n]*expected javac-graph did not serve`, ), - "a static strict-off cell must be reported as non-comparable", + "Gson's scip-java fallback must be reported as a primary-route failure", ); assert.doesNotMatch( out, @@ -353,7 +388,7 @@ export function assertStrictComparisonArithmetic() { assert.match( out, new RegExp( - `${unknownProvenanceProject}[^\\n]*at least one cell produced no semantic index`, + `${unknownProvenanceProject}[^\\n]*expected roslyn-workspace did not serve`, ), "a completed cell without strict-provider provenance must be non-comparable", ); @@ -372,6 +407,30 @@ export function assertStrictComparisonArithmetic() { new RegExp(`${hybridProject}[^\\n]*x$`, "m"), "a hybrid strict cell must not report a semantic savings ratio", ); + assert.match( + out, + new RegExp( + `${redisFallbackProject}[^\\n]*expected clangd-snapshot did not serve`, + ), + "Redis's scip-clang result must be a primary-route failure", + ); + assert.doesNotMatch( + out, + new RegExp(`${redisFallbackProject}[^\\n]*x$`, "m"), + "Redis's fallback result must never receive a strict-provider ratio", + ); + for (const [project, provider] of [ + ["gson", "javac-graph"], + ["redis", "clangd-snapshot"], + ]) { + assert.match( + out, + new RegExp( + `::warning title=index-time ${project} primary route::expected ${provider}; observed fallback`, + ), + `${project}'s publishable fallback must emit its own workflow warning`, + ); + } const jsonRan = cp.spawnSync( process.execPath, @@ -404,3 +463,23 @@ export function assertStrictComparisonArithmetic() { "the JSON summary names how many comparable measurement groups it contains", ); } + +function routeSummary(language, provider, tool, version) { + return { + schemaVersion: 1, + indexer: "lsp", + provenance: [ + { + provider, + languages: [language], + authority: "semantic-index", + producer: { + tool, + version, + schemaVersion: 6, + protocolVersion: 1, + }, + }, + ], + }; +} diff --git a/tests/experiment/package.json b/tests/experiment/package.json index c54b083a..4362de9e 100644 --- a/tests/experiment/package.json +++ b/tests/experiment/package.json @@ -7,9 +7,11 @@ "scripts": { "list": "node src/list-languages.mjs", "setup": "node src/setup-language.mjs", - "start": "node src/run-language.mjs" + "start": "node src/run-language.mjs", + "topology": "pnpm --filter @samchon/graph build && node src/run-topology-orientation.mjs" }, "dependencies": { + "@modelcontextprotocol/sdk": "^1.26.0", "@samchon/graph": "workspace:*" }, "devDependencies": { diff --git a/tests/experiment/src/catalog.mjs b/tests/experiment/src/catalog.mjs index c8d5b9c4..9a506fcb 100644 --- a/tests/experiment/src/catalog.mjs +++ b/tests/experiment/src/catalog.mjs @@ -1,10 +1,12 @@ -// `minEdges` gates the relationship edges an experiment must produce. It is set -// to 1 for languages whose reference edges are empirically confirmed against a -// real server (see the LSP experiment CI matrix) so a regression back to the -// "symbols but no edges" failure is caught. Languages still awaiting a first -// real-server measurement keep 0; the runner always records the observed count, -// so the CI artifact reports the true number and the gate can be tightened once -// a language is confirmed. +import { + CLANG_PRODUCER_COMMIT, + CLANG_PRODUCER_REPOSITORY, +} from "./clang-producer.mjs"; + +// `minNodes` and `minEdges` gate the graph a pinned experiment must produce. +// Rows use measured lower bounds only where the fixture has established them; +// the result artifact always records the observed counts so a gate can be +// tightened without guessing. export const LANGUAGE_EXPERIMENTS = [ { language: "typescript", @@ -81,7 +83,8 @@ export const LANGUAGE_EXPERIMENTS = [ strictAuthority: "analyzer", strictTool: "samchon-rust-analyzer", producerRepository: "https://github.com/samchon/rust-analyzer.git", - producerCommit: "2850ecba80311bebd4cdaa9fedc5321533b5b1e7", + producerCommit: "378f220482c298775910f0fc46e8fda1bc516ecc", + nativeBaseline: "samchon-rust-analyzer prime-caches .", requiredCapabilities: [ "coverage", "diagnostics", @@ -109,7 +112,7 @@ export const LANGUAGE_EXPERIMENTS = [ ], crossFileEdge: "references", lifecycle: { - sourceFile: "src/lib.rs", + sourceFile: "src/server.rs", editSuffix: "\n// samchon-graph lifecycle edit\n", createFile: "examples/samchon_graph_experiment.rs", renamedFile: "examples/samchon_graph_experiment_renamed.rs", @@ -128,24 +131,37 @@ export const LANGUAGE_EXPERIMENTS = [ failureFile: "Cargo.toml", failureSuffix: "\n[malformed", failurePolicy: "reject", + performance: { + noopSamples: 20, + editSamples: 20, + noopP95MaxMs: 250, + editP95MaxMs: 2_000, + editFind: "broadcast::channel(1)", + editReplacements: ["broadcast::channel(2)", "broadcast::channel(3)"], + }, }, }, { language: "cpp", - repository: "https://github.com/fmtlib/fmt.git", - commit: "bcaa44d05579c75a83571821faee7acf6a9a0d55", + repository: "https://github.com/samchon/graph-benchmark-leveldb.git", + commit: "7ee830d02b623e8ffe0b95d59a74db1e58da04c5", // Uncapped: the native snapshot publishes a whole-compilation-database // generation and refuses a file cap. // // The compilation database enumerates every native clangd graph view and // is what a CMake project has to be configured to produce; preparation // itself compiles nothing. - prepare: "cmake -S . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + prepare: + "cmake -S . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DLEVELDB_BUILD_TESTS=OFF -DLEVELDB_BUILD_BENCHMARKS=OFF", strictProvider: "clangd-snapshot", strictAuthority: "compiler", strictTool: "samchon-clangd", - producerRepository: "https://github.com/samchon/llvm-project.git", - producerCommit: "e33d8f51552a523b5696691738f1ef95f8e3a730", + producerRepository: CLANG_PRODUCER_REPOSITORY, + producerCommit: CLANG_PRODUCER_COMMIT, + nativeBaseline: { + kind: "clang-background-index", + command: "samchon-clangd", + }, // A whole-compilation-database producer is not ready when it starts; it // is ready when clangd has background-indexed every translation unit the // database registers. The 180-second default expired on libuv with 62 of @@ -200,16 +216,34 @@ export const LANGUAGE_EXPERIMENTS = [ "references", ], crossFileEdge: "references", + representativeEdges: [ + { + kind: "calls", + from: "leveldb::DBImpl::Get", + to: "leveldb::MemTable::Get", + }, + { + kind: "accesses", + from: "leveldb::DBImpl::Get", + to: "leveldb::DBImpl::mutex_", + }, + { + kind: "type_ref", + from: "leveldb::DBImpl::Get", + to: "leveldb::Slice", + }, + { kind: "extends", from: "leveldb::DBImpl", to: "leveldb::DB" }, + ], semanticLimitation: "The native Clang lane retains exact TU/configuration facts, while calls, instantiation, exports, implements and dispatch stay explicitly partial and C/C++ have no decorates, renders or tests family.", // Background jobs may finish in any order, but the native shard set, // manifest and generation digest are canonical and publish only after all // registered configurations agree on one complete source state. lifecycle: { - sourceFile: "src/format.cc", + sourceFile: "db/db_impl.cc", editSuffix: "\n// samchon-graph lifecycle edit\n", - createFile: "samchon_graph_experiment.cc", - renamedFile: "samchon_graph_experiment_renamed.cc", + createFile: "db/samchon_graph_experiment.cc", + renamedFile: "db/samchon_graph_experiment_renamed.cc", createText: "int samchonGraphExperiment(void) { return 0; }\n", createdSymbol: "samchonGraphExperiment", @@ -223,26 +257,34 @@ export const LANGUAGE_EXPERIMENTS = [ // A malformed compilation database invalidates the native universe, so // the strict resident rejects publication until it is repaired. failurePolicy: "reject", + noopPerformance: { + samples: 20, + p95MaxMs: 250, + }, }, minNodes: 1, minEdges: 1, }, { language: "c", - repository: "https://github.com/libuv/libuv.git", - commit: "9d51562c10be60bc1126a3d71803b1038f4fbb7e", + repository: "https://github.com/samchon/graph-benchmark-redis.git", + commit: "6bf6224c3dad518329ddc893ef9c5d58dcbabdeb", // Uncapped: the native snapshot publishes a whole-compilation-database // generation and refuses a file cap. // // The compilation database enumerates every native clangd graph view and // is what a CMake project has to be configured to produce; preparation // itself compiles nothing. - prepare: "cmake -S . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + prepare: "bear -- make -j2", strictProvider: "clangd-snapshot", strictAuthority: "compiler", strictTool: "samchon-clangd", - producerRepository: "https://github.com/samchon/llvm-project.git", - producerCommit: "e33d8f51552a523b5696691738f1ef95f8e3a730", + producerRepository: CLANG_PRODUCER_REPOSITORY, + producerCommit: CLANG_PRODUCER_COMMIT, + nativeBaseline: { + kind: "clang-background-index", + command: "samchon-clangd", + }, // A whole-compilation-database producer is not ready when it starts; it // is ready when clangd has background-indexed every translation unit the // database registers. The 180-second default expired on libuv with 62 of @@ -297,26 +339,36 @@ export const LANGUAGE_EXPERIMENTS = [ "references", ], crossFileEdge: "references", + representativeEdges: [ + { kind: "calls", from: "processCommand", to: "lookupCommand" }, + { kind: "calls", from: "processCommand", to: "call" }, + { kind: "accesses", from: "processCommand", to: "server" }, + { kind: "type_ref", from: "processCommand", to: "client" }, + ], semanticLimitation: "The native Clang lane retains exact TU/configuration facts, while calls, instantiation, exports, implements and dispatch stay explicitly partial and C/C++ have no decorates, renders or tests family.", // C and C++ share the same atomic, canonical generation boundary. lifecycle: { - sourceFile: "src/uv-common.c", + sourceFile: "src/server.c", editSuffix: "\n// samchon-graph lifecycle edit\n", - createFile: "samchon_graph_experiment.c", - renamedFile: "samchon_graph_experiment_renamed.c", + createFile: "src/samchon_graph_experiment.c", + renamedFile: "src/samchon_graph_experiment_renamed.c", createText: "int samchonGraphExperiment(void) { return 0; }\n", createdSymbol: "samchonGraphExperiment", // The database itself, because that is what this producer reads. Breaking // CMakeLists would leave an already-generated database untouched and test // nothing. - buildFile: "build/compile_commands.json", - compilationDatabase: "build/compile_commands.json", - failureFile: "build/compile_commands.json", + buildFile: "compile_commands.json", + compilationDatabase: "compile_commands.json", + failureFile: "compile_commands.json", failureSuffix: "\n[ not json", // The C and C++ slices share the same strict rejection boundary. failurePolicy: "reject", + noopPerformance: { + samples: 20, + p95MaxMs: 250, + }, }, minNodes: 1, minEdges: 1, @@ -327,14 +379,24 @@ export const LANGUAGE_EXPERIMENTS = [ // Use scip-java's own pinned Maven fixture for that contract; Gson remains // the separate large-corpus timing proof. repository: "https://github.com/samchon/scip-java.git", - commit: "32eca214a413d1b8a375c481f666ff8a4ec96773", + commit: "fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a", projectRoot: "scip-java/src/test/resources/fixtures/maven/basic", // The producer and the corpus are one checkout on purpose. The fixture is // the producer's own Maven project, so a pin that named a different // revision for each would measure a plugin against a build it was never // tested with. producerRepository: "https://github.com/samchon/scip-java.git", - producerCommit: "32eca214a413d1b8a375c481f666ff8a4ec96773", + producerCommit: "fefb1bfb2e3fac90cd90f64fc07cc57fb533b49a", + producerTree: "8cb3dd9b84fbbbb8dba22827b9d8e7dd21c3f46e", + jdtProducerRepository: "https://github.com/samchon/eclipse.jdt.ls.git", + jdtProducerCommit: "0d55a6c13d14e0d0466eeb021920349b3d0c6d35", + jdtProducerTree: "18937e87ae9b42098100b398fde8cfb87f4c9b7c", + nativeBaseline: { + kind: "shell", + command: "mvn -q test-compile", + warmup: true, + clean: ["target"], + }, strictProvider: "javac-graph", strictAuthority: "compiler", // The launcher and the producer are two names. `scip-java` is the command @@ -357,15 +419,6 @@ export const LANGUAGE_EXPERIMENTS = [ // counts that transition rather than pre-editing the pinned baseline. semanticEdges: ["contains", "instantiates"], crossFileEdge: "instantiates", - // The producer cannot yet reproduce its own generation, and the exemption - // names the exact reason rather than accepting an unexplained difference. - // Restoring the original sources returns identical facts — five nodes and - // eight edges both times — and a different build universe, because the - // universe is digested from the raw javac invocation and that invocation - // names the temporary directory the launcher unpacked its compiler plugin - // into, which is new on every run. Filed at samchon/scip-java#1. - regenerationLimitation: - "scip-java 32eca214a413d1b8a375c481f666ff8a4ec96773 digests its build universe from the raw javac invocation, which names the per-run temporary directory its embedded plugin jar is unpacked into, so an unchanged checkout reproduces identical facts under a different universe", lifecycle: { sourceFile: "src/main/java/com/Example.java", editSuffix: "\n// samchon-graph lifecycle edit\n", @@ -391,120 +444,263 @@ export const LANGUAGE_EXPERIMENTS = [ failureFile: "pom.xml", failureSuffix: "\n \"strict-lifecycle\";\n}\n", + "namespace Serilog;\n\ninternal static class SamchonGraphExperiment\n{\n internal static ILogger Run() => Log.Logger;\n}\n", createdSymbol: "SamchonGraphExperiment", + createdEdge: { + kind: "accesses", + from: "Run", + to: "Logger", + crossFile: true, + }, buildFile: "src/Serilog/Serilog.csproj", failureFile: "src/Serilog/Serilog.csproj", failureSuffix: "\n", - // scip-dotnet 0.2.14 writes the index before it reads and logs - // MSBuildWorkspace failures, then returns zero. The malformed project is - // therefore not a fail-closed boundary for this producer. - failurePolicy: "published", - failureLimitation: - "scip-dotnet 0.2.14 logs MSBuildWorkspace failures only after writing the index and exits successfully, so a malformed C# project file publishes a degraded generation instead of rejecting it", + failurePolicy: "reject", + performance: { + noopSamples: 5, + editSamples: 3, + noopP95MaxMs: 250, + editP95MaxMs: 2000, + editFind: "if (Value == null) return 0;", + editReplacements: [ + "if (Value == null) return 1;", + "if (Value == null) return 2;", + ], + }, }, - minNodes: 1, - minEdges: 0, - // The upstream solution's perf/AOT entries make csharp-ls return no symbols. - // Keep the product and main test projects so experiments retain both the - // runtime graph and its test anchors without loading those broken entries. + // Select and restore exactly the product and test projects before the + // resident service starts; refreshes themselves never invoke restore. + // NuGet's live advisory feed is not part of this commit-pinned compiler + // fixture, and Serilog promotes its changing audit warnings to errors. prepare: - "dotnet new sln -n Serilog --format sln --force && dotnet sln Serilog.sln add src/Serilog/Serilog.csproj test/Serilog.Tests/Serilog.Tests.csproj", + "dotnet new sln -n Serilog --format sln --force && dotnet sln Serilog.sln add src/Serilog/Serilog.csproj test/Serilog.Tests/Serilog.Tests.csproj && dotnet restore Serilog.sln -p:NuGetAudit=false", }, { language: "kotlin", - // The producer's exact Kotlin 2.3.20 fixture keeps ten clean lifecycle - // builds bounded and exercises the same compiler minor as Koin. Language - // setup builds the producer from this same commit and supplies one verified - // Gradle distribution because the fixture intentionally carries no wrapper. - repository: "https://github.com/scip-code/scip-java.git", - commit: "e940c1889767a81347387067a375320dc6f5d83e", - projectRoot: - "scip-java/src/test/resources/fixtures/gradle/kotlin2", - strictProvider: "scip-kotlinc", - strictAuthority: "semantic-index", - strictTool: "scip-java", - requiredCapabilities: ["universe", "diskDigests"], - semanticEdges: ["references"], - crossFileEdge: "references", - // This source snapshot and its setup manifest pin the plugin build to Kotlin - // 2.3.20. scip-java still does not publish the compiler revision selected by - // the indexed build itself, so the empty runtime field remains explicit. - compilerLimitation: - "scip-java e940c1889767a81347387067a375320dc6f5d83e is built with Kotlin 2.3.20 but does not expose the compiler revision selected by the indexed Gradle build, so runtime compiler provenance cannot name it without guessing", + // This immutable fork keeps Koin's JVM performance module independent of + // the repository-wide multiplatform build while retaining its 400-module, + // roughly 1,600-class graph. The producer is pinned independently because + // the experiment must prove the exact compiler plugin that wrote its facts. + repository: "https://github.com/samchon/graph-benchmark-koin.git", + commit: "cca45c63d1088888f445304e13f9fbc310f62078", + projectRoot: "examples/jvm-perfs", + producerRepository: "https://github.com/samchon/scip-java.git", + producerCommit: "3a1565d0647d89a28880fa40ecbef0966a1a328c", + producerTree: "3b5c24126b0670c9c9bd9369df71fcd112b34b67", + // The isolated copy has no build outputs. Compile the same Kotlin/JVM + // target once without the injected graph plugin so the cold strict row + // reports exporter overhead against Kotlin's own ordinary build rather + // than against the historical scip-java or language-server lanes. + nativeBaseline: "gradle compileKotlin", + strictProvider: "kotlinc-graph", + strictAuthority: "compiler", + strictTool: "scip-kotlinc-k2-graph", + strictMinimums: true, + requiredCapabilities: [ + "coverage", + "diagnostics", + "diskDigests", + "incremental", + "sourceDigests", + "universe", + "unresolved", + ], + semanticEdges: ["calls"], + crossFileEdge: "calls", lifecycle: { - sourceFile: "src/main/kotlin/foo/Example.kt", + sourceFile: + "src/main/kotlin/org/koin/benchmark/GraphLifecycle.kt", editSuffix: "\n// samchon-graph lifecycle edit\n", - createFile: "src/main/kotlin/foo/SamchonGraphExperiment.kt", + createFile: + "src/main/kotlin/org/koin/benchmark/SamchonGraphExperiment.kt", renamedFile: - "src/main/kotlin/foo/SamchonGraphExperimentRenamed.kt", + "src/main/kotlin/org/koin/benchmark/SamchonGraphExperimentRenamed.kt", createText: - "package foo\n\nfun samchonGraphExperiment(): Example = Example\n", + "package org.koin.benchmark\n\ninternal fun samchonGraphExperiment() = perfModule400()\n", createdSymbol: "samchonGraphExperiment", createdEdge: { - kind: "references", + kind: "calls", from: "samchonGraphExperiment", - to: "Example", + to: "perfModule400", crossFile: true, }, - buildFile: "build.gradle", - failureFile: "build.gradle", - failureSuffix: "\n}\n", + buildFile: "build.gradle.kts", + buildEditSuffix: + '\n\ntasks.withType().configureEach {\n compilerOptions {\n moduleName.set("samchonGraphExperiment")\n }\n}\n', + failureFile: "build.gradle.kts", + failureSuffix: "\nnotAValidGradleBlock {\n", failurePolicy: "reject", + kotlinBuildReportRoot: + "build/scip-targetroot/META-INF/kotlin-build-reports", + performance: { + noopSamples: 5, + editSamples: 3, + noopP95MaxMs: 250, + editP95MaxMs: 2000, + editFind: "graphLifecycleMarker(): Int = 1", + editReplacements: [ + "graphLifecycleMarker(): Int = 2", + "graphLifecycleMarker(): Int = 3", + ], + }, }, - minNodes: 1, - minEdges: 0, + minNodes: 1_000, + minEdges: 1_000, }, { language: "swift", repository: "https://github.com/apple/swift-argument-parser.git", commit: "2f77f2fccb6e84fecff338c37b199e33e7dfd119", + nativeBaseline: + "swift build --enable-index-store --build-tests -Xswiftc -index-include-locals", + strictProvider: "swift-indexstore", + strictAuthority: "compiler", + strictTool: "samchon-swift-graph", + strictMinimums: true, + requiredCapabilities: [ + "coverage", + "diagnostics", + "diskDigests", + "incremental", + "sourceDigests", + "universe", + "unresolved", + "explicitOutputUnits", + "indexStoreDB", + "sourceEnrichment", + "swiftpm", + ], + semanticEdges: ["calls", "references"], + crossFileEdge: "calls", + lifecycle: { + sourceFile: "Sources/ArgumentParser/Utilities/CollectionExtensions.swift", + editSuffix: "\n// samchon-graph lifecycle edit\n", + createFile: "Sources/ArgumentParser/Utilities/SamchonGraphExperiment.swift", + renamedFile: + "Sources/ArgumentParser/Utilities/SamchonGraphExperimentRenamed.swift", + createText: + "func samchonGraphExperiment(_ values: [Int]) -> [Int] {\n values.mapEmpty { [1] }\n}\n", + createdSymbol: "samchonGraphExperiment", + createdEdge: { + kind: "calls", + from: "samchonGraphExperiment", + to: "mapEmpty", + crossFile: true, + }, + buildFile: "Package.swift", + buildEditSuffix: "\n// samchon-graph build-universe edit\n", + failureFile: "Package.swift", + failureSuffix: "\nthis is not valid Swift package syntax\n", + failurePolicy: "reject", + performance: { + noopSamples: 5, + editSamples: 3, + noopP95MaxMs: 250, + editP95MaxMs: 20_000, + editFind: "isEmpty ? replacement() : self", + editReplacements: [ + "isEmpty ? replacement() : self /* graph edit 1 */", + "isEmpty ? replacement() : self /* graph edit 2 */", + ], + }, + }, maxFiles: 120, - minNodes: 1, - minEdges: 1, - feasibilityBlocked: - "swift build emits an index store during an ordinary debug build and its records carry RelChild, but the on-disk format is toolchain-internal, versioned v5, with no third-party stability claim and no binary specification; reading it requires a compiled Swift program linking IndexStoreDB, which does not exist yet", + minNodes: 100, + minEdges: 100, }, { language: "scala", - repository: "https://github.com/scala/scala3-example-project.git", - commit: "a327177a2bc8ef9c499726d038e56694d6f7cddb", - maxFiles: 120, - minNodes: 1, - minEdges: 1, - feasibilityBlocked: - "scip-java is a Java and Kotlin indexer by its own README, supported-language table, and source tree, so registering it for Scala made an installed scip-java displace the real Scala language server with a producer that cannot index the language; a SemanticDB or TASTy channel through BSP is unwritten work", + repository: "https://github.com/samchon/graph-benchmark-scala.git", + commit: "b11f22758c902bffa29513c9fcda07863a2ad996", + prepare: "sbt bspConfig", + nativeBaseline: + "env -u SAMCHON_GRAPH_SCALA2_PLUGIN -u SAMCHON_GRAPH_SCALA3_PLUGIN -u SAMCHON_GRAPH_SCALA_PLUGIN_VERSION sbt compile", + strictProvider: "scalac-graph", + strictAuthority: "compiler", + strictTool: "samchon-scala-graph", + strictMinimums: true, + requiredCapabilities: [ + "coverage", + "diagnostics", + "diskDigests", + "incremental", + "sourceDigests", + "universe", + "unresolved", + "bsp", + "semanticdb", + "typedPlugins", + "zinc", + ], + semanticEdges: ["calls"], + crossFileEdge: "calls", + lifecycle: { + sourceFile: "scala3/src/main/scala/demo/Api.scala", + editSuffix: "\n// samchon-graph lifecycle edit\n", + createFile: "scala3/src/main/scala/demo/SamchonGraphExperiment.scala", + renamedFile: + "scala3/src/main/scala/demo/SamchonGraphExperimentRenamed.scala", + createText: + 'package demo\n\nobject SamchonGraphExperiment:\n def samchonGraphExperiment(): String = Helper.render("graph")\n', + createdSymbol: "samchonGraphExperiment", + createdEdge: { + kind: "calls", + from: "samchonGraphExperiment", + to: "render", + crossFile: true, + }, + buildFile: "build.sbt", + buildEditSuffix: '\nThisBuild / scalacOptions += "-deprecation"\n', + failureFile: "build.sbt", + failureSuffix: "\nthis is not valid sbt syntax {\n", + failurePolicy: "reject", + performance: { + noopSamples: 5, + editSamples: 3, + noopP95MaxMs: 500, + editP95MaxMs: 15_000, + editFind: "graphLifecycleMarker(): Int = 1", + editReplacements: [ + "graphLifecycleMarker(): Int = 2", + "graphLifecycleMarker(): Int = 3", + ], + }, + }, + minNodes: 30, + minEdges: 150, }, { language: "zig", diff --git a/tests/experiment/src/clang-background-baseline.mjs b/tests/experiment/src/clang-background-baseline.mjs new file mode 100644 index 00000000..70385bc4 --- /dev/null +++ b/tests/experiment/src/clang-background-baseline.mjs @@ -0,0 +1,94 @@ +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +/** Measure native clangd background indexing without requesting a graph. */ +export async function measureClangBackgroundIndex({ + command, + compilationDatabase, + cwd, + language, + sourceFile, + timeoutMs, + createClient, + now = () => performance.now(), + readSource = (file) => fs.readFileSync(file, "utf8"), +}) { + const started = now(); + const rootUri = pathToFileURL(cwd).href; + const sourceUri = pathToFileURL(sourceFile).href; + const client = createClient(command, [ + "--background-index", + `--compile-commands-dir=${path.dirname(compilationDatabase)}`, + ]); + let began = false; + let completed = false; + let timer; + let rejectProgress; + const settled = new Promise((resolve, reject) => { + rejectProgress = reject; + client.onNotification("$/progress", (params) => { + if ( + params?.token !== "backgroundIndexProgress" || + typeof params.value !== "object" || + params.value === null + ) { + return; + } + if (params.value.kind === "begin") { + began = true; + return; + } + if (params.value.kind === "end" && began) { + completed = true; + resolve(); + } + }); + }); + let progressError; + const observed = settled.catch((error) => { + progressError = error; + }); + try { + await client.request( + "initialize", + { + processId: process.pid, + rootUri, + capabilities: { window: { workDoneProgress: true } }, + workspaceFolders: [ + { + uri: rootUri, + name: "samchon-graph-clang-native-baseline", + }, + ], + }, + timeoutMs, + ); + if (!completed) { + timer = setTimeout( + () => + rejectProgress( + new Error("native clangd background indexing timed out"), + ), + timeoutMs, + ); + timer.unref?.(); + } + client.notify("initialized", {}); + client.notify("textDocument/didOpen", { + textDocument: { + uri: sourceUri, + languageId: language, + version: 1, + text: readSource(sourceFile), + }, + }); + await observed; + if (progressError !== undefined) throw progressError; + return Math.round(now() - started); + } finally { + clearTimeout(timer); + await client.close(); + } +} diff --git a/tests/experiment/src/clang-producer.mjs b/tests/experiment/src/clang-producer.mjs new file mode 100644 index 00000000..99c36a46 --- /dev/null +++ b/tests/experiment/src/clang-producer.mjs @@ -0,0 +1,345 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = path.resolve(here, "..", "..", ".."); +const workRoot = path.join(repositoryRoot, "tests", "experiment", ".work"); + +/** + * The complete owner of the native Clang producer cache. + * + * Workflows hash this file and the adapter pin, so changing the repository, + * revision, build dependencies, configure flags, build target, installed + * layout, or version admission changes the cache key. Language catalog rows + * import the same constants, while setup executes the recipe below. + */ +export const CLANG_PRODUCER_REPOSITORY = + "https://github.com/samchon/llvm-project.git"; +export const CLANG_PRODUCER_COMMIT = + "d6371c37445998d24776692a27e086bb24f9916a"; +export const CLANG_PRODUCER_BUILD_PACKAGES = Object.freeze([ + "clang", + "cmake", + "ninja-build", +]); +export const CLANG_PRODUCER_CACHE_INPUTS = Object.freeze([ + "packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts", + "tests/experiment/src/clang-producer.mjs", +]); + +const adapterPinFile = path.join( + repositoryRoot, + "packages", + "graph", + "src", + "provider", + "cpp", + "CPP_CLANG_PRODUCER_COMMIT.ts", +); + +/** Refuse a cache generation whose experiment and adapter pins diverge. */ +export function assertClangProducerAdapterPin( + source = fs.readFileSync(adapterPinFile, "utf8"), +) { + const match = + /CPP_CLANG_PRODUCER_COMMIT\s*=\s*"([0-9a-f]{40})"/u.exec(source); + if (match?.[1] !== CLANG_PRODUCER_COMMIT) { + throw new Error( + `native Clang owner pins ${CLANG_PRODUCER_COMMIT}, but the adapter pins ${match?.[1] ?? "no exact commit"}`, + ); + } + return match[1]; +} + +/** The decision used by both the standalone cache owner and lane setup. */ +export function clangProducerProvisionDecision({ installed, allowBuild }) { + if (installed) return "reuse"; + if (allowBuild) return "build"; + throw new Error( + "the pinned Clang producer cache is absent or invalid; only the workflow producer job may build it", + ); +} + +/** Resolve the native executable name without relying on PATHEXT. */ +export function clangProducerExecutable(command, platform = process.platform) { + return `${command}${platform === "win32" ? ".exe" : ""}`; +} + +/** Install or verify the one pinned native Clang producer. */ +export function installClangGraphProducer({ + language, + toolsRoot, + binRoot, + producerRepository, + producerCommit, + record, + allowBuild = true, + prepareBuild = () => undefined, + platform = process.platform, +}) { + if ( + producerRepository !== CLANG_PRODUCER_REPOSITORY || + producerCommit !== CLANG_PRODUCER_COMMIT + ) { + throw new Error( + `${language}: native Clang catalog pins ${String(producerRepository)}@${String(producerCommit)}, ` + + `expected ${CLANG_PRODUCER_REPOSITORY}@${CLANG_PRODUCER_COMMIT}`, + ); + } + assertClangProducerAdapterPin(); + + const installed = installedClangGraphProducer({ + toolsRoot, + binRoot, + platform, + }); + const decision = clangProducerProvisionDecision({ installed, allowBuild }); + if (decision === "reuse") { + recordClangTools(record); + return decision; + } + prepareBuild(); + + // The producer is a pinned commit, so its binary is a pure function of that + // commit and this recipe. A restored cache remains untrusted input: the same + // complete resource-tree and version checks used for a fresh build admit it. + const source = path.join(toolsRoot, "samchon-clangd-source"); + const build = path.join(source, "build"); + fs.rmSync(source, { force: true, recursive: true }); + ensureDir(source); + run("git", ["init", "--quiet"], { cwd: source }); + run("git", ["remote", "add", "origin", CLANG_PRODUCER_REPOSITORY], { + cwd: source, + }); + run( + "git", + ["fetch", "--depth=1", "origin", CLANG_PRODUCER_COMMIT], + { cwd: source }, + ); + run("git", ["checkout", "--detach", "FETCH_HEAD"], { cwd: source }); + const revision = String( + run("git", ["rev-parse", "HEAD"], { + cwd: source, + stdio: "pipe", + }).stdout, + ).trim(); + if (revision !== CLANG_PRODUCER_COMMIT) { + throw new Error( + `${language}: checked out native Clang ${revision}, expected ${CLANG_PRODUCER_COMMIT}`, + ); + } + run("cmake", [ + "-S", + path.join(source, "llvm"), + "-B", + build, + "-G", + "Ninja", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_CXX_COMPILER=clang++", + "-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra", + "-DLLVM_TARGETS_TO_BUILD=Native", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_INCLUDE_TESTS=OFF", + "-DCLANG_INCLUDE_TESTS=OFF", + "-DLLVM_INCLUDE_BENCHMARKS=OFF", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + `-DLLVM_FORCE_VC_REVISION=${CLANG_PRODUCER_COMMIT}`, + `-DLLVM_FORCE_VC_REPOSITORY=${CLANG_PRODUCER_REPOSITORY}`, + ]); + + // Hosted runners with the same advertised shape have varied from 56 to 107 + // minutes for this build. Use the machine rather than inventing a faster + // fixed count, while bounding compile concurrency by two GiB of total memory + // per job. This is a machine-class bound, not an out-of-memory guard. + const jobs = Math.max( + 1, + Math.min( + os.availableParallelism(), + Math.floor(os.totalmem() / (2 * 1024 * 1024 * 1024)), + ), + ); + console.log( + `${language}: building the pinned Clang producer with ${String(jobs)} jobs ` + + `(cores ${String(os.availableParallelism())}, ` + + `memory ${String(Math.round(os.totalmem() / (1024 * 1024 * 1024)))} GiB)`, + ); + run("cmake", [ + "--build", + build, + "--parallel", + String(jobs), + "--target", + "clangd", + ]); + + const binary = path.join( + build, + "bin", + clangProducerExecutable("clangd", platform), + ); + assertVersion(language, binary); + const builtResources = path.join(build, "lib", "clang"); + const resourceVersions = fs + .readdirSync(builtResources, { withFileTypes: true }) + .filter( + (entry) => + entry.isDirectory() && + fs + .statSync(path.join(builtResources, entry.name, "include"), { + throwIfNoEntry: false, + }) + ?.isDirectory(), + ) + .map((entry) => entry.name); + if (resourceVersions.length !== 1) { + throw new Error( + `${language}: native Clang produced ${resourceVersions.length} resource-header trees`, + ); + } + const installedResources = path.join(toolsRoot, "lib", "clang"); + fs.rmSync(installedResources, { force: true, recursive: true }); + ensureDir(path.dirname(installedResources)); + fs.cpSync(builtResources, installedResources, { recursive: true }); + const installedStddef = path.join( + installedResources, + resourceVersions[0], + "include", + "stddef.h", + ); + if (!fs.statSync(installedStddef, { throwIfNoEntry: false })?.isFile()) { + throw new Error( + `${language}: native Clang resource headers were not installed at ${installedStddef}`, + ); + } + for (const command of ["samchon-clangd", "clangd"]) { + const link = path.join(binRoot, clangProducerExecutable(command, platform)); + fs.rmSync(link, { force: true }); + fs.linkSync(binary, link); + } + assertVersion( + language, + path.join(binRoot, clangProducerExecutable("samchon-clangd", platform)), + ); + recordClangTools(record); + fs.rmSync(source, { force: true, recursive: true }); + return decision; +} + +function installedClangGraphProducer({ toolsRoot, binRoot, platform }) { + try { + const installed = path.join( + binRoot, + clangProducerExecutable("samchon-clangd", platform), + ); + const alias = path.join( + binRoot, + clangProducerExecutable("clangd", platform), + ); + if ( + !fs.statSync(installed, { throwIfNoEntry: false })?.isFile() || + !fs.statSync(alias, { throwIfNoEntry: false })?.isFile() + ) { + return false; + } + const resources = path.join(toolsRoot, "lib", "clang"); + const versions = fs + .readdirSync(resources, { withFileTypes: true }) + .filter( + (entry) => + entry.isDirectory() && + fs + .statSync(path.join(resources, entry.name, "include", "stddef.h"), { + throwIfNoEntry: false, + }) + ?.isFile(), + ); + if (versions.length !== 1) return false; + for (const binary of [installed, alias]) assertVersion("cache", binary); + return true; + } catch { + return false; + } +} + +function assertVersion(language, binary) { + const version = String(run(binary, ["--version"], { stdio: "pipe" }).stdout); + if (!version.includes(CLANG_PRODUCER_COMMIT)) { + throw new Error( + `${language}: native Clang version omits ${CLANG_PRODUCER_COMMIT}:\n${version}`, + ); + } +} + +function recordClangTools(record) { + record({ + tool: "samchon-clangd", + version: CLANG_PRODUCER_COMMIT, + source: `${CLANG_PRODUCER_REPOSITORY}@${CLANG_PRODUCER_COMMIT}`, + digest: `git:${CLANG_PRODUCER_COMMIT}`, + }); + record({ + tool: "clangd", + version: CLANG_PRODUCER_COMMIT, + source: "alias of samchon-clangd", + digest: `git:${CLANG_PRODUCER_COMMIT}`, + }); +} + +async function main() { + const language = "cpp"; + const toolsRoot = path.join(workRoot, "tools"); + const binRoot = path.join(toolsRoot, "bin"); + ensureDir(binRoot); + installClangGraphProducer({ + language, + toolsRoot, + binRoot, + producerRepository: CLANG_PRODUCER_REPOSITORY, + producerCommit: CLANG_PRODUCER_COMMIT, + record: () => undefined, + allowBuild: + process.env.SAMCHON_GRAPH_CLANG_PRODUCER_ALLOW_BUILD !== "0", + prepareBuild: () => { + shell("sudo apt-get update"); + shell( + `sudo apt-get install -y ${CLANG_PRODUCER_BUILD_PACKAGES.join(" ")}`, + ); + }, + }); +} + +function run(command, args = [], options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd ?? repositoryRoot, + env: { ...process.env, ...(options.env ?? {}) }, + shell: options.shell ?? false, + stdio: options.stdio ?? "inherit", + windowsHide: true, + }); + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(" ")} failed with exit code ${String(result.status)}`, + ); + } + return result; +} + +function shell(command) { + return run(command, [], { shell: true }); +} + +function ensureDir(directory) { + fs.mkdirSync(directory, { recursive: true }); +} + +if ( + process.argv[1] !== undefined && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + await main(); +} diff --git a/tests/experiment/src/evidence-summary.mjs b/tests/experiment/src/evidence-summary.mjs new file mode 100644 index 00000000..97d4e392 --- /dev/null +++ b/tests/experiment/src/evidence-summary.mjs @@ -0,0 +1,52 @@ +import { GRAPH_EDGE_KINDS } from "@samchon/graph"; + +const UNRESOLVED_REASONS = [ + "dynamic", + "reflection", + "macro-or-generated", + "conditional-build", + "external-boundary", + "analysis-error", + "excluded-input", + "identity-unstable", + "provider-gap", +]; + +/** All fifteen relationship families, compacted to state counts. */ +export function summarizeCoverage(dump, provider) { + const rows = (dump.coverage ?? []).filter( + (row) => provider === undefined || row.provider === provider, + ); + return { + provider: provider ?? null, + families: GRAPH_EDGE_KINDS.map((family) => { + const familyRows = rows.filter((row) => row.family === family); + return { + family, + complete: familyRows.filter((row) => row.state === "complete").length, + partial: familyRows.filter((row) => row.state === "partial").length, + unsupported: familyRows.filter((row) => row.state === "unsupported") + .length, + }; + }), + }; +} + +/** Stable uncertainty totals without copying any unbounded evidence sites. */ +export function summarizeUnresolved(dump, provider) { + const sites = (dump.unresolved ?? []).filter( + (site) => provider === undefined || site.provider === provider, + ); + return { + provider: provider ?? null, + total: sites.length, + byFamily: GRAPH_EDGE_KINDS.map((family) => ({ + family, + count: sites.filter((site) => site.family === family).length, + })), + byReason: UNRESOLVED_REASONS.map((reason) => ({ + reason, + count: sites.filter((site) => site.reason === reason).length, + })), + }; +} diff --git a/tests/experiment/src/git-tree.mjs b/tests/experiment/src/git-tree.mjs new file mode 100644 index 00000000..82bbea76 --- /dev/null +++ b/tests/experiment/src/git-tree.mjs @@ -0,0 +1,25 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { run } from "./process.mjs"; + +/** Prove that an extracted source archive is the exact pinned Git tree. */ +export const verifyGitTree = (source, expected) => { + const repository = path.join(source, ".git"); + fs.rmSync(repository, { force: true, recursive: true }); + try { + run("git", ["init", "--quiet"], { cwd: source }); + run("git", ["config", "core.autocrlf", "false"], { cwd: source }); + run("git", ["add", "--all", "--force"], { cwd: source }); + const actual = String( + run("git", ["write-tree"], { cwd: source, stdio: "pipe" }).stdout, + ).trim(); + if (actual !== expected) { + throw new Error( + `${source} has Git tree ${actual}, expected ${expected}`, + ); + } + } finally { + fs.rmSync(repository, { force: true, recursive: true }); + } +}; diff --git a/tests/experiment/src/java-producer-agreement.mjs b/tests/experiment/src/java-producer-agreement.mjs new file mode 100644 index 00000000..743dd47f --- /dev/null +++ b/tests/experiment/src/java-producer-agreement.mjs @@ -0,0 +1,276 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { + buildGraphDump, + javaDeclarationSymbol, + semanticGraphNodeId, +} from "@samchon/graph"; + +import { run } from "./process.mjs"; + +const JAVAC_OVERRIDE = "SAMCHON_GRAPH_JAVAC_GRAPH"; +const JDT_OVERRIDE = "SAMCHON_GRAPH_JDT_WORKSPACE"; +const JAVAC_PROVIDER = "javac-graph"; +const JDT_PROVIDER = "jdt-workspace"; + +/** Prove that the two compiler-owned Java lanes agree on persistent IDs. */ +export const runJavaProducerAgreement = async (experiment, root) => { + const javacLauncher = process.env[JAVAC_OVERRIDE]; + const jdtLauncher = process.env[JDT_OVERRIDE]; + if (typeof javacLauncher !== "string" || !path.isAbsolute(javacLauncher)) { + throw new Error("java agreement: the pinned javac producer is not configured"); + } + if (typeof jdtLauncher !== "string" || !path.isAbsolute(jdtLauncher)) { + throw new Error("java agreement: the pinned JDT producer is not configured"); + } + + writeAgreementClass( + path.join(root, "src", "main", "java", "com"), + "ProducerAgreement", + ); + const maven = await compareProducers({ + experiment, + root, + javacLauncher, + label: "Maven root", + declarations: declarationsFor("ProducerAgreement", "maven:."), + }); + + const gradleRoot = path.join(root, ".samchon-graph-gradle-agreement"); + prepareGradleAgreement(gradleRoot); + const gradle = await compareProducers({ + experiment, + root: gradleRoot, + javacLauncher, + label: "Gradle main/test/module", + declarations: [ + ...declarationsFor("GradleMainAgreement", ":compileJava"), + ...declarationsFor("GradleTestAgreement", ":compileTestJava"), + ...declarationsFor( + "GradleModuleAgreement", + ":module:compileJava", + ), + ], + }); + return { maven, gradle }; +}; + +async function compareProducers({ + experiment, + root, + javacLauncher, + label, + declarations, +}) { + const options = { + cwd: root, + mode: "lsp", + languages: ["java"], + lspTimeoutMs: experiment.timeoutMs ?? 60_000, + lspReadyTimeoutMs: experiment.readyTimeoutMs ?? 180_000, + lspWarmupTimeoutMs: experiment.warmupTimeoutMs ?? 180_000, + }; + const javac = await buildGraphDump(options); + const previousPath = process.env.PATH ?? ""; + const javacBin = path.dirname(path.resolve(javacLauncher)); + try { + delete process.env[JAVAC_OVERRIDE]; + process.env.PATH = previousPath + .split(path.delimiter) + .filter( + (candidate) => + candidate !== "" && path.resolve(candidate) !== javacBin, + ) + .join(path.delimiter); + const jdt = await buildGraphDump(options); + return assertAgreement(javac, jdt, label, declarations); + } finally { + process.env[JAVAC_OVERRIDE] = javacLauncher; + process.env.PATH = previousPath; + } +} + +function assertAgreement(javac, jdt, label, declarations) { + const javacProvenance = strictProvenance(javac, JAVAC_PROVIDER); + const jdtProvenance = strictProvenance(jdt, JDT_PROVIDER); + if ( + javacProvenance.provider === jdtProvenance.provider || + javacProvenance.producer.tool === jdtProvenance.producer.tool || + javacProvenance.universe === jdtProvenance.universe + ) { + throw new Error( + `java agreement (${label}): distinct producers published indistinguishable provenance: ${JSON.stringify({ javac: javacProvenance, jdt: jdtProvenance })}`, + ); + } + if ( + jdtProvenance.facts.length !== 1 || + jdtProvenance.facts[0] !== "contains" + ) { + throw new Error( + `java agreement (${label}): JDT published facts beyond containment: ${jdtProvenance.facts.join(", ")}`, + ); + } + + for (const expected of declarations) { + const javacNode = javac.nodes.find((node) => node.id === expected.id); + const jdtNode = jdt.nodes.find((node) => node.id === expected.id); + if (javacNode === undefined || jdtNode === undefined) { + throw new Error( + `java agreement (${label}): ${expected.kind} ${expected.qualifiedName} did not share ${expected.id}`, + ); + } + for (const node of [javacNode, jdtNode]) { + if ( + node.kind !== expected.kind || + node.name !== expected.name || + node.qualifiedName !== expected.qualifiedName + ) { + throw new Error( + `java agreement (${label}): ${expected.id} carried incompatible declaration metadata: ${JSON.stringify(node)}`, + ); + } + } + } + + return { + targets: [...new Set(declarations.map((row) => row.target))], + declarations: declarations.map(({ id, kind, qualifiedName }) => ({ + id, + kind, + qualifiedName, + })), + javac: provenanceSummary(javacProvenance), + jdt: provenanceSummary(jdtProvenance), + }; +} + +function declarationsFor(className, target) { + const qualified = `com.${className}`; + return [ + declaration("class", className, qualified, target), + declaration("field", "value", `${qualified}.value`, target), + declaration( + "constructor", + className, + `${qualified}.${className}`, + target, + "int", + ), + declaration("method", "twice", `${qualified}.twice`, target, "int"), + ]; +} + +function declaration( + kind, + name, + qualifiedName, + target, + parameters = "", +) { + const symbol = javaDeclarationSymbol({ + kind, + name, + qualifiedName, + ...(parameters === "" ? {} : { signature: `(${parameters})` }), + }); + return { + id: semanticGraphNodeId( + { + version: 2, + language: "java", + symbol, + role: kind, + native: { key: symbol, stability: "semantic" }, + scope: { target }, + stability: "persistent", + }, + qualifiedName, + ), + kind, + name, + qualifiedName, + target, + }; +} + +function prepareGradleAgreement(root) { + fs.rmSync(root, { force: true, recursive: true }); + fs.mkdirSync(root, { recursive: true }); + fs.writeFileSync( + path.join(root, "settings.gradle"), + "rootProject.name = 'producer-agreement'\ninclude 'module'\n", + ); + fs.writeFileSync( + path.join(root, "build.gradle"), + "plugins { id 'java' }\n", + ); + fs.mkdirSync(path.join(root, "module"), { recursive: true }); + fs.writeFileSync( + path.join(root, "module", "build.gradle"), + "plugins { id 'java' }\n", + ); + writeAgreementClass( + path.join(root, "src", "main", "java", "com"), + "GradleMainAgreement", + ); + writeAgreementClass( + path.join(root, "src", "test", "java", "com"), + "GradleTestAgreement", + ); + writeAgreementClass( + path.join(root, "module", "src", "main", "java", "com"), + "GradleModuleAgreement", + ); + run( + "gradle", + ["wrapper", "--gradle-version", "9.4.1", "--no-daemon"], + { cwd: root }, + ); +} + +function writeAgreementClass(directory, className) { + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync( + path.join(directory, `${className}.java`), + [ + "package com;", + "", + `public final class ${className} {`, + " public final int value;", + "", + ` public ${className}(int value) {`, + " this.value = value;", + " }", + "", + " public int twice(int factor) {", + " return value * factor;", + " }", + "}", + "", + ].join("\n"), + ); +} + +function strictProvenance(dump, provider) { + const provenance = dump.provenance?.find( + (candidate) => candidate.provider === provider, + ); + if (provenance === undefined) { + throw new Error( + `java agreement: ${provider} did not publish strict provenance: ${(dump.warnings ?? []).join("; ")}`, + ); + } + return provenance; +} + +function provenanceSummary(provenance) { + return { + provider: provenance.provider, + producer: provenance.producer, + universe: provenance.universe, + manifest: provenance.manifest, + content: provenance.content, + facts: provenance.facts, + }; +} diff --git a/tests/experiment/src/kotlin-build-report.mjs b/tests/experiment/src/kotlin-build-report.mjs new file mode 100644 index 00000000..8012cba7 --- /dev/null +++ b/tests/experiment/src/kotlin-build-report.mjs @@ -0,0 +1,131 @@ +import fs from "node:fs"; +import path from "node:path"; + +/** Publish the latest KGP invalidation decision without leaking host paths. */ +export function captureKotlinBuildReport(projectRoot, relativeReportRoot) { + const project = fs.realpathSync(projectRoot); + const requested = path.resolve(project, relativeReportRoot); + assertDescendant(project, requested); + if (!fs.statSync(requested, { throwIfNoEntry: false })?.isDirectory()) { + throw new Error(`Kotlin build-report directory does not exist: ${requested}`); + } + const reportRoot = fs.realpathSync(requested); + assertDescendant(project, reportRoot); + const reports = walkFiles(reportRoot).filter((file) => file.endsWith(".json")); + if (reports.length === 0) { + throw new Error(`Kotlin build-report directory contains no JSON report: ${reportRoot}`); + } + reports.sort((left, right) => { + const elapsed = fs.statSync(left).mtimeMs - fs.statSync(right).mtimeMs; + return elapsed === 0 ? compareUtf8(left, right) : elapsed; + }); + const report = JSON.parse(fs.readFileSync(reports.at(-1), "utf8")); + if (!isRecord(report) || !Array.isArray(report.buildOperationRecord)) { + throw new Error("Kotlin build report contains no buildOperationRecord list"); + } + const tasks = report.buildOperationRecord + .filter( + (operation) => + isRecord(operation) && + typeof operation.path === "string" && + /compile.*Kotlin$/iu.test(operation.path), + ) + .map((operation) => summarizeTask(project, operation)); + if (tasks.length === 0) { + throw new Error("Kotlin build report contains no Kotlin compilation task"); + } + return { tasks }; +} + +function summarizeTask(project, operation) { + const lines = Array.isArray(operation.icLogLines) + ? operation.icLogLines.filter((line) => typeof line === "string") + : []; + const nonIncremental = lines.find((line) => + line.startsWith("Non-incremental compilation will be performed:"), + ); + const classpath = lines.find((line) => + line.startsWith("Classpath changes info passed from Gradle task:"), + ); + const completedIncrementally = lines.includes("Incremental compilation completed"); + const attributes = operation.buildMetrics?.buildAttributes?.myAttributes; + return { + task: operation.path, + didWork: operation.didWork === true, + ...(typeof operation.skipMessage === "string" + ? { skipMessage: operation.skipMessage } + : {}), + ...(Number.isFinite(operation.totalTimeMs) + ? { elapsedMs: operation.totalTimeMs } + : {}), + ...(nonIncremental !== undefined + ? { incremental: false, invalidation: nonIncremental } + : completedIncrementally + ? { incremental: true } + : {}), + ...(classpath === undefined ? {} : { classpath }), + ...changedFiles(project, operation.changedFiles), + ...(isRecord(attributes) + ? { + buildAttributes: Object.keys(attributes) + .filter((key) => Number(attributes[key]) > 0) + .sort(compareUtf8), + } + : {}), + daemon: lines.some((line) => line.includes("DAEMON strategy")), + }; +} + +function changedFiles(project, value) { + if (!isRecord(value)) return {}; + const normalize = (rows) => + Array.isArray(rows) + ? rows + .filter((file) => typeof file === "string") + .map((file) => { + const relative = path.relative(project, path.resolve(file)); + return relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ? "" + : relative.split(path.sep).join("/"); + }) + .filter((file, index, files) => files.indexOf(file) === index) + .sort(compareUtf8) + : []; + const modified = normalize(value.modifiedFiles); + const removed = normalize(value.removedFiles); + return modified.length === 0 && removed.length === 0 + ? {} + : { changedFiles: { modified, removed } }; +} + +function walkFiles(root) { + const files = []; + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const file = path.join(root, entry.name); + if (entry.isDirectory()) files.push(...walkFiles(file)); + else if (entry.isFile()) files.push(file); + } + return files; +} + +function assertDescendant(root, candidate) { + const relative = path.relative(root, candidate); + if ( + relative === "" || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw new Error(`Kotlin build-report directory escapes its project: ${candidate}`); + } +} + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function compareUtf8(left, right) { + return Buffer.compare(Buffer.from(left, "utf8"), Buffer.from(right, "utf8")); +} diff --git a/tests/experiment/src/lifecycle-performance.mjs b/tests/experiment/src/lifecycle-performance.mjs new file mode 100644 index 00000000..b3682746 --- /dev/null +++ b/tests/experiment/src/lifecycle-performance.mjs @@ -0,0 +1,171 @@ +/** Measure validated no-op and real body-edit samples, then restore the source. */ +export async function measureLifecyclePerformance(props) { + validate(props); + let currentDump = props.currentDump; + let currentIdentity = props.currentIdentity; + const initialDump = props.currentDump; + const initialIdentity = props.currentIdentity; + const noops = []; + for (let index = 0; index < props.noopSamples; index++) { + const sample = await props.load(); + noops.push(sample.elapsedMs); + if ( + sample.dump !== currentDump || + sample.mode !== "unchanged" || + sample.identity !== currentIdentity + ) { + throw new Error( + `${props.language}: performance no-op ${String(index + 1)} replaced the resident generation`, + ); + } + } + + const edits = []; + const editEvidence = []; + for (let index = 0; index < props.editSamples; index++) { + props.writeSource( + props.sourceText.replace( + props.editFind, + props.editReplacements[index % props.editReplacements.length], + ), + ); + const sample = await props.load(); + edits.push(sample.elapsedMs); + if ( + !props.changedModes.includes(sample.mode) || + sample.dump === currentDump || + sample.identity === currentIdentity + ) { + throw new Error( + `${props.language}: performance edit ${String(index + 1)} did not replace strict provenance`, + ); + } + currentDump = sample.dump; + currentIdentity = sample.identity; + const evidence = props.captureEditEvidence?.(); + if (evidence !== undefined) editEvidence.push(evidence); + } + + props.writeSource(props.sourceText); + const restored = await props.load(); + if ( + !props.changedModes.includes(restored.mode) || + restored.dump === currentDump || + restored.identity !== initialIdentity + ) { + throw new Error( + `${props.language}: performance sampling did not restore its source generation ` + + `(mode=${String(restored.mode)}, ` + + `dumpChanged=${String(restored.dump !== currentDump)}, ` + + `identityRestored=${String(restored.identity === initialIdentity)}, ` + + `initial=${initialIdentity}, restored=${restored.identity}, ` + + `difference=${props.describeDifference(initialDump, restored.dump)})`, + ); + } + currentDump = restored.dump; + currentIdentity = restored.identity; + + const noopP95Ms = nearestRankP95(noops); + const editP95Ms = nearestRankP95(edits); + if (noopP95Ms >= props.noopP95MaxMs || editP95Ms >= props.editP95MaxMs) { + throw new Error( + `${props.language}: lifecycle performance missed its target: ` + + `no-op p95 ${String(noopP95Ms)}/${String(props.noopP95MaxMs)} ms, ` + + `edit p95 ${String(editP95Ms)}/${String(props.editP95MaxMs)} ms`, + ); + } + return { + dump: currentDump, + identity: currentIdentity, + row: { + name: "performance", + status: "passed", + noopSamples: noops, + editSamples: edits, + noopP95Ms, + editP95Ms, + noopP95MaxMs: props.noopP95MaxMs, + editP95MaxMs: props.editP95MaxMs, + ...(editEvidence.length === 0 ? {} : { editEvidence }), + }, + }; +} + +/** Measure a resident no-op p95 without inventing an edit ceiling. */ +export async function measureLifecycleNoopPerformance(props) { + for (const [name, value] of Object.entries({ + samples: props.samples, + p95MaxMs: props.p95MaxMs, + })) { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error( + `${props.language}: lifecycle no-op performance ${name} must be a positive integer`, + ); + } + } + const samples = []; + for (let index = 0; index < props.samples; index++) { + const sample = await props.load(); + samples.push(sample.elapsedMs); + if ( + sample.dump !== props.currentDump || + sample.mode !== "unchanged" || + sample.identity !== props.currentIdentity + ) { + throw new Error( + `${props.language}: performance no-op ${String(index + 1)} replaced the resident generation`, + ); + } + } + const p95Ms = nearestRankP95(samples); + if (p95Ms >= props.p95MaxMs) { + throw new Error( + `${props.language}: lifecycle no-op performance missed its target: ` + + `p95 ${String(p95Ms)}/${String(props.p95MaxMs)} ms`, + ); + } + return { + name: "noop-performance", + status: "passed", + samples, + p95Ms, + p95MaxMs: props.p95MaxMs, + }; +} + +export function nearestRankP95(samples) { + if (samples.length === 0) { + throw new Error("nearestRankP95 requires at least one sample"); + } + const sorted = [...samples].sort((left, right) => left - right); + return sorted[Math.ceil(sorted.length * 0.95) - 1]; +} + +function validate(props) { + for (const [name, value] of Object.entries({ + noopSamples: props.noopSamples, + editSamples: props.editSamples, + noopP95MaxMs: props.noopP95MaxMs, + editP95MaxMs: props.editP95MaxMs, + })) { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error( + `${props.language}: lifecycle performance ${name} must be a positive integer`, + ); + } + } + if ( + typeof props.editFind !== "string" || + props.editFind === "" || + !Array.isArray(props.editReplacements) || + props.editReplacements.length < 2 || + props.editReplacements.some( + (value) => typeof value !== "string" || value === "", + ) || + !props.sourceText.includes(props.editFind) + ) { + throw new Error( + `${props.language}: lifecycle performance requires two real body-edit replacements`, + ); + } +} diff --git a/tests/experiment/src/process.mjs b/tests/experiment/src/process.mjs index 2f4c8757..12d791d0 100644 --- a/tests/experiment/src/process.mjs +++ b/tests/experiment/src/process.mjs @@ -38,7 +38,7 @@ export const run = (command, args = [], options = {}) => { stdio: options.stdio ?? "inherit", windowsHide: true, }); - if (result.status !== 0) { + if (result.status !== 0 && options.check !== false) { throw new Error(`${command} ${args.join(" ")} failed with exit code ${result.status}`); } return result; diff --git a/tests/experiment/src/regeneration-evidence.mjs b/tests/experiment/src/regeneration-evidence.mjs new file mode 100644 index 00000000..338226a5 --- /dev/null +++ b/tests/experiment/src/regeneration-evidence.mjs @@ -0,0 +1,243 @@ +import fs from "node:fs"; +import path from "node:path"; + +const SHA256 = /^[0-9a-f]{64}$/u; + +/** + * Read the producer-owned universe rows behind every current generation. + * + * The public dump intentionally carries only universe digests. When a strict + * regeneration check fails, those digests prove that something moved but hide + * which producer input moved. This reader stays inside the isolated experiment + * corpus and reports only the committed universe records that produced the + * rejected public result. + */ +export function captureGenerationEvidence(projectRoot, relativeStoreRoot) { + if ( + typeof relativeStoreRoot !== "string" || + relativeStoreRoot.trim() === "" + ) { + throw new Error("regeneration evidence requires a non-empty store root"); + } + const project = fs.realpathSync(projectRoot); + const requestedStore = path.resolve(project, relativeStoreRoot); + assertDescendant(project, requestedStore, "regeneration evidence store"); + if (!fs.statSync(requestedStore, { throwIfNoEntry: false })?.isDirectory()) { + throw new Error( + `regeneration evidence store does not exist: ${requestedStore}`, + ); + } + const store = fs.realpathSync(requestedStore); + assertDescendant(project, store, "regeneration evidence store"); + + const currents = walkFiles(store).filter( + (file) => path.basename(file) === "CURRENT", + ); + if (currents.length === 0) { + throw new Error(`regeneration evidence store has no CURRENT pointer: ${store}`); + } + + const rows = []; + for (const current of currents) { + const generation = fs.readFileSync(current, "utf8").trim(); + if (!SHA256.test(generation)) { + throw new Error(`regeneration evidence has an invalid CURRENT: ${current}`); + } + const target = path.dirname(current); + const requestedGeneration = path.resolve(target, "generations", generation); + assertDescendant( + target, + requestedGeneration, + "regeneration evidence generation", + ); + if ( + !fs.statSync(requestedGeneration, { throwIfNoEntry: false })?.isDirectory() + ) { + throw new Error( + `regeneration evidence CURRENT names no generation: ${current}`, + ); + } + const committed = fs.realpathSync(requestedGeneration); + assertDescendant(target, committed, "regeneration evidence generation"); + const requestedUniverse = path.join(committed, "UNIVERSE"); + if (!fs.statSync(requestedUniverse, { throwIfNoEntry: false })?.isFile()) { + throw new Error(`regeneration evidence generation has no UNIVERSE: ${committed}`); + } + const universe = fs.realpathSync(requestedUniverse); + assertDescendant(committed, universe, "regeneration evidence universe"); + const universeLines = readLines(universe); + for (const line of universeLines) { + rows.push(`generation universe:${line}`); + } + const requestedCompiler = path.join(committed, ".universe"); + if ( + fs.statSync(requestedCompiler, { throwIfNoEntry: false })?.isDirectory() + ) { + const compiler = fs.realpathSync(requestedCompiler); + assertDescendant( + committed, + compiler, + "regeneration compiler universe", + ); + for (const file of walkFiles(compiler).filter((path) => + path.endsWith(".args"), + )) { + const invocation = readLines(file).map((line) => + diagnosticLine(file, line), + ); + rows.push(`compiler invocation:${JSON.stringify(invocation)}`); + } + } + } + return rows.sort(compareUtf8); +} + +/** Name the first exact committed producer row that moved. */ +export function firstEvidenceDifference(left, right) { + if (left === undefined && right === undefined) { + return "no producer evidence was configured"; + } + if (left === undefined || right === undefined) { + return `producer evidence ${left === undefined ? "appeared" : "disappeared"}`; + } + const removedRows = multisetRemainder(left, right); + const addedRows = multisetRemainder(right, left); + const removedCompiler = removedRows.find(isCompilerInvocation); + const addedCompiler = addedRows.find(isCompilerInvocation); + const compilerMoved = + removedCompiler !== undefined || addedCompiler !== undefined; + const removed = compilerMoved ? removedCompiler : removedRows.at(0); + const added = compilerMoved ? addedCompiler : addedRows.at(0); + if (removed !== undefined || added !== undefined) { + const focus = firstDifferenceIndex(removed, added); + return `${bounded(removed, focus)} -> ${bounded(added, focus)}`; + } + for (let index = 0; index < left.length; index++) { + if (left[index] !== right[index]) { + const focus = firstDifferenceIndex(left[index], right[index]); + return `${bounded(left[index], focus)} -> ${bounded(right[index], focus)}`; + } + } + return "committed producer universe rows are equal"; +} + +function isCompilerInvocation(row) { + return row.startsWith("compiler invocation:"); +} + +function multisetRemainder(source, matched) { + const counts = new Map(); + for (const row of matched) counts.set(row, (counts.get(row) ?? 0) + 1); + const remainder = []; + for (const row of source) { + const count = counts.get(row) ?? 0; + if (count === 0) remainder.push(row); + else counts.set(row, count - 1); + } + return remainder; +} + +function diagnosticLine(file, line) { + if (!file.endsWith(".args") || line.startsWith("@")) return line; + let decoded; + try { + decoded = Buffer.from(line, "base64url").toString("utf8"); + } catch { + return line; + } + if (Buffer.from(decoded, "utf8").toString("base64url") !== line) return line; + return decoded.replaceAll(/\|literal:([^|]*)/gu, (_, token) => { + const literal = Buffer.from(token, "base64url").toString("utf8"); + return `|literal:${JSON.stringify(literal)}`; + }); +} + +function readLines(file) { + const lines = fs.readFileSync(file, "utf8").split(/\r?\n/u); + if (lines.at(-1) === "") lines.pop(); + return lines; +} + +function walkFiles(root) { + const files = []; + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const file = path.join(root, entry.name); + if (entry.isDirectory()) files.push(...walkFiles(file)); + else if (entry.isFile()) files.push(file); + } + return files.sort(compareUtf8); +} + +function assertDescendant(root, candidate, label) { + const relative = path.relative(root, candidate); + if ( + relative === "" || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw new Error(`${label} escapes its project: ${candidate}`); + } +} + +function compareUtf8(left, right) { + return Buffer.compare(Buffer.from(left, "utf8"), Buffer.from(right, "utf8")); +} + +function firstDifferenceIndex(left, right) { + if (left === undefined || right === undefined) return undefined; + const limit = Math.min(left.length, right.length); + for (let index = 0; index < limit; index++) { + if (left[index] !== right[index]) return index; + } + return limit; +} + +function bounded(value, focus) { + if (value === undefined) return "missing"; + const limit = 480; + if (value.length <= limit) return value; + if (focus === undefined) { + const half = (limit - 3) / 2; + return `${codePointSlice(value, 0, Math.ceil(half))}...${codePointSlice( + value, + value.length - Math.floor(half), + value.length, + )}`; + } + const contentLimit = limit - 6; + const initialStart = Math.max(0, focus - Math.floor(contentLimit / 2)); + const end = Math.min(value.length, initialStart + contentLimit); + const start = Math.max(0, end - contentLimit); + return `${start > 0 ? "..." : ""}${codePointSlice(value, start, end)}${end < value.length ? "..." : ""}`; +} + +function codePointSlice(value, requestedStart, requestedEnd) { + let start = requestedStart; + let end = requestedEnd; + if ( + start > 0 && + start < value.length && + isLowSurrogate(value.charCodeAt(start)) && + isHighSurrogate(value.charCodeAt(start - 1)) + ) { + start++; + } + if ( + end > 0 && + end < value.length && + isHighSurrogate(value.charCodeAt(end - 1)) && + isLowSurrogate(value.charCodeAt(end)) + ) { + end--; + } + return value.slice(start, end); +} + +function isHighSurrogate(value) { + return value >= 0xd800 && value <= 0xdbff; +} + +function isLowSurrogate(value) { + return value >= 0xdc00 && value <= 0xdfff; +} diff --git a/tests/experiment/src/representative-edges.mjs b/tests/experiment/src/representative-edges.mjs new file mode 100644 index 00000000..dc652a65 --- /dev/null +++ b/tests/experiment/src/representative-edges.mjs @@ -0,0 +1,14 @@ +/** Require one exact qualified/name endpoint pair from a real graph. */ +export function hasRepresentativeEdge(dump, claim) { + const nodes = new Map(dump.nodes.map((node) => [node.id, node])); + const nameOf = (id) => { + const node = nodes.get(id); + return node?.qualifiedName ?? node?.name; + }; + return dump.edges.some( + (edge) => + edge.kind === claim.kind && + nameOf(edge.from) === claim.from && + nameOf(edge.to) === claim.to, + ); +} diff --git a/tests/experiment/src/run-language.mjs b/tests/experiment/src/run-language.mjs index e4d6146e..55357adb 100644 --- a/tests/experiment/src/run-language.mjs +++ b/tests/experiment/src/run-language.mjs @@ -5,6 +5,10 @@ import path from "node:path"; import { buildGraphDump } from "@samchon/graph"; import { findExperiment } from "./catalog.mjs"; +import { + summarizeCoverage, + summarizeUnresolved, +} from "./evidence-summary.mjs"; import { activateProvisionedTools, assertPinnedCorpus, @@ -17,6 +21,8 @@ import { toolManifest, } from "./process.mjs"; import { runStrictLifecycle } from "./strict-lifecycle.mjs"; +import { hasRepresentativeEdge } from "./representative-edges.mjs"; +import { runJavaProducerAgreement } from "./java-producer-agreement.mjs"; activateProvisionedTools(); @@ -160,11 +166,16 @@ if (dump.indexer === "static") { if (dump.languages.includes(experiment.language) === false) { throw new Error(`${experiment.language}: dump languages did not include ${experiment.language}`); } -if (!strict && dump.nodes.length < experiment.minNodes) { +const enforceMinimums = !strict || experiment.strictMinimums === true; +if ( + enforceMinimums && + experiment.minNodes !== undefined && + dump.nodes.length < experiment.minNodes +) { throw new Error(`${experiment.language}: expected at least ${experiment.minNodes} nodes, got ${dump.nodes.length}`); } const minEdges = experiment.minEdges ?? 0; -if (!strict && dump.edges.length < minEdges) { +if (enforceMinimums && dump.edges.length < minEdges) { throw new Error(`${experiment.language}: expected at least ${minEdges} relationship edges, got ${dump.edges.length}`); } const provenance = strict ? declaredProvenance : undefined; @@ -223,6 +234,13 @@ const edgeKindCounts = Object.fromEntries( dump.edges.filter((edge) => edge.kind === kind).length, ]), ); +for (const claim of strict ? experiment.representativeEdges ?? [] : []) { + if (!hasRepresentativeEdge(dump, claim)) { + throw new Error( + `${experiment.language}: representative ${claim.from} -[${claim.kind}]-> ${claim.to} edge was not proved`, + ); + } +} // A small pinned build fixture can truthfully exercise a relationship only in // the isolated create/rename transition. `runStrictLifecycle` has already // required this exact edge in both generations; count that evidence instead of @@ -297,6 +315,11 @@ if (warnings.some((warning) => /LSP indexing failed|LSP returned no symbols|serv throw new Error(`${experiment.language}: LSP warning failed experiment: ${warnings.join("; ")}`); } +const producerAgreement = + experiment.language === "java" + ? await runJavaProducerAgreement(experiment, cwd) + : undefined; + // Read after the whole run rather than before it: what has to be proved is that // nothing this run did — preparation, indexing, or lifecycle editing — reached // the clone whose commit the result publishes. @@ -319,7 +342,10 @@ const result = { diagnosticCount: dump.diagnostics?.length ?? 0, strictProvider: experiment.strictProvider, provenance, + coverageSummary: summarizeCoverage(dump, provenance?.provider), + unresolvedSummary: summarizeUnresolved(dump, provenance?.provider), edgeKindCounts, + representativeEdges: experiment.representativeEdges, semanticLimitation: experiment.semanticLimitation, compilerLimitation: experiment.compilerLimitation, regenerationLimitation: experiment.regenerationLimitation, @@ -328,6 +354,7 @@ const result = { crossFileCalls, crossFileRelationships, lifecycle: lifecycle?.rows, + producerAgreement, warnings, sampleNodes: dump.nodes.slice(0, 20).map((node) => ({ id: node.id, diff --git a/tests/experiment/src/run-topology-orientation.mjs b/tests/experiment/src/run-topology-orientation.mjs new file mode 100644 index 00000000..0ba8bab0 --- /dev/null +++ b/tests/experiment/src/run-topology-orientation.mjs @@ -0,0 +1,468 @@ +#!/usr/bin/env node +import cp from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { createResidentRepositoryContextSource } from "@samchon/graph"; + +import { parseArgs, repositoryRoot } from "./process.mjs"; +import { TOPOLOGY_ORIENTATION_ORACLE } from "./topology-orientation-oracle.mjs"; + +const QUESTION = + "Explain the workspaces, applications and packages, their source/test/generated roots and entrypoints, and the project dependency flow relevant to @samchon/graph."; + +const args = parseArgs(process.argv.slice(2)); +const root = path.resolve(args.cwd ?? repositoryRoot); +const out = + args.out === undefined ? undefined : path.resolve(process.cwd(), args.out); + +const baseline = directOrientation(root); +const oracleInputDigest = orientationInputDigest( + root, + TOPOLOGY_ORIENTATION_ORACLE.inputFiles, +); +const client = new Client({ + name: "samchon-graph-topology-orientation", + version: "1.0.0", +}); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [ + path.join(repositoryRoot, "packages", "graph", "lib", "bin.js"), + "--mode", + "static", + "--cwd", + root, + ], + stderr: "pipe", + env: { ...process.env, SAMCHON_GRAPH_TOPOLOGY_TRACE: "1" }, +}); +const topologyResident = createResidentRepositoryContextSource(root); +let serverStderr = ""; + +try { + const request = { + question: QUESTION, + draft: { reason: "repository orientation", type: "topology" }, + review: "compare the indexed topology with direct manifest orientation", + request: { + type: "topology", + relations: [ + "contains", + "depends-on", + "source-of", + "test-of", + "produces", + "invokes", + "entrypoint-of", + "joins-file", + ], + limit: 200, + joinLimit: 64, + }, + }; + const modelColdStarted = performance.now(); + const modelCold = await topologyResident.load(); + const topologyModelColdMs = Math.round( + performance.now() - modelColdStarted, + ); + const modelNoopStarted = performance.now(); + const modelNoop = await topologyResident.load(); + const topologyModelNoopMs = Math.round( + performance.now() - modelNoopStarted, + ); + const readyStarted = performance.now(); + await client.connect(transport); + transport.stderr?.setEncoding("utf8"); + transport.stderr?.on("data", (chunk) => { + serverStderr += chunk; + }); + const mcpReadyMs = Math.round(performance.now() - readyStarted); + const coldStarted = performance.now(); + const cold = await callTopology(client, request); + const coldMs = Math.round(performance.now() - coldStarted); + const phaseTrace = await waitForTopologyPhases(() => serverStderr); + const toolStartupPhase = onePhase( + phaseTrace, + "pnpm-workspace", + "tool-startup", + ); + const modelQueryPhase = onePhase( + phaseTrace, + "pnpm-workspace", + "model-query", + ); + const normalizationPhase = onePhase( + phaseTrace, + "pnpm-workspace", + "normalization", + ); + const joinPhase = onePhase(phaseTrace, "repository-context", "join"); + const toolStartupMs = toolStartupPhase.durationMs; + const modelQueryMs = modelQueryPhase.durationMs; + const normalizationMs = normalizationPhase.durationMs; + const joinMs = joinPhase.durationMs; + const normalizationJoinMs = normalizationMs + joinMs; + const noopStarted = performance.now(); + const noop = await callTopology(client, request); + const noopMs = Math.round(performance.now() - noopStarted); + if (cold.result.type !== "topology" || noop.result.type !== "topology") { + throw new Error( + "topology orientation experiment received a non-topology result", + ); + } + + const projects = cold.result.nodes.filter((node) => + ["workspace", "project"].includes(node.kind), + ); + const packages = cold.result.nodes.filter((node) => node.kind === "package"); + const roots = cold.result.nodes.filter((node) => + ["source-root", "generated-root"].includes(node.kind), + ); + const entrypoints = cold.result.nodes.filter( + (node) => node.kind === "entrypoint", + ); + const dependencies = cold.result.edges.filter( + (edge) => edge.kind === "depends-on", + ); + const names = new Map(cold.result.nodes.map((node) => [node.id, node.name])); + const dependencyFacts = dependencies.map((edge) => ({ + from: names.get(edge.from) ?? edge.from, + to: names.get(edge.to) ?? edge.to, + authority: edge.authority, + })); + const joins = cold.result.edges.filter((edge) => edge.kind === "joins-file"); + const joinedFiles = new Set(joins.map((edge) => edge.to)); + const semanticFollowUpFiles = [ + "packages/graph/src/index.ts", + "packages/graph/src/SamchonGraphApplication.ts", + ].filter((file) => joinedFiles.has(file)); + const unsupported = cold.result.coverage + .filter((row) => row.state === "unsupported") + .map((row) => row.family) + .sort(compareText); + const oracleNodes = TOPOLOGY_ORIENTATION_ORACLE.nodes; + const result = { + schemaVersion: 1, + question: QUESTION, + root, + method: { + agentCalls: 0, + inputTokens: 0, + paidAgent: false, + note: "This deterministic zero-spend run compares two real stdio MCP topology calls with direct repository orientation; it is not an agent A/B benchmark.", + }, + indexed: { + topologyMcpCalls: 2, + topologyModelLoads: 2, + rgCalls: 0, + directoryWalks: 0, + rawManifestReads: 0, + mcpReadyMs, + coldTopologyMs: coldMs, + warmTopologyMcpMs: noopMs, + topologyModelColdMs, + topologyModelNoopMs, + normalizationJoinMs, + phases: { + toolStartupMs, + modelQueryMs, + normalizationMs, + joinMs, + normalizationJoinMs, + mcpReadyMs, + }, + phaseTrace, + provenance: cold.result.provenance, + coverage: cold.result.coverage, + join: cold.result.join, + projects: projects.length, + packages: packages.length, + roots: roots.length, + entrypoints: entrypoints.length, + dependencies: dependencies.length, + joins: joins.length, + semanticFollowUpFiles, + unsupported, + inferredFacts: cold.result.nodes.filter( + (node) => node.authority === "inferred", + ).length, + facts: { + projects, + packages, + roots, + entrypoints, + dependencies, + }, + }, + oracle: { + inputFiles: TOPOLOGY_ORIENTATION_ORACLE.inputFiles, + inputDigest: oracleInputDigest, + }, + direct: baseline, + correctness: { + oracleInputsMatch: + oracleInputDigest === TOPOLOGY_ORIENTATION_ORACLE.inputDigest, + projectFactsMatch: factSetMatches( + projects, + oracleNodes.filter((node) => + ["workspace", "project"].includes(node.kind), + ), + ), + packageFactsMatch: factSetMatches( + packages, + oracleNodes.filter((node) => node.kind === "package"), + ), + rootFactsMatch: factSetMatches( + roots, + oracleNodes.filter((node) => + ["source-root", "generated-root"].includes(node.kind), + ), + ), + entrypointFactsMatch: factSetMatches( + entrypoints, + oracleNodes.filter((node) => node.kind === "entrypoint"), + ), + dependencyFactsMatch: factSetMatches( + dependencies, + TOPOLOGY_ORIENTATION_ORACLE.dependencies, + ), + everyPackageHasEvidence: packages.every( + (node) => node.evidence !== undefined, + ), + noInferredClaims: cold.result.nodes.every( + (node) => node.authority !== "inferred", + ), + hasWorkspaceProject: projects.some( + (node) => node.kind === "workspace" && node.coordinate === ".", + ), + hasGraphSourceAndGeneratedRoots: + roots.some((node) => node.root === "packages/graph/src") && + roots.some((node) => node.root === "packages/graph/lib"), + hasGraphCliEntrypoint: entrypoints.some( + (node) => node.file === "packages/graph/lib/bin.js", + ), + hasExperimentDependencyOnGraph: dependencyFacts.some( + (edge) => + edge.from === "@samchon/graph-experiment" && + edge.to === "@samchon/graph", + ), + fileJoinsWereFenced: cold.result.join.state === "compatible", + warmMcpMatchesCold: + JSON.stringify(noop.result) === JSON.stringify(cold.result), + topologyModelNoopReusedGeneration: modelCold === modelNoop, + topologyModelNoopUnder250Ms: topologyModelNoopMs < 250, + tracedActualJoinWasCompatible: + joinPhase.compatible === true && + typeof joinPhase.codeFiles === "number" && + joinPhase.codeFiles > joins.length, + semanticFollowUpReachedGraphApi: semanticFollowUpFiles.length === 2, + }, + limitations: [ + "The direct comparison uses pnpm's resolved member list plus raw manifests; it does not claim shell or source reads are eliminated.", + "The public MCP boundary exposes handshake, cold-call and warm-call latency; those calls include code-graph validation and wire costs, while the separate topology-model no-op is the issue's validated-input target.", + "The opt-in phase trace comes from the actual MCP server's pnpm provider, transaction normalizer and full generation-fenced code-file join; these nested durations are not claimed to sum to the independently measured cold MCP call.", + ...unsupported.map((family) => + `The pnpm provider reports ${family} as unsupported for this generation.`, + ), + ], + }; + if (!Object.values(result.correctness).every((value) => value === true)) { + const latencies = { + mcpReadyMs, + coldMs, + noopMs, + topologyModelColdMs, + topologyModelNoopMs, + normalizationJoinMs, + }; + throw new Error( + `topology orientation correctness failed: ${JSON.stringify(result.correctness)}; ` + + `latencies=${JSON.stringify(latencies)}`, + ); + } + const text = `${JSON.stringify(result, null, 2)}\n`; + if (out === undefined) process.stdout.write(text); + else { + fs.mkdirSync(path.dirname(out), { recursive: true }); + fs.writeFileSync(out, text); + process.stdout.write(`topology orientation report: ${out}\n`); + } +} finally { + await client.close().catch(() => undefined); + await topologyResident.close(); +} + +async function callTopology(client, request) { + const response = await client.callTool( + { name: "inspect_code_graph", arguments: request }, + undefined, + { timeout: 120_000 }, + ); + const payload = response.structuredContent; + if (payload === undefined || payload === null || typeof payload !== "object") { + throw new Error( + "topology orientation MCP call returned no structured content", + ); + } + return payload; +} + +function topologyPhaseRows(stderr) { + const prefix = "@samchon/graph: topology-phase="; + const lastNewline = stderr.lastIndexOf("\n"); + if (lastNewline === -1) return []; + return stderr + .slice(0, lastNewline + 1) + .split(/\r?\n/) + .filter((line) => line.startsWith(prefix)) + .map((line) => JSON.parse(line.slice(prefix.length))); +} + +async function waitForTopologyPhases(read, timeoutMs = 5_000) { + const deadline = performance.now() + timeoutMs; + for (;;) { + const rows = topologyPhaseRows(read()); + if ( + [ + ["pnpm-workspace", "model-query"], + ["pnpm-workspace", "tool-startup"], + ["pnpm-workspace", "normalization"], + ["repository-context", "join"], + ].every( + ([provider, phase]) => + rows.filter( + (row) => row.provider === provider && row.phase === phase, + ).length === 1, + ) + ) { + return rows; + } + if (performance.now() >= deadline) { + throw new Error( + `topology phase trace did not settle within ${String(timeoutMs)} ms: ${read()}`, + ); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +function onePhase(rows, provider, phase) { + const matches = rows.filter( + (row) => row.provider === provider && row.phase === phase, + ); + if ( + matches.length !== 1 || + typeof matches[0].durationMs !== "number" || + matches[0].durationMs < 0 + ) { + throw new Error( + `topology orientation expected one ${provider}/${phase} phase: ` + + JSON.stringify(matches), + ); + } + return matches[0]; +} + +function orientationInputDigest(root, files) { + const hash = createHash("sha256"); + for (const file of files) { + hash.update(file); + hash.update("\0"); + hash.update( + fs.readFileSync(path.join(root, file), "utf8").replace(/\r\n?/g, "\n"), + ); + hash.update("\0"); + } + return hash.digest("hex"); +} + +function factSetMatches(actual, expected) { + return ( + JSON.stringify( + actual + .map((value) => JSON.stringify(canonicalFact(value))) + .sort(compareText), + ) === + JSON.stringify( + expected + .map((value) => JSON.stringify(canonicalFact(value))) + .sort(compareText), + ) + ); +} + +function canonicalFact(value) { + if (Array.isArray(value)) return value.map(canonicalFact); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => compareText(left, right)) + .map(([key, child]) => [key, canonicalFact(child)]), + ); + } + return value; +} + +function directOrientation(root) { + const started = performance.now(); + const startupStarted = performance.now(); + const versioned = runPnpm(root, ["--version"]); + const toolStartupMs = Math.round(performance.now() - startupStarted); + if (versioned.status !== 0) { + throw new Error(`direct pnpm startup failed: ${versioned.stderr}`); + } + const modelStarted = performance.now(); + const listed = runPnpm(root, ["list", "-r", "--json", "--depth", "0"]); + const modelQueryMs = Math.round(performance.now() - modelStarted); + if (listed.status !== 0) { + throw new Error(`direct pnpm orientation failed: ${listed.stderr}`); + } + const packages = JSON.parse(listed.stdout); + if (!Array.isArray(packages)) { + throw new Error("direct pnpm orientation returned a non-array model"); + } + const manifestStarted = performance.now(); + const unique = [ + ...new Set( + packages.map((pkg) => + path.join(path.resolve(pkg.path), "package.json"), + ), + ), + ]; + for (const file of unique) JSON.parse(fs.readFileSync(file, "utf8")); + const manifestReadMs = Math.round(performance.now() - manifestStarted); + return { + shellCalls: 2, + rgCalls: 0, + directoryWalks: 0, + rawManifestReads: unique.length, + tool: "pnpm", + toolVersion: versioned.stdout.trim(), + toolStartupMs, + modelQueryMs, + manifestReadMs, + packages: packages.length, + elapsedMs: Math.round(performance.now() - started), + }; +} + +function runPnpm(root, args) { + return cp.spawnSync( + process.platform === "win32" ? "pnpm.cmd" : "pnpm", + args, + { + cwd: root, + encoding: "utf8", + windowsHide: true, + shell: process.platform === "win32", + }, + ); +} + +function compareText(left, right) { + return left < right ? -1 : 1; +} diff --git a/tests/experiment/src/rust-producer.mjs b/tests/experiment/src/rust-producer.mjs new file mode 100644 index 00000000..142e1688 --- /dev/null +++ b/tests/experiment/src/rust-producer.mjs @@ -0,0 +1,80 @@ +export const RUST_GRAPH_PRODUCER_UNIT_TEST = + "static_index::tests::graph_covers_trait_generic_async_and_macro_semantics"; +export const RUST_GRAPH_PRODUCER_SLOW_TEST = + "graph_snapshot_covers_semantic_breadth_and_build_universe"; + +/** Run the two exact acceptance fixtures and reject Cargo's zero-test success. */ +export function verifyRustGraphProducer({ + cargo, + producerRoot, + run, + emit = emitTestOutput, +}) { + const tests = [ + { + label: "Rust HIR unit fixture", + args: [ + "test", + "--locked", + "--release", + "-p", + "ide", + "--lib", + RUST_GRAPH_PRODUCER_UNIT_TEST, + "--", + "--exact", + ], + }, + { + label: "Rust HIR slow fixture", + args: [ + "test", + "--locked", + "--release", + "-p", + "rust-analyzer", + "--test", + "slow-tests", + RUST_GRAPH_PRODUCER_SLOW_TEST, + "--", + "--exact", + ], + env: { RUN_SLOW_TESTS: "1" }, + }, + ]; + for (const test of tests) { + const result = run(cargo, test.args, { + cwd: producerRoot, + stdio: "pipe", + check: false, + ...(test.env === undefined ? {} : { env: test.env }), + }); + const stdout = String(result.stdout ?? ""); + const stderr = String(result.stderr ?? ""); + emit(stdout, stderr); + if (result.status !== 0) { + const failure = + result.error instanceof Error + ? `could not start: ${result.error.message}` + : typeof result.signal === "string" && result.signal !== "" + ? `terminated by signal ${result.signal}` + : `exited with code ${String(result.status)}`; + throw new Error( + `${test.label} failed at the pinned producer commit: ${failure}`, + ); + } + const summaries = `${stdout}\n${stderr}`.match( + /(?:^|\r?\n)test result: ok\. 1 passed; 0 failed;/gu, + ); + if (summaries?.length !== 1) { + throw new Error( + `${test.label} did not run exactly one passing test at the pinned producer commit`, + ); + } + } +} + +function emitTestOutput(stdout, stderr) { + if (stdout !== "") process.stdout.write(stdout); + if (stderr !== "") process.stderr.write(stderr); +} diff --git a/tests/experiment/src/setup-language.mjs b/tests/experiment/src/setup-language.mjs index c16e5df7..404673f5 100644 --- a/tests/experiment/src/setup-language.mjs +++ b/tests/experiment/src/setup-language.mjs @@ -6,6 +6,11 @@ import os from "node:os"; import path from "node:path"; import { findExperiment } from "./catalog.mjs"; +import { + CLANG_PRODUCER_BUILD_PACKAGES, + installClangGraphProducer, +} from "./clang-producer.mjs"; +import { verifyGitTree } from "./git-tree.mjs"; import { appendGithubPath, ensureDir, @@ -19,6 +24,7 @@ import { resetToolManifest, workRoot, } from "./process.mjs"; +import { verifyRustGraphProducer } from "./rust-producer.mjs"; const args = parseArgs(process.argv.slice(2)); const experiment = findExperiment(args.language); @@ -313,21 +319,10 @@ const installScipRuby = () => "a068c7c3b2042b9eac563ce77ce35dcaca666b418530b1db9f932a3dbc7175dd", }); -// Indexes through the real compiler: it drives Gradle or Maven with the -// SemanticDB plugin injected, so the JVM lanes pay a build rather than skipping -// one. That is the trade — wall clock for facts the compiler itself produced — -// and it has to be measured rather than assumed. Kotlin support is upstream's -// own "less mature" than Java's, and Maven cannot index Kotlin at all; koin is -// Gradle, so it is on the supported path. -// scip-java v0.13.1 predates Kotlin 2.3's CompilerPluginRegistrar.pluginId -// contract, so it cannot index current Koin. Compiler plugins are also coupled -// to the compiler minor that loads them: #973's merged 2.4.0 tree builds but -// fails inside Koin's 2.3.20 compiler with NoClassDefFoundError. Pin the exact -// upstream #973 commit that completed the 2.3.20 port, before its next commit -// moved the plugin and fixture to 2.4.0. The source archive, compiler minor and -// fixture revision are then one reviewable generation instead of a local patch. -const SCIP_JAVA_KOTLIN_COMMIT = - "e940c1889767a81347387067a375320dc6f5d83e"; +// The Kotlin graph producer is a K2 compiler plugin injected into ordinary +// Kotlin/JVM Gradle compile tasks. Its compiler minor must match the project, +// so the experiment pins the exact fork revision and verifies both its +// artifact option and resident Tooling API protocol before indexing Koin. const SCIP_JAVA_KOTLIN_VERSION = "2.3.20"; /** @@ -336,21 +331,24 @@ const SCIP_JAVA_KOTLIN_VERSION = "2.3.20"; * Two rows need this and they need different revisions: Kotlin needs the * upstream commit that completed the 2.3.20 plugin port, and Java needs the * fork whose `index` command writes a graph artifact at all. The revision, its - * archive digest and the version string a run records are therefore arguments - * rather than constants — one builder, two pins, and no local patch on either. + * Git tree and the version string a run records are therefore arguments rather + * than constants. GitHub may repackage a generated source archive without + * changing its contents, so the immutable extracted tree is the security and + * reproducibility boundary: one builder, two pins, and no local patch on either. */ const installScipJavaSource = async (gradle, pin) => { const url = `https://codeload.github.com/${pin.repository}/tar.gz/${pin.commit}`; const archive = path.join(toolsRoot, `scip-java-${pin.commit}.tar.gz`); const source = path.join(toolsRoot, `scip-java-${pin.commit}`); await downloadFile(url, archive); - verifySha256(archive, pin.digest); fs.rmSync(source, { force: true, recursive: true }); ensureDir(source); run( "tar", ["-xzf", archive, "--strip-components=1", "-C", source], ); + verifyGitTree(source, pin.tree); + if (pin.verify !== undefined) pin.verify({ gradle, source }); run(gradle, ["--no-daemon", ":scip-java:installDist"], { cwd: source }); const launcher = path.join( source, @@ -373,19 +371,55 @@ const installScipJavaSource = async (gradle, pin) => { tool: "scip-java", version: pin.version, source: url, - digest: `sha256:${pin.digest}`, + digest: `git-tree:${pin.tree}`, }); return link; }; -const installScipJavaKotlinSnapshot = (gradle) => - installScipJavaSource(gradle, { - repository: "scip-code/scip-java", - commit: SCIP_JAVA_KOTLIN_COMMIT, - version: `${SCIP_JAVA_KOTLIN_COMMIT}+kotlin-${SCIP_JAVA_KOTLIN_VERSION}`, - digest: - "985eb03ef165864dbae3db4453d4566e699f78761bace3e4614bf67d38ce76cf", +const installScipJavaKotlinSnapshot = async (gradle) => { + if ( + typeof experiment.producerRepository !== "string" || + typeof experiment.producerCommit !== "string" || + typeof experiment.producerTree !== "string" + ) { + throw new Error( + "kotlin: the compiler graph setup requires an exact producer repository, commit, and tree", + ); + } + const repository = experiment.producerRepository + .replace(/^https:\/\/github\.com\//u, "") + .replace(/\.git$/u, ""); + const link = await installScipJavaSource(gradle, { + repository, + commit: experiment.producerCommit, + tree: experiment.producerTree, + version: `${experiment.producerCommit}+kotlin-${SCIP_JAVA_KOTLIN_VERSION}`, }); + const indexHelp = String( + run(link, ["index", "--help"], { stdio: "pipe" }).stdout, + ); + const serverHelp = String( + run(link, ["kotlin-graph-server", "--help"], { + stdio: "pipe", + }).stdout, + ); + if (!indexHelp.includes("--kotlin-graph-output")) { + throw new Error( + `kotlin: the installed scip-java does not publish --kotlin-graph-output:\n${indexHelp}`, + ); + } + if ( + !serverHelp.includes( + "Serve compiler-owned Kotlin graph generations over NDJSON.", + ) + ) { + throw new Error( + `kotlin: the installed scip-java does not publish the resident graph protocol:\n${serverHelp}`, + ); + } + process.env.SAMCHON_GRAPH_KOTLINC_GRAPH = link; + recordProvisionedEnvironment("SAMCHON_GRAPH_KOTLINC_GRAPH", link); +}; /** * The javac graph producer, built from the exact fork revision the consumer @@ -401,10 +435,11 @@ const installScipJavaKotlinSnapshot = (gradle) => const installJavacGraphProducer = async (gradle) => { if ( typeof experiment.producerRepository !== "string" || - typeof experiment.producerCommit !== "string" + typeof experiment.producerCommit !== "string" || + typeof experiment.producerTree !== "string" ) { throw new Error( - "java: the javac graph setup requires an exact producer repository and commit", + "java: the javac graph setup requires an exact producer repository, commit, and tree", ); } const repository = experiment.producerRepository @@ -413,9 +448,53 @@ const installJavacGraphProducer = async (gradle) => { const link = await installScipJavaSource(gradle, { repository, commit: experiment.producerCommit, + tree: experiment.producerTree, version: experiment.producerCommit, - digest: - "3ef45fedc5ad60ca6af0200a9b3fe7e978eadc8df63dda0a9dcba677f50b1417", + verify: ({ gradle: verifiedGradle, source }) => { + run( + verifiedGradle, + [ + ":scip-javac:test", + "--tests", + "org.scip_code.scip_java.javac.JavaGraphShardTest", + "--no-daemon", + "--no-configuration-cache", + ], + { cwd: source }, + ); + run( + verifiedGradle, + [ + ":scip-gradle-plugin:test", + "--tests", + "org.scip_code.scip_java.gradle.GraphGenerationStoreTest", + "--no-daemon", + "--no-configuration-cache", + ], + { cwd: source }, + ); + run( + verifiedGradle, + [ + ":scip-java:test", + "--tests", + "tests.GradleGraphLifecycleTest", + "--tests", + "tests.MavenGraphLifecycleTest", + "--tests", + "tests.MavenGraphPluginTest", + "--tests", + "tests.GraphAggregateRunnerTest", + "--tests", + "tests.GradleBuildToolTest", + "--no-daemon", + "--no-configuration-cache", + "-Pkotlin.compiler.execution.strategy=in-process", + "-Pkotlin.incremental=false", + ], + { cwd: source }, + ); + }, }); const help = String( run(link, ["index", "--help"], { stdio: "pipe" }).stdout, @@ -426,263 +505,97 @@ const installJavacGraphProducer = async (gradle) => { ${help}`, ); } + process.env.SAMCHON_GRAPH_JAVAC_GRAPH = link; + recordProvisionedEnvironment("SAMCHON_GRAPH_JAVAC_GRAPH", link); }; -// Needs a compilation database, which is why the provider carries -// `--compdb-path` and the corpus fixtures for redis and leveldb have to produce -// `compile_commands.json` before this can say anything. -const installScipClang = () => - installPinnedBinary({ - tool: "scip-clang", - version: "v0.4.0", - url: "https://github.com/sourcegraph/scip-clang/releases/download/v0.4.0/scip-clang-x86_64-linux", - digest: - "06fd18c576f979a726c651594644ec4a35db4f471f2160b3f72eb89fa6001784", - }); - -/** - * Accept an already-installed pinned producer, or report that there is none. - * - * Deliberately total: any missing file, any unreadable resource tree, any - * version string that does not name the pinned commit, and any error at all - * means "build it". Reuse is an optimisation, so it may only ever be taken - * when the evidence for it is complete. - */ -const installedClangGraphProducer = () => { - try { - const installed = path.join(binRoot, "samchon-clangd"); - const alias = path.join(binRoot, "clangd"); - if ( - !fs.statSync(installed, { throwIfNoEntry: false })?.isFile() || - !fs.statSync(alias, { throwIfNoEntry: false })?.isFile() - ) { - return false; - } - const resources = path.join(toolsRoot, "lib", "clang"); - const versions = fs - .readdirSync(resources, { withFileTypes: true }) - .filter( - (entry) => - entry.isDirectory() && - fs - .statSync(path.join(resources, entry.name, "include", "stddef.h"), { - throwIfNoEntry: false, - }) - ?.isFile(), - ); - if (versions.length !== 1) return false; - for (const binary of [installed, alias]) { - const reported = run(binary, ["--version"], { stdio: "pipe" }); - if (!String(reported.stdout).includes(experiment.producerCommit)) { - return false; - } +/** Build and install the exact JDT workspace graph producer revision. */ +const installJdtGraphProducer = async () => { + for (const field of [ + "jdtProducerRepository", + "jdtProducerCommit", + "jdtProducerTree", + ]) { + if (typeof experiment[field] !== "string" || experiment[field] === "") { + throw new Error(`java: the JDT graph setup requires an exact ${field}`); } - } catch { - return false; - } - record({ - tool: "samchon-clangd", - version: experiment.producerCommit, - source: `${experiment.producerRepository}@${experiment.producerCommit}`, - digest: `git:${experiment.producerCommit}`, - }); - record({ - tool: "clangd", - version: experiment.producerCommit, - source: "alias of samchon-clangd", - digest: `git:${experiment.producerCommit}`, - }); - return true; -}; - -const installClangGraphProducer = () => { - if ( - typeof experiment.producerRepository !== "string" || - typeof experiment.producerCommit !== "string" - ) { - throw new Error( - `${experiment.language}: native Clang setup requires an exact producer repository and commit`, - ); } - // The producer is a pinned commit, so its binary is a pure function of that - // commit and this toolchain. Rebuilding it on every push was the actual - // waste: roughly two CPU-hours per workflow to reproduce bytes that cannot - // have changed. A restored install is therefore reused rather than rebuilt — - // but only after it says, itself, that it is the pinned producer. A cache is - // untrusted input, and the same `--version` check the fresh build has to - // pass is what admits a restored one, so a stale or foreign artifact fails - // closed here instead of quietly indexing a corpus with the wrong compiler. - if (installedClangGraphProducer()) return; - const source = path.join(toolsRoot, "samchon-clangd-source"); - const build = path.join(source, "build"); + const repository = experiment.jdtProducerRepository + .replace(/^https:\/\/github\.com\//u, "") + .replace(/\.git$/u, ""); + const url = `https://codeload.github.com/${repository}/tar.gz/${experiment.jdtProducerCommit}`; + const archive = path.join( + toolsRoot, + `eclipse-jdt-ls-${experiment.jdtProducerCommit}.tar.gz`, + ); + const source = path.join( + toolsRoot, + `eclipse-jdt-ls-${experiment.jdtProducerCommit}`, + ); + await downloadFile(url, archive); fs.rmSync(source, { force: true, recursive: true }); ensureDir(source); - run("git", ["init", "--quiet"], { cwd: source }); - run("git", ["remote", "add", "origin", experiment.producerRepository], { + run("tar", ["-xzf", archive, "--strip-components=1", "-C", source]); + verifyGitTree(source, experiment.jdtProducerTree); + const maven = path.join(source, "mvnw"); + if (!fs.statSync(maven, { throwIfNoEntry: false })?.isFile()) { + throw new Error(`java: the pinned JDT Maven wrapper is missing at ${maven}`); + } + fs.chmodSync(maven, 0o755); + run(maven, ["clean", "install", "-U", "-DskipTests=true"], { cwd: source, }); run( - "git", - ["fetch", "--depth=1", "origin", experiment.producerCommit], + maven, + [ + "verify", + "-pl", + "org.eclipse.jdt.ls.tests", + "-am", + "-Dtest=GraphSnapshotCommandTest,UnresolvedTypesQuickFixTest#testTypeInSealedTypeDeclaration,FileEventHandlerTest,CleanUpsTest", + ], { cwd: source }, ); - run("git", ["checkout", "--detach", "FETCH_HEAD"], { cwd: source }); - const revision = String( - run("git", ["rev-parse", "HEAD"], { - cwd: source, - stdio: "pipe", - }).stdout, - ).trim(); - if (revision !== experiment.producerCommit) { - throw new Error( - `${experiment.language}: checked out native Clang ${revision}, expected ${experiment.producerCommit}`, - ); - } - run("cmake", [ - "-S", - path.join(source, "llvm"), - "-B", - build, - "-G", - "Ninja", - "-DCMAKE_BUILD_TYPE=Release", - "-DCMAKE_C_COMPILER=clang", - "-DCMAKE_CXX_COMPILER=clang++", - "-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra", - "-DLLVM_TARGETS_TO_BUILD=Native", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DLLVM_INCLUDE_TESTS=OFF", - "-DCLANG_INCLUDE_TESTS=OFF", - "-DLLVM_INCLUDE_BENCHMARKS=OFF", - "-DLLVM_INCLUDE_EXAMPLES=OFF", - `-DLLVM_FORCE_VC_REVISION=${experiment.producerCommit}`, - `-DLLVM_FORCE_VC_REPOSITORY=${experiment.producerRepository}`, - ]); - // Build with the machine, not with a number. Note what that is and is not - // claiming, because two earlier versions of this comment claimed more. - // - // Every recorded build of this producer, all at the advertised job count - // except the first: 2,431 of 3,125 steps in 85 minutes and killed unfinished - // at a fixed `2`; 2,431 steps in 81.2 minutes; a complete build in 56.1; a - // complete build in 107. The last two are the same commit and the same job - // count, in one workflow, on two runners. Hosted-runner performance varies - // by roughly a factor of two, which swamps the difference this line makes - // and leaves no clean two-against-four comparison in the data at all. - // - // So the reason for sizing by the machine is the principle, not a measured - // speedup: a constant that leaves half a runner idle is wrong wherever it - // runs, and the effect size here is unmeasured. An earlier comment reported - // "roughly half" and another "barely five percent"; both read a difference - // out of numbers that could not support one. - // - // Also bounded by installed memory, which is a machine-class bound and not - // an out-of-memory guard — worth being exact about, because the two are easy - // to confuse and only the first is what this computes. It reads total rather - // than free memory, so it says "this machine should not run more than N - // concurrent compiles", not "this machine has room right now". It bounds - // compile concurrency only; the `clangd` link is a single build edge that - // runs whatever this number is, and LLVM's own controls for that - // (`LLVM_PARALLEL_LINK_JOBS` and friends) are deliberately not set here - // because the runs that reached the link reached it without trouble, so - // there is nothing yet to size them against. Two GiB per compile - // job is this repository's figure, chosen as a conventional one; it is not - // quoted from LLVM. - // - // Logged because it is otherwise invisible. Ninja does not print its job - // count and `run` does not echo argv, so a machine whose memory quietly - // halves the count would look exactly like a slow build, which is the - // confusion that cost this lane two CI runs already. - const jobs = Math.max( - 1, - Math.min( - os.availableParallelism(), - Math.floor(os.totalmem() / (2 * 1024 * 1024 * 1024)), - ), - ); - console.log( - `${experiment.language}: building the pinned Clang producer with ${String(jobs)} jobs ` + - `(cores ${String(os.availableParallelism())}, ` + - `memory ${String(Math.round(os.totalmem() / (1024 * 1024 * 1024)))} GiB)`, - ); - run("cmake", [ - "--build", - build, - "--parallel", - String(jobs), - "--target", - "clangd", - ]); - const binary = path.join(build, "bin", "clangd"); - const version = String( - run(binary, ["--version"], { stdio: "pipe" }).stdout, - ); - if (!version.includes(experiment.producerCommit)) { - throw new Error( - `${experiment.language}: native Clang version omits ${experiment.producerCommit}:\n${version}`, - ); - } - const builtResources = path.join(build, "lib", "clang"); - const resourceVersions = fs - .readdirSync(builtResources, { withFileTypes: true }) - .filter( - (entry) => - entry.isDirectory() && - fs.statSync( - path.join(builtResources, entry.name, "include"), - { throwIfNoEntry: false }, - )?.isDirectory(), - ) - .map((entry) => entry.name); - if (resourceVersions.length !== 1) { - throw new Error( - `${experiment.language}: native Clang produced ${resourceVersions.length} resource-header trees`, - ); - } - const installedResources = path.join(toolsRoot, "lib", "clang"); - fs.rmSync(installedResources, { force: true, recursive: true }); - ensureDir(path.dirname(installedResources)); - fs.cpSync(builtResources, installedResources, { recursive: true }); - const installedStddef = path.join( - installedResources, - resourceVersions[0], - "include", - "stddef.h", + const launcher = path.join( + source, + "org.eclipse.jdt.ls.product", + "target", + "repository", + "bin", + "jdtls", ); - if (!fs.statSync(installedStddef, { throwIfNoEntry: false })?.isFile()) { - throw new Error( - `${experiment.language}: native Clang resource headers were not installed at ${installedStddef}`, - ); + if (!fs.statSync(launcher, { throwIfNoEntry: false })?.isFile()) { + throw new Error(`java: the pinned JDT launcher is missing at ${launcher}`); } - for (const command of ["samchon-clangd", "clangd"]) { - const link = path.join(binRoot, command); + fs.chmodSync(launcher, 0o755); + const dedicated = path.join(binRoot, "samchon-jdtls"); + const generic = path.join(binRoot, "jdtls"); + for (const link of [dedicated, generic]) { fs.rmSync(link, { force: true }); - fs.linkSync(binary, link); - } - const installedVersion = String( - run(path.join(binRoot, "samchon-clangd"), ["--version"], { - stdio: "pipe", - }).stdout, - ); - if (!installedVersion.includes(experiment.producerCommit)) { - throw new Error( - `${experiment.language}: installed native Clang omits ${experiment.producerCommit}:\n${installedVersion}`, - ); + fs.symlinkSync(launcher, link); } + process.env.SAMCHON_GRAPH_JDT_WORKSPACE = dedicated; + recordProvisionedEnvironment("SAMCHON_GRAPH_JDT_WORKSPACE", dedicated); record({ - tool: "samchon-clangd", - version: experiment.producerCommit, - source: `${experiment.producerRepository}@${experiment.producerCommit}`, - digest: `git:${experiment.producerCommit}`, - }); - record({ - tool: "clangd", - version: experiment.producerCommit, - source: "alias of samchon-clangd", - digest: `git:${experiment.producerCommit}`, + tool: "eclipse-jdtls-graph-snapshot", + version: experiment.jdtProducerCommit, + source: url, + digest: `git-tree:${experiment.jdtProducerTree}`, }); - fs.rmSync(source, { force: true, recursive: true }); }; +// Needs a compilation database, which is why the provider carries +// `--compdb-path` and the corpus fixtures for redis and leveldb have to produce +// `compile_commands.json` before this can say anything. +const installScipClang = () => + installPinnedBinary({ + tool: "scip-clang", + version: "v0.4.0", + url: "https://github.com/sourcegraph/scip-clang/releases/download/v0.4.0/scip-clang-x86_64-linux", + digest: + "06fd18c576f979a726c651594644ec4a35db4f471f2160b3f72eb89fa6001784", + }); + // The published tarball is a webpack bundle whose only runtime `require`s are // Node built-ins, so extracting the integrity-verified archive installs exactly // the bytes the digest covers. `npm install` would instead resolve the package's @@ -887,11 +800,14 @@ switch (experiment.language) { `rust producer checkout is ${producerHead}, not ${experiment.producerCommit}`, ); } - run( - path.join(cargoBin, process.platform === "win32" ? "cargo.exe" : "cargo"), - ["build", "--locked", "--release", "-p", "rust-analyzer"], - { cwd: producerRoot }, + const cargo = path.join( + cargoBin, + process.platform === "win32" ? "cargo.exe" : "cargo", ); + verifyRustGraphProducer({ cargo, producerRoot, run }); + run(cargo, ["build", "--locked", "--release", "-p", "rust-analyzer"], { + cwd: producerRoot, + }); const producerBinary = path.join( producerRoot, "target", @@ -938,8 +854,21 @@ switch (experiment.language) { // pinned corpora are CMake and emit one themselves, so nothing here uses it // today; it stays because the database is what makes this route selectable // at all, and a Makefile corpus would otherwise be unable to produce one. - apt(["clang", "cmake", "ninja-build", "bear"]); - installClangGraphProducer(); + const allowClangProducerBuild = + process.env.SAMCHON_GRAPH_CLANG_PRODUCER_ALLOW_BUILD !== "0"; + apt([ + ...(allowClangProducerBuild ? CLANG_PRODUCER_BUILD_PACKAGES : []), + "bear", + ]); + installClangGraphProducer({ + language: experiment.language, + toolsRoot, + binRoot, + producerRepository: experiment.producerRepository, + producerCommit: experiment.producerCommit, + record, + allowBuild: allowClangProducerBuild, + }); record({ tool: "bear", version: "unpinned", @@ -950,10 +879,9 @@ switch (experiment.language) { await installScip(); break; case "java": { - // jdtls is not an apt package and requires Java 21+; install the JDK and the - // Eclipse JDT.LS snapshot tarball, then put its bin on PATH. The launcher is - // a Python script that locates its plugins relative to its own path, so it - // must run from the extracted tree rather than a symlink. + // Both compiler-owned lanes are built from exact source archives. JDT.LS + // needs Java 21+ and its launcher is a Python script that locates plugins + // relative to the built product repository. apt(["openjdk-21-jdk", "python3"]); // jdtls crashes on the runner's default JDK; point it at Java 21. const javaHome = "/usr/lib/jvm/java-21-openjdk-amd64"; @@ -963,23 +891,6 @@ switch (experiment.language) { fs.appendFileSync(process.env.GITHUB_ENV, `JAVA_HOME=${javaHome}${os.EOL}`); } appendGithubPath(path.join(javaHome, "bin")); - const target = path.join(toolsRoot, "jdtls"); - const archive = path.join(toolsRoot, "jdtls.tar.gz"); - await downloadFile( - "https://download.eclipse.org/jdtls/snapshots/jdt-language-server-latest.tar.gz", - archive, - ); - fs.rmSync(target, { force: true, recursive: true }); - ensureDir(target); - run("tar", ["-xzf", archive, "-C", target]); - appendGithubPath(path.join(target, "bin")); - record({ - tool: "jdtls", - version: "unpinned", - source: - "https://download.eclipse.org/jdtls/snapshots/jdt-language-server-latest.tar.gz", - digest: "unpinned", - }); // One launcher, built from the pinned fork. It serves both the strict // javac route and the SCIP lane behind it, so installing the released // binary first only downloaded a `scip-java` the source build then @@ -987,6 +898,7 @@ switch (experiment.language) { // though a run had used it. await installScip(); await installJavacGraphProducer(await installGradle()); + await installJdtGraphProducer(); break; } case "csharp": { @@ -1013,9 +925,36 @@ switch (experiment.language) { source: "dotnet tool install --global csharp-ls", digest: "unpinned", }); - // The strict C# producer, built on Roslyn like csharp-ls but reading the - // solution once instead of answering a request per symbol. Installed as a - // global dotnet tool, so the SDK above is its only prerequisite. + const producerRoot = path.join(toolsRoot, "samchon-roslyn"); + fs.rmSync(producerRoot, { force: true, recursive: true }); + run(dotnet, [ + "publish", + path.join(repositoryRoot, "sidecars", "csharp", "Samchon.Graph.CSharp.csproj"), + "--configuration", + "Release", + "--output", + producerRoot, + "--no-self-contained", + "-p:RestoreLockedMode=true", + ]); + const producer = path.join(producerRoot, "samchon-roslyn"); + fs.chmodSync(producer, 0o755); + const producerLink = path.join(binRoot, "samchon-roslyn"); + fs.rmSync(producerLink, { force: true }); + fs.symlinkSync(producer, producerLink); + process.env.SAMCHON_GRAPH_ROSLYN_WORKSPACE = producerLink; + recordProvisionedEnvironment( + "SAMCHON_GRAPH_ROSLYN_WORKSPACE", + producerLink, + ); + record({ + tool: "samchon-roslyn", + version: "workspace", + source: "sidecars/csharp", + digest: "built-from-locked-source", + }); + // Keep scip-dotnet installed only for the registered optional navigation + // fallback; the Roslyn workspace service above owns strict C# facts. shell(`"${dotnet}" tool install --global scip-dotnet || "${dotnet}" tool update --global scip-dotnet`); record({ tool: "scip-dotnet", @@ -1029,12 +968,10 @@ switch (experiment.language) { case "kotlin": await installKotlinLanguageServer(); const gradle = await installGradle(); - // scip-java covers Kotlin through semanticdb-kotlinc, and it needs a JDK to - // run the Gradle build it indexes through. koin is the worst lane measured - // at 1349 s, almost all of it kotlin-language-server's Gradle sync before it - // answers `initialize` at all. The strict path does not skip that build; it - // performs one. Whether that is faster, slower, or merely truer is the thing - // to find out. + // The strict lane launches one persistent Gradle Tooling connection and + // drives the project's ordinary Kotlin/JVM compile task. KGP therefore + // retains its daemon, configuration, classpath and incremental caches + // across lifecycle requests while the compiler plugin owns every fact. apt(["openjdk-21-jdk"]); const javaHome = "/usr/lib/jvm/java-21-openjdk-amd64"; process.env.JAVA_HOME = javaHome; @@ -1049,7 +986,7 @@ switch (experiment.language) { await installScipJavaKotlinSnapshot(gradle); await installScip(); break; - case "swift": + case "swift": { // sourcekit-lsp ships with the toolchain installed by the workflow's Setup // Swift step. It has no `--version` flag (that exits 64), so just confirm it // resolves on PATH. @@ -1060,21 +997,160 @@ switch (experiment.language) { source: "swift toolchain installed by the workflow", digest: "unpinned", }); + const sidecar = path.join(repositoryRoot, "sidecars", "swift"); + const swift = String(run("which", ["swift"], { stdio: "pipe" }).stdout).trim(); + const swiftRoot = path.dirname(path.dirname(swift)); + const swiftBuildArguments = [ + "build", + "--package-path", + sidecar, + "--configuration", + "release", + ...(process.platform === "linux" + ? [ + "-Xcxx", + `-I${path.join(swiftRoot, "lib", "swift")}`, + "-Xcxx", + `-I${path.join(swiftRoot, "lib", "swift", "Block")}`, + ] + : []), + ]; + run("swift", swiftBuildArguments); + const sidecarBin = String( + run( + "swift", + ["build", "--package-path", sidecar, "--show-bin-path", "--configuration", "release"], + { stdio: "pipe" }, + ).stdout, + ).trim(); + const producer = path.join(sidecarBin, "samchon-swift-graph"); + run(producer, ["--version"]); + process.env.SAMCHON_GRAPH_SWIFT_GRAPH = producer; + process.env.SAMCHON_GRAPH_SWIFT_TOOLCHAIN = swift; + recordProvisionedEnvironment("SAMCHON_GRAPH_SWIFT_GRAPH", producer); + recordProvisionedEnvironment("SAMCHON_GRAPH_SWIFT_TOOLCHAIN", swift); + record({ + tool: "samchon-swift-graph", + version: "0.1.0", + source: "sidecars/swift", + digest: "built-from-workspace-source:indexstore-db-54212fce1aecb199070808bdb265e7f17e396015", + }); break; - case "scala": - apt(["openjdk-17-jdk", "gzip"]); - await downloadFile("https://github.com/coursier/coursier/releases/latest/download/cs-x86_64-pc-linux.gz", path.join(toolsRoot, "cs.gz")); - shell(`gzip -dc "${path.join(toolsRoot, "cs.gz")}" > "${path.join(binRoot, "cs")}"`); - shell(`chmod +x "${path.join(binRoot, "cs")}"`); - run(path.join(binRoot, "cs"), ["install", "metals"]); - appendGithubPath(path.join(os.homedir(), ".local", "share", "coursier", "bin")); + } + case "scala": { + apt(["openjdk-21-jdk", "maven"]); + const javaHome = "/usr/lib/jvm/java-21-openjdk-amd64"; + const java = path.join(javaHome, "bin", "java"); + process.env.JAVA_HOME = javaHome; + recordProvisionedEnvironment("JAVA_HOME", javaHome); + if (process.env.GITHUB_ENV !== undefined) { + fs.appendFileSync( + process.env.GITHUB_ENV, + `JAVA_HOME=${javaHome}${os.EOL}`, + ); + } + appendGithubPath(path.join(javaHome, "bin")); + + run("mvn", [ + "--batch-mode", + "--file", + path.join(repositoryRoot, "sidecars", "scala", "pom.xml"), + "verify", + ]); + const sidecarRoot = path.join(repositoryRoot, "sidecars", "scala"); + const version = "0.1.0-SNAPSHOT"; + const scala2Plugin = path.join( + sidecarRoot, + "scala2-plugin", + "target", + `scala-graph-plugin_2.13.18-${version}.jar`, + ); + const scala3Plugin = path.join( + sidecarRoot, + "scala3-plugin", + "target", + `scala-graph-plugin_3.9.0-${version}.jar`, + ); + const server = path.join( + sidecarRoot, + "server", + "target", + `samchon-scala-graph-${version}.jar`, + ); + for (const artifact of [scala2Plugin, scala3Plugin, server]) { + if (!fs.statSync(artifact, { throwIfNoEntry: false })?.isFile()) { + throw new Error(`Scala graph build omitted ${artifact}`); + } + } + const producer = path.join(binRoot, "samchon-scala-graph"); + fs.writeFileSync( + producer, + `#!/bin/sh\nexec '${java}' -jar '${server}' "$@"\n`, + ); + fs.chmodSync(producer, 0o755); + run(producer, ["--version"]); + const serverHelp = String( + run(producer, ["graph-server", "--help"], { stdio: "pipe" }).stdout, + ); + if ( + !serverHelp.includes( + "Serve BSP-driven Scala compiler graph generations over NDJSON.", + ) + ) { + throw new Error( + `scala: the built producer does not publish the resident graph protocol:\n${serverHelp}`, + ); + } + + const sbtVersion = "1.11.7"; + const sbtJar = path.join(toolsRoot, `sbt-launch-${sbtVersion}.jar`); + const sbtUrl = `https://repo.maven.apache.org/maven2/org/scala-sbt/sbt-launch/${sbtVersion}/sbt-launch-${sbtVersion}.jar`; + await downloadFile(sbtUrl, sbtJar); + verifySha256( + sbtJar, + "f92a2095ac75008764fe3b2b793ffe624c4fbef5bfd9b0022e4bc2daf668c651", + ); + const sbt = path.join(binRoot, "sbt"); + fs.writeFileSync(sbt, `#!/bin/sh\nexec '${java}' -jar '${sbtJar}' "$@"\n`); + fs.chmodSync(sbt, 0o755); + + for (const [name, value] of Object.entries({ + SAMCHON_GRAPH_SCALA_GRAPH: producer, + SAMCHON_GRAPH_SCALA2_PLUGIN: scala2Plugin, + SAMCHON_GRAPH_SCALA3_PLUGIN: scala3Plugin, + SAMCHON_GRAPH_SCALA_PLUGIN_VERSION: version, + SAMCHON_GRAPH_JAVA_TOOLCHAIN: java, + })) { + process.env[name] = value; + recordProvisionedEnvironment(name, value); + } + record({ + tool: "samchon-scala-graph", + version, + source: "sidecars/scala", + digest: "built-from-workspace-source", + }); record({ - tool: "metals", + tool: "scalac-graph plugins", + version: "Scala 2.13.18; Scala 3.9.0", + source: "sidecars/scala", + digest: "built-from-workspace-source", + }); + record({ + tool: "sbt", + version: sbtVersion, + source: sbtUrl, + digest: + "sha256:f92a2095ac75008764fe3b2b793ffe624c4fbef5bfd9b0022e4bc2daf668c651", + }); + record({ + tool: "maven", version: "unpinned", - source: "coursier install metals", + source: "apt maven", digest: "unpinned", }); break; + } case "zig": await installZls(); break; diff --git a/tests/experiment/src/strict-lifecycle.mjs b/tests/experiment/src/strict-lifecycle.mjs index 7a868245..81695150 100644 --- a/tests/experiment/src/strict-lifecycle.mjs +++ b/tests/experiment/src/strict-lifecycle.mjs @@ -2,9 +2,20 @@ import fs from "node:fs"; import path from "node:path"; import { createResidentGraphSource } from "@samchon/graph"; +import { LspClient } from "../../../packages/graph/lib/lsp/LspClient.js"; +import { measureClangBackgroundIndex } from "./clang-background-baseline.mjs"; import { compilationDatabaseLifecycle } from "./compilation-database-lifecycle.mjs"; +import { + measureLifecycleNoopPerformance, + measureLifecyclePerformance, +} from "./lifecycle-performance.mjs"; +import { captureKotlinBuildReport } from "./kotlin-build-report.mjs"; import { isolateCorpus, shell } from "./process.mjs"; +import { + captureGenerationEvidence, + firstEvidenceDifference, +} from "./regeneration-evidence.mjs"; /** Measure one strict provider without ever editing the pinned corpus clone. */ export const runStrictLifecycle = async (experiment, pinnedRoot) => { @@ -35,6 +46,76 @@ export const runStrictLifecycle = async (experiment, pinnedRoot) => { const buildText = fs.readFileSync(buildFile, "utf8"); const failureText = fs.readFileSync(failureFile, "utf8"); const rows = []; + if (experiment.nativeBaseline !== undefined) { + const baselineRoot = isolateCorpus( + experiment, + pinnedRoot, + "native-baseline", + ); + if (experiment.prepare !== undefined) { + shell(experiment.prepare, { cwd: baselineRoot }); + } + let nativeElapsedMs; + if (typeof experiment.nativeBaseline === "string") { + const started = performance.now(); + shell(experiment.nativeBaseline, { cwd: baselineRoot }); + nativeElapsedMs = Math.round(performance.now() - started); + } else if (experiment.nativeBaseline.kind === "clang-background-index") { + nativeElapsedMs = await measureClangBackgroundIndex({ + command: experiment.nativeBaseline.command, + compilationDatabase: path.join( + baselineRoot, + fixture.compilationDatabase, + ), + cwd: baselineRoot, + language: experiment.language, + sourceFile: path.join(baselineRoot, fixture.sourceFile), + timeoutMs: experiment.readyTimeoutMs ?? 180_000, + createClient: (command, commandArgs) => + new LspClient( + command, + commandArgs, + experiment.readyTimeoutMs ?? 180_000, + baselineRoot, + ), + }); + } else if (experiment.nativeBaseline.kind === "shell") { + if (experiment.nativeBaseline.warmup === true) { + shell(experiment.nativeBaseline.command, { cwd: baselineRoot }); + } + for (const relative of experiment.nativeBaseline.clean ?? []) { + const target = path.resolve(baselineRoot, relative); + if ( + target === baselineRoot || + !target.startsWith(`${baselineRoot}${path.sep}`) + ) { + throw new Error( + `${experiment.language}: native baseline cleanup escapes its corpus`, + ); + } + fs.rmSync(target, { force: true, recursive: true }); + } + const started = performance.now(); + shell(experiment.nativeBaseline.command, { cwd: baselineRoot }); + nativeElapsedMs = Math.round(performance.now() - started); + } else { + throw new Error( + `${experiment.language}: unknown native baseline ${String(experiment.nativeBaseline.kind)}`, + ); + } + rows.push({ + name: "native-baseline", + status: "passed", + command: + typeof experiment.nativeBaseline === "string" + ? experiment.nativeBaseline + : experiment.nativeBaseline.kind === "clang-background-index" + ? `${experiment.nativeBaseline.command} --background-index` + : experiment.nativeBaseline.command, + project: baselineRoot, + elapsedMs: nativeElapsedMs, + }); + } const resident = createResidentGraphSource({ cwd: lifecycleRoot, mode: "lsp", @@ -77,6 +158,14 @@ export const runStrictLifecycle = async (experiment, pinnedRoot) => { nodeCount: next.nodes.length, edgeCount: next.edges.length, diagnosticCount: next.diagnostics?.length ?? 0, + ...(fixture.kotlinBuildReportRoot === undefined || mode === "unchanged" + ? {} + : { + kotlinBuildReport: captureKotlinBuildReport( + lifecycleRoot, + fixture.kotlinBuildReportRoot, + ), + }), }; if (name === "unchanged" && identity !== previousIdentity) { throw new Error( @@ -102,6 +191,13 @@ export const runStrictLifecycle = async (experiment, pinnedRoot) => { try { const cold = await load("cold", ["initial"]); + const coldRegenerationEvidence = + fixture.regenerationEvidenceRoot === undefined + ? undefined + : captureGenerationEvidence( + lifecycleRoot, + fixture.regenerationEvidenceRoot, + ); const unchanged = await load("unchanged", ["unchanged"]); if (cold !== unchanged) { throw new Error( @@ -109,6 +205,74 @@ export const runStrictLifecycle = async (experiment, pinnedRoot) => { ); } + if (fixture.noopPerformance !== undefined) { + rows.push( + await measureLifecycleNoopPerformance({ + language: experiment.language, + ...fixture.noopPerformance, + currentDump: dump, + currentIdentity: previousIdentity, + load: async () => { + const started = performance.now(); + const next = await resident.load(); + const provenance = strictProvenance(next, experiment); + return { + dump: next, + mode: resident.modes().get(experiment.strictProvider), + identity: [ + provenance.manifest, + provenance.content, + provenance.universe, + ].join(":"), + elapsedMs: Math.round(performance.now() - started), + }; + }, + }), + ); + } + + if (fixture.performance !== undefined) { + const measured = await measureLifecyclePerformance({ + language: experiment.language, + ...fixture.performance, + sourceText, + currentDump: dump, + currentIdentity: previousIdentity, + describeDifference: firstGenerationDifference, + changedModes: CHANGED_MODES, + writeSource: (text) => fs.writeFileSync(sourceFile, text), + ...(fixture.kotlinBuildReportRoot === undefined + ? {} + : { + captureEditEvidence: () => + captureKotlinBuildReport( + lifecycleRoot, + fixture.kotlinBuildReportRoot, + ), + }), + load: async () => { + const started = performance.now(); + const next = await resident.load(); + const provenance = strictProvenance(next, experiment); + return { + dump: next, + mode: resident.modes().get(experiment.strictProvider), + identity: [ + provenance.manifest, + provenance.content, + provenance.universe, + ].join(":"), + elapsedMs: Math.round(performance.now() - started), + }; + }, + }); + dump = measured.dump; + previousIdentity = measured.identity; + previousProvenance = strictProvenance(dump, experiment); + previousDiagnostics = dump.diagnostics?.length ?? 0; + rows.push(measured.row); + } + fs.writeFileSync(sourceFile, sourceText + fixture.editSuffix); await load("edit", CHANGED_MODES); @@ -169,7 +333,10 @@ export const runStrictLifecycle = async (experiment, pinnedRoot) => { ); } - fs.writeFileSync(buildFile, `${buildText}\n`); + fs.writeFileSync( + buildFile, + `${buildText}${fixture.buildEditSuffix ?? "\n"}`, + ); await load("build-config", CHANGED_MODES); const failedAt = performance.now(); @@ -485,6 +652,13 @@ export const runStrictLifecycle = async (experiment, pinnedRoot) => { } const coldProvenance = strictProvenance(cold, experiment); const retryProvenance = strictProvenance(retried, experiment); + const retryRegenerationEvidence = + fixture.regenerationEvidenceRoot === undefined + ? undefined + : captureGenerationEvidence( + lifecycleRoot, + fixture.regenerationEvidenceRoot, + ); // Restoring the sources restores the generation, or the row says why not. // // This was written as two claims, on the theory that a source manifest is a @@ -507,13 +681,17 @@ export const runStrictLifecycle = async (experiment, pinnedRoot) => { const limitation = experiment.regenerationLimitation; if (!reproduced && limitation === undefined) { const difference = firstGenerationDifference(cold, retried); + const producerDifference = firstEvidenceDifference( + coldRegenerationEvidence, + retryRegenerationEvidence, + ); throw new Error( `${experiment.language}: restoring the original sources did not reproduce the generation ` + `(manifest ${reproducedManifest ? "unchanged" : "moved"}, ` + `facts ${reproducedContent ? "unchanged" : "moved"}; ` + `cold ${String(cold.nodes.length)} nodes/${String(cold.edges.length)} edges, ` + `retry ${String(retried.nodes.length)} nodes/${String(retried.edges.length)} edges; ` + - `first difference: ${difference})`, + `first difference: ${difference}; producer evidence: ${producerDifference})`, ); } rows.push({ diff --git a/tests/experiment/src/topology-orientation-oracle.mjs b/tests/experiment/src/topology-orientation-oracle.mjs new file mode 100644 index 00000000..d4cf64d7 --- /dev/null +++ b/tests/experiment/src/topology-orientation-oracle.mjs @@ -0,0 +1,239 @@ +const INPUT_FILES = [ + "package.json", + "pnpm-workspace.yaml", + "config/package.json", + "packages/graph-sitter/package.json", + "packages/graph/package.json", + "tests/benchmark/package.json", + "tests/experiment/package.json", + "tests/test-graph/package.json", +]; + +/** + * Manually reviewed facts for the pinned repository-orientation question. + * + * The input digest makes this oracle fail closed when an owning manifest moves; + * update the declarations and digest together after reviewing that change. + */ +export const TOPOLOGY_ORIENTATION_ORACLE = { + inputFiles: INPUT_FILES, + inputDigest: "d9db1c54b7caf4dc9f40566b459ed35efd8147209218f89955a530adcbdf17ed", + nodes: [ + workspaceNode(), + packageNode("@samchon/graph-workspace", ".", "package.json"), + packageNode("@samchon/graph-config", "config", "config/package.json"), + packageNode( + "@samchon/graph-sitter", + "packages/graph-sitter", + "packages/graph-sitter/package.json", + ), + packageNode( + "@samchon/graph", + "packages/graph", + "packages/graph/package.json", + ), + packageNode( + "@samchon/graph-benchmark", + "tests/benchmark", + "tests/benchmark/package.json", + ), + packageNode( + "@samchon/graph-experiment", + "tests/experiment", + "tests/experiment/package.json", + ), + packageNode( + "@samchon/graph-test", + "tests/test-graph", + "tests/test-graph/package.json", + ), + rootNode( + "generated-root", + "lib", + "packages/graph-sitter/lib", + "packages/graph-sitter/package.json", + ), + rootNode( + "source-root", + "src", + "packages/graph-sitter/src", + "packages/graph-sitter/package.json", + ), + rootNode( + "generated-root", + "lib", + "packages/graph/lib", + "packages/graph/package.json", + ), + rootNode( + "source-root", + "sidecars", + "packages/graph/sidecars", + "packages/graph/package.json", + ), + rootNode( + "source-root", + "src", + "packages/graph/src", + "packages/graph/package.json", + ), + entrypointNode( + "config", + "exports:./lint:default", + "config/lint.config.ts", + "config/package.json", + ), + entrypointNode( + "config", + "exports:./lint:types", + "config/lint.config.ts", + "config/package.json", + ), + entrypointNode( + "config", + "exports:./package.json", + "config/package.json", + "config/package.json", + ), + entrypointNode( + "config", + "exports:./tsconfig", + "config/tsconfig.json", + "config/package.json", + ), + entrypointNode( + "packages/graph", + "bin:samchon-graph", + "packages/graph/lib/bin.js", + "packages/graph/package.json", + ), + entrypointNode( + "packages/graph", + "exports:./package.json", + "packages/graph/package.json", + "packages/graph/package.json", + ), + entrypointNode( + "packages/graph", + "exports:.:default", + "packages/graph/lib/index.js", + "packages/graph/package.json", + ), + entrypointNode( + "packages/graph", + "exports:.:types", + "packages/graph/lib/index.d.ts", + "packages/graph/package.json", + ), + entrypointNode( + "packages/graph", + "main", + "packages/graph/lib/index.js", + "packages/graph/package.json", + ), + entrypointNode( + "packages/graph", + "types", + "packages/graph/lib/index.d.ts", + "packages/graph/package.json", + ), + entrypointNode( + "packages/graph-sitter", + "exports:./package.json", + "packages/graph-sitter/package.json", + "packages/graph-sitter/package.json", + ), + entrypointNode( + "packages/graph-sitter", + "exports:.:default", + "packages/graph-sitter/lib/index.js", + "packages/graph-sitter/package.json", + ), + entrypointNode( + "packages/graph-sitter", + "exports:.:types", + "packages/graph-sitter/lib/index.d.ts", + "packages/graph-sitter/package.json", + ), + entrypointNode( + "packages/graph-sitter", + "main", + "packages/graph-sitter/lib/index.js", + "packages/graph-sitter/package.json", + ), + entrypointNode( + "packages/graph-sitter", + "types", + "packages/graph-sitter/lib/index.d.ts", + "packages/graph-sitter/package.json", + ), + ], + dependencies: [ + dependency("@samchon/graph", "@samchon/graph-sitter"), + dependency("@samchon/graph-benchmark", "@samchon/graph"), + dependency("@samchon/graph-experiment", "@samchon/graph"), + dependency("@samchon/graph-test", "@samchon/graph"), + dependency("@samchon/graph-test", "@samchon/graph-sitter"), + ], +}; + +function workspaceNode() { + return { + ...node("workspace", ".", "tool-resolved"), + name: "compiler-knowledge-graph", + evidence: evidence("pnpm-workspace.yaml"), + }; +} + +function packageNode(name, coordinate, manifest) { + return { + ...node("package", name, "declared"), + name, + coordinate, + evidence: evidence(manifest), + }; +} + +function rootNode(kind, name, root, manifest) { + return { + ...node(kind, root, "declared"), + name, + root, + evidence: evidence(manifest), + }; +} + +function entrypointNode(packageCoordinate, name, file, manifest) { + const coordinate = `${packageCoordinate}:${name}`; + return { + ...node("entrypoint", coordinate, "declared"), + name, + file, + evidence: evidence(manifest), + }; +} + +function node(kind, identity, authority) { + return { + id: `repository://pnpm/default/${kind}/${encodeURIComponent(identity)}`, + authority, + kind, + ecosystem: "pnpm", + coordinate: identity, + configuration: "default", + external: false, + }; +} + +function evidence(file) { + return { file, startLine: 1, startColumn: 1 }; +} + +function dependency(from, to) { + return { + authority: "tool-resolved", + kind: "depends-on", + from: `repository://pnpm/default/package/${encodeURIComponent(from)}`, + to: `repository://pnpm/default/package/${encodeURIComponent(to)}`, + }; +} diff --git a/tests/test-graph/src/features/test_cli_dump_prints_graph_json.ts b/tests/test-graph/src/features/test_cli_dump_prints_graph_json.ts index 80cd5209..8dc93f1c 100644 --- a/tests/test-graph/src/features/test_cli_dump_prints_graph_json.ts +++ b/tests/test-graph/src/features/test_cli_dump_prints_graph_json.ts @@ -3,8 +3,11 @@ import { execFileSync, spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; +import type { ISamchonGraphDump } from "@samchon/graph"; +import { routeSummary } from "../../../../packages/graph/src/routeSummary"; import { GraphFixtures } from "../internal/GraphFixtures"; import { GraphPaths } from "../internal/GraphPaths"; +import { waitForProcessId } from "../internal/waitForProcessId"; export const test_cli_dump_prints_graph_json = async () => { const root = GraphFixtures.createOrderFixture(); @@ -19,12 +22,68 @@ export const test_cli_dump_prints_graph_json = async () => { const dump = JSON.parse(output); TestValidator.equals("CLI dump indexer", dump.indexer, "static"); TestValidator.predicate("CLI dump has nodes", dump.nodes.length > 0); + assertStructuredRouteProvenance(dump); assertTheDumpSaysWhatProducedIt(root); assertPrefixedWarningsRemainSingle(); await assertTimedOutDumpRetiresItsLanguageServer(); }; +function assertStructuredRouteProvenance(dump: Record): void { + const digest = "a".repeat(64); + const enriched = { + ...dump, + provenance: [ + { + provider: "fixture-provider", + languages: ["typescript"], + authority: "compiler", + facts: ["calls"], + capabilities: ["fixture"], + producer: { + tool: "fixture-compiler", + version: "1.2.3", + compiler: "TypeScript 6", + schemaVersion: 2, + protocolVersion: 1, + }, + universe: digest, + manifest: digest, + content: digest, + }, + ], + } as unknown as ISamchonGraphDump; + const route = JSON.parse(routeSummary(enriched)); + TestValidator.equals( + "the route record keeps compact serving provenance", + route.provenance, + [ + { + provider: "fixture-provider", + languages: ["typescript"], + authority: "compiler", + producer: { + tool: "fixture-compiler", + version: "1.2.3", + schemaVersion: 2, + protocolVersion: 1, + }, + }, + ], + ); + enriched.provenance![0]!.producer.version = "x".repeat(17_000); + TestValidator.equals( + "an oversized route record drops unbounded evidence explicitly", + JSON.parse(routeSummary(enriched)), + { + schemaVersion: 1, + indexer: dump.indexer, + provenance: [], + truncated: true, + }, + ); +} + /** * A dump says which path produced it, and why the better ones did not. * @@ -68,6 +127,26 @@ function assertTheDumpSaysWhatProducedIt(root: string): void { "the dump names the indexer that answered", summary.some((line) => line.includes("indexer=")), ); + const routeLine = summary.find((line) => + line.startsWith("@samchon/graph: route="), + ); + const route = JSON.parse( + routeLine?.slice("@samchon/graph: route=".length) ?? "null", + ) as { + schemaVersion?: number; + indexer?: string; + provenance?: unknown[]; + } | null; + TestValidator.equals( + "the discarded payload keeps a bounded machine-readable route record", + [ + route?.schemaVersion, + route?.indexer, + route?.provenance?.length, + (routeLine?.length ?? Number.POSITIVE_INFINITY) < 17_000, + ], + [1, "static", 0, true], + ); TestValidator.predicate( "and reports the reasons nothing better served", summary.length > 1, @@ -181,8 +260,7 @@ async function assertTimedOutDumpRetiresItsLanguageServer(): Promise { stderr += chunk; }); try { - await waitForFile(pidFile, 5_000); - serverPid = Number(fs.readFileSync(pidFile, "utf8")); + serverPid = await waitForProcessId(pidFile); child.kill("SIGTERM"); const code = await waitForExit(child, 5_000); TestValidator.equals( @@ -220,16 +298,6 @@ async function assertTimedOutDumpRetiresItsLanguageServer(): Promise { /* c8 ignore stop */ } -async function waitForFile(file: string, timeoutMs: number): Promise { - const deadline = Date.now() + timeoutMs; - while (!fs.existsSync(file)) { - if (Date.now() >= deadline) { - throw new Error(`Timed out waiting for ${file}`); - } - await new Promise((resolve) => setTimeout(resolve, 10)); - } -} - function waitForExit( child: import("node:child_process").ChildProcess, timeoutMs: number, diff --git a/tests/test-graph/src/features/test_cpp_clang_snapshot_adapter_and_client_are_atomic.ts b/tests/test-graph/src/features/test_cpp_clang_snapshot_adapter_and_client_are_atomic.ts index c565e3cf..2053c3e7 100644 --- a/tests/test-graph/src/features/test_cpp_clang_snapshot_adapter_and_client_are_atomic.ts +++ b/tests/test-graph/src/features/test_cpp_clang_snapshot_adapter_and_client_are_atomic.ts @@ -146,6 +146,9 @@ export const test_cpp_clang_snapshot_adapter_and_client_are_atomic = async () => await assertProvider(root); await assertClientLifecycle(root); await assertClientInputShapes(); + await assertClientReusesUnchangedInputDigests(); + await assertClientWatchesExternalDependencies(); + await assertClientCloseDoesNotReopenInputWatches(); await assertClientReportsItsOwnSize(); await assertClientReadsPublishedBodies(publishedFixtureRoot()); assertDeltaThatLosesAnOwnerAsksForTheWhole(publishedFixtureRoot()); @@ -239,14 +242,24 @@ async function assertProvider(root: string): Promise { } finally { await session.close(); } - // Asked to pass the server's log through, the provider also asks the server - // to write one. A producer that stops answering explains itself there and - // nowhere else, and a run that waited twenty minutes for one had nothing - // but its own request lines to show. + // Asked to pass the server's log through, the provider asks for bounded info + // diagnostics. Verbose transport logging mirrors every successful graph + // response into stderr, duplicating the largest payload in the protocol. + const loggedCommand = nodeShim(root, "info-log-clangd", COMMIT, [ + "--require-info-log", + ]); + const logged = cppGraphProvider.resolve(root, { + ...process.env, + [override]: loggedCommand, + }); + TestValidator.predicate( + "the C/C++ provider resolves its diagnostic logging fixture", + logged !== undefined, + ); process.env.SAMCHON_GRAPH_LSP_SERVER_LOG = "1"; const cppOnly = cppGraphProvider.open({ root, - command: resolved!, + command: logged!, languages: ["cpp"], options: {}, }); @@ -805,6 +818,7 @@ async function assertClientLifecycle(root: string): Promise { [edited.changed, edited.mode, edited.generation], [deleted.changed, deleted.mode, deleted.generation], client.generation, + client.current === deleted.snapshot, edited.snapshot.nodes.some((node) => node.name === "editedCaller"), // A refresh opens with the one request that carries no cursor, and that // is where it declares the generation it already holds. Continuations @@ -823,6 +837,7 @@ async function assertClientLifecycle(root: string): Promise { [true, "reload", 3], 3, true, + true, // Five openers for four refreshes. Deleting a compile command moves the // universe, which re-adapts every surviving shard -- and the adapter // remembers a shard by seven strings, not by its body, so it refuses the @@ -1301,15 +1316,501 @@ async function assertClientInputShapes(): Promise { } } +async function assertClientReusesUnchangedInputDigests(): Promise { + const root = fixtureRoot(); + const source = path.join(root, "main.cpp"); + const watchLog = path.join(root, "digest-watches.ndjson"); + fs.mkdirSync(path.join(root, ".clangd")); + const originalReadFileSync = fs.readFileSync; + const originalStatSync = fs.statSync; + let sourceReads = 0; + let moveSourceAfterReads = false; + let sourceMovements = 0; + fs.readFileSync = ((file: fs.PathOrFileDescriptor, ...args: unknown[]) => { + const result = Reflect.apply(originalReadFileSync, fs, [ + file, + ...args, + ]) as unknown; + if (typeof file === "string" && path.resolve(file) === source) { + ++sourceReads; + if (moveSourceAfterReads) { + const touched = originalStatSync(source); + fs.utimesSync( + source, + touched.atime, + new Date(touched.mtimeMs + 1_000), + ); + ++sourceMovements; + } + } + return result; + }) as typeof fs.readFileSync; + const client = cppClient(root, [`--watch-log=${watchLog}`]); + try { + await client.refresh(); + const initialReads = sourceReads; + await client.refresh(); + await client.refresh(); + TestValidator.equals( + "unchanged C/C++ inputs reuse their stable digest instead of rereading source bytes", + sourceReads, + initialReads, + ); + + const beforeTouch = fs.statSync(source); + fs.utimesSync( + source, + beforeTouch.atime, + new Date(beforeTouch.mtimeMs + 1_000), + ); + await client.refresh(); + TestValidator.equals( + "metadata movement is rehashed once but does not become a content notification", + [sourceReads, readLines(watchLog).length], + [initialReads + 1, 1], + ); + + const beforeMovement = originalStatSync(source); + fs.utimesSync( + source, + beforeMovement.atime, + new Date(beforeMovement.mtimeMs + 1_000), + ); + moveSourceAfterReads = true; + await client.refresh(); + moveSourceAfterReads = false; + TestValidator.equals( + "an input moving throughout every stable-read attempt is published as unknown", + [sourceReads, sourceMovements], + [initialReads + 7, 6], + ); + await client.refresh(); + TestValidator.equals( + "a settled input recovers from the unknown digest without reusing it", + sourceReads, + initialReads + 8, + ); + + await new Promise((resolve) => setTimeout(resolve, 20)); + const touched = fs.statSync(source); + fs.writeFileSync(source, "void called() {}\n"); + fs.utimesSync(source, touched.atime, touched.mtime); + await client.refresh(); + TestValidator.equals( + "same-size content movement cannot hide behind a restored modification time", + [ + sourceReads, + readLines(watchLog) + .flatMap((row) => row.changes) + .filter((row) => String(row.uri).endsWith("/main.cpp")) + .map((row) => row.type), + ], + [initialReads + 9, [1, 3, 2]], + ); + } finally { + fs.readFileSync = originalReadFileSync; + await client.close(); + } +} + +async function assertClientWatchesExternalDependencies(): Promise { + const root = GraphPaths.createTempDirectory("samchon-graph-cpp-watch-root-"); + const external = GraphPaths.createTempDirectory( + "samchon-graph-cpp-watch-external-", + ); + const include = path.join(external, "include"); + const source = path.join(external, "main.cpp"); + const header = path.join(include, "fixture.h"); + fs.mkdirSync(include); + fs.writeFileSync(source, '#include "include/fixture.h"\nvoid caller() {}\n'); + fs.writeFileSync(header, "void callee();\n"); + fs.writeFileSync( + path.join(root, "compile_commands.json"), + JSON.stringify([ + { + directory: external, + file: source, + arguments: ["clang++", "-x", "c++", "-c", source], + }, + ]), + ); + const watchLog = path.join(root, "external-watches.ndjson"); + const originalStatSync = fs.statSync; + const originalWatch = fs.watch; + let headerStats = 0; + let directoryStats = 0; + let suppressIncludeEvents = false; + fs.statSync = ((file: fs.PathLike, ...args: unknown[]) => { + if (typeof file === "string") { + const resolved = path.resolve(file); + if (resolved === header) ++headerStats; + else if (resolved === include) ++directoryStats; + } + return Reflect.apply(originalStatSync, fs, [file, ...args]) as fs.Stats; + }) as typeof fs.statSync; + fs.watch = ((...args: unknown[]): fs.FSWatcher => { + const callbackIndex = args.length - 1; + const callback = args[callbackIndex]; + if ( + path.resolve(String(args[0])) !== include || + typeof callback !== "function" + ) { + return Reflect.apply( + originalWatch as (...values: unknown[]) => fs.FSWatcher, + fs, + args, + ); + } + const forwarded = [...args]; + forwarded[callbackIndex] = (...values: unknown[]): void => { + if (!suppressIncludeEvents) Reflect.apply(callback, undefined, values); + }; + return Reflect.apply( + originalWatch as (...values: unknown[]) => fs.FSWatcher, + fs, + forwarded, + ); + }) as typeof fs.watch; + const client = cppClient(root, [`--watch-log=${watchLog}`]); + try { + await client.refresh(); + const initialHeaderStats = headerStats; + const initialDirectoryStats = directoryStats; + await client.refresh(); + TestValidator.equals( + "unchanged external dependencies avoid corpus-wide file and directory stats", + [headerStats, directoryStats > initialDirectoryStats], + [initialHeaderStats, false], + ); + + fs.writeFileSync(header, "void callee();\nvoid external_change();\n"); + await new Promise((resolve) => setTimeout(resolve, 100)); + const changed = await client.refresh(); + TestValidator.predicate( + "an external dependency event rehashes and republishes its affected generation", + changed.changed && + headerStats > initialHeaderStats && + readLines(watchLog) + .flatMap((row) => row.changes) + .some( + (row) => + String(row.uri) === pathToFileURL(header).href && row.type === 2, + ), + ); + + const watches = ( + client as unknown as { + inputWatches: Map; + } + ).inputWatches; + const parentWatches = ( + client as unknown as { + inputParentWatches: Map; + } + ).inputParentWatches; + const watchTransitions = client as unknown as { + releaseInputParentWatch(directory: string): void; + retireInputWatch(directory: string): void; + }; + const watchCount = watches.size; + watchTransitions.releaseInputParentWatch(root); + watchTransitions.releaseInputParentWatch( + path.join(external, "missing", "child"), + ); + watchTransitions.retireInputWatch(path.join(external, "missing")); + TestValidator.equals( + "stale and project-local parent-watch transitions are harmless", + watches.size, + watchCount, + ); + const externalWatch = watches.get(include)?.watcher; + const externalParentWatch = parentWatches.get(external)?.watcher; + TestValidator.predicate( + "the external dependency directory owns child and parent-entry watches", + externalWatch !== undefined && + externalParentWatch !== undefined, + ); + externalParentWatch!.emit("change", "change", "include"); + externalParentWatch!.emit("change", "rename", "unrelated"); + externalParentWatch!.emit("change", "rename", null); + await client.refresh(); + + const retiredInclude = path.join(external, "include-retired"); + const replacementInclude = path.join(external, "include-replacement"); + fs.mkdirSync(replacementInclude); + fs.writeFileSync( + path.join(replacementInclude, "fixture.h"), + "void callee();\nvoid replaced_directory();\n", + ); + // Suppress the child inode's callback on every host. The parent directory + // entry must retire and replace it without a corpus-wide identity poll. + suppressIncludeEvents = true; + fs.renameSync(include, retiredInclude); + fs.renameSync(replacementInclude, include); + const replaced = await client.refresh(); + suppressIncludeEvents = false; + const replacementWatch = watches.get(include)?.watcher; + TestValidator.predicate( + "an atomically replaced dependency directory reattaches through its platform change signal", + replaced.changed && + replacementWatch !== undefined && + replacementWatch !== externalWatch, + ); + fs.writeFileSync( + header, + "void callee();\nvoid replaced_directory();\nvoid replacement_change();\n", + ); + await new Promise((resolve) => setTimeout(resolve, 100)); + const replacementChanged = await client.refresh(); + TestValidator.predicate( + "the reattached directory watch observes later dependency edits", + replacementChanged.changed, + ); + + const failedParentWatch = parentWatches.get(external)?.watcher; + TestValidator.predicate( + "the replacement directory retains an active parent-entry watch", + failedParentWatch !== undefined, + ); + failedParentWatch!.emit("error", new Error("fixture parent watch failure")); + failedParentWatch!.emit("change", "rename", "include"); + failedParentWatch!.emit("error", new Error("retired parent watch")); + const parentFailureStats = directoryStats; + const parentGuardedWatch = fs.watch; + fs.watch = ((...args: unknown[]): fs.FSWatcher => { + if (path.resolve(String(args[0])) === external) { + throw new Error("fixture parent watch unavailable"); + } + return Reflect.apply( + parentGuardedWatch as (...values: unknown[]) => fs.FSWatcher, + fs, + args, + ); + }) as typeof fs.watch; + await client.refresh(); + const degradedRetired = path.join(external, "include-degraded-retired"); + const degradedReplacement = path.join(external, "include-degraded-next"); + fs.mkdirSync(degradedReplacement); + fs.writeFileSync( + path.join(degradedReplacement, "fixture.h"), + "void callee();\nvoid degraded_parent_change();\n", + ); + suppressIncludeEvents = true; + fs.renameSync(include, degradedRetired); + fs.renameSync(degradedReplacement, include); + const degraded = await client.refresh(); + suppressIncludeEvents = false; + fs.watch = parentGuardedWatch; + TestValidator.predicate( + "an unavailable parent watch falls back to one directory identity and republishes its replacement", + degraded.changed && + directoryStats > parentFailureStats && + headerStats > initialHeaderStats, + ); + + // A delayed native event from the earlier directory replacement may have + // retired and replaced the handle while the edit above was committed. + // Fail the handle that is active now so every platform exercises the + // intended error-to-fallback transition. + const activeReplacementWatch = watches.get(include)?.watcher; + TestValidator.predicate( + "the replacement directory retains an active watch after its edit", + activeReplacementWatch !== undefined, + ); + activeReplacementWatch!.emit( + "error", + new Error("fixture watch replacement"), + ); + const reattachReplacement = path.join(external, "include-reattach"); + const reattachRetired = path.join(external, "include-replaced-again"); + fs.mkdirSync(reattachReplacement); + fs.writeFileSync( + path.join(reattachReplacement, "fixture.h"), + "void callee();\nvoid replaced_directory();\nvoid reattach_change();\n", + ); + let reattachMoves = 0; + fs.watch = ((...args: unknown[]): fs.FSWatcher => { + const watcher = Reflect.apply( + originalWatch as (...values: unknown[]) => fs.FSWatcher, + fs, + args, + ); + if ( + path.resolve(String(args[0])) === include && + reattachMoves === 0 + ) { + ++reattachMoves; + fs.renameSync(include, reattachRetired); + fs.renameSync(reattachReplacement, include); + } + return watcher; + }) as typeof fs.watch; + const reattached = await client.refresh(); + fs.watch = originalWatch; + const reattachedWatch = watches.get(include)?.watcher; + TestValidator.predicate( + "a failed directory watch reattaches before its fallback digest closes the change window", + reattachMoves === 1 && + reattached.changed && + reattachedWatch !== undefined, + ); + + reattachedWatch?.emit("error", new Error("fixture watch failure")); + fs.writeFileSync(header, "void callee();\nvoid fallback_change();\n"); + const polled = await client.refresh(); + TestValidator.predicate( + "a failed external directory watch falls back to input polling", + polled.changed, + ); + + const relocated = GraphPaths.createTempDirectory( + "samchon-graph-cpp-watch-relocated-", + ); + const relocatedInclude = path.join(relocated, "include"); + const relocatedSource = path.join(relocated, "main.cpp"); + fs.mkdirSync(relocatedInclude); + fs.writeFileSync( + relocatedSource, + '#include "include/fixture.h"\nvoid relocated_caller() {}\n', + ); + fs.writeFileSync( + path.join(relocatedInclude, "fixture.h"), + "void relocated_callee();\n", + ); + fs.writeFileSync( + path.join(root, "compile_commands.json"), + JSON.stringify([ + { + directory: relocated, + file: relocatedSource, + arguments: ["clang++", "-x", "c++", "-c", relocatedSource], + }, + ]), + ); + const relocatedSnapshot = await client.refresh(); + const relocatedSourceWatch = watches.get(relocated)?.watcher; + const relocatedHeaderWatch = watches.get(relocatedInclude)?.watcher; + TestValidator.predicate( + "a compilation database that discovers a new external directory attaches its watch before publication", + relocatedSnapshot.changed && + relocatedSourceWatch !== undefined && + relocatedHeaderWatch !== undefined, + ); + + const localInclude = path.join(root, "include"); + const localSource = path.join(root, "main.cpp"); + fs.mkdirSync(localInclude); + fs.writeFileSync(localSource, "void caller() {}\n"); + fs.writeFileSync(path.join(localInclude, "fixture.h"), "void callee();\n"); + fs.writeFileSync( + path.join(root, "compile_commands.json"), + JSON.stringify([ + { + directory: root, + file: localSource, + arguments: ["clang++", "-x", "c++", "-c", localSource], + }, + ]), + ); + await client.refresh(); + TestValidator.predicate( + "dependency directories removed from the compilation universe release their watches", + !watches.has(external) && + !watches.has(include) && + !watches.has(relocated) && + !watches.has(relocatedInclude), + ); + relocatedSourceWatch?.emit("change", "change", "main.cpp"); + relocatedHeaderWatch?.emit("error", new Error("retired directory watch")); + const dirtyInputs = ( + client as unknown as { dirtyInputs: Set } + ).dirtyInputs; + TestValidator.predicate( + "late events from retired directory watches cannot reintroduce stale inputs", + !dirtyInputs.has(relocatedSource) && + !dirtyInputs.has(path.join(relocatedInclude, "fixture.h")), + ); + fs.writeFileSync( + path.join(root, "compile_commands.json"), + JSON.stringify([ + { + directory: external, + file: source, + arguments: ["clang++", "-x", "c++", "-c", source], + }, + ]), + ); + await client.refresh(); + TestValidator.predicate( + "shutdown retains an active external parent watch to close", + [...parentWatches.values()].some((watch) => watch.watcher !== undefined), + ); + } finally { + fs.watch = originalWatch; + fs.statSync = originalStatSync; + await client.close(); + } +} + +async function assertClientCloseDoesNotReopenInputWatches(): Promise { + const root = GraphPaths.createTempDirectory("samchon-graph-cpp-close-watch-"); + const source = path.join(root, "main.cpp"); + fs.writeFileSync(source, "void caller() {}\n"); + fs.writeFileSync( + path.join(root, "compile_commands.json"), + JSON.stringify([ + { + directory: root, + file: source, + arguments: ["clang++", "-x", "c++", "-c", source], + }, + ]), + ); + const client = cppClient(root, []); + await client.refresh(); + const state = client as unknown as { + dirtyInputs: Set; + inputWatches: Map; + }; + const retiredWatcher = [...state.inputWatches.values()].find( + (watch) => watch.watcher !== undefined, + )?.watcher; + const pending = client.refresh(); + const rejection = rejected( + "closing a refresh prevents its deferred input event turn from reopening directory watches", + pending, + "session is closed", + ); + await client.close(); + await rejection; + retiredWatcher?.emit("change", "change", "main.cpp"); + retiredWatcher?.emit("error", new Error("retired fixture watcher")); + TestValidator.equals( + "a closed C/C++ session neither reopens nor accepts late events into its input watch state", + [state.inputWatches.size, state.dirtyInputs.size], + [0, 0], + ); +} + async function assertClientFailures(root: string): Promise { const retry = cppClient(root, ["--retry=1", "--content-modified=1"]); TestValidator.equals( - "retryable Clang readiness and movement errors are polled to success", - (await retry.refresh()).changed, + "an undefined deadline keeps retrying Clang after the former private ceiling", + (await beyondLegacyReadyDeadline(() => retry.refresh())).changed, true, ); await retry.close(); + const boundedRetry = cppClient(root, ["--retry=1"], { + readyTimeoutMs: 10_000, + }); + TestValidator.equals( + "a bounded readiness wait clamps its backoff and can still recover", + (await boundedRetry.refresh()).changed, + true, + ); + await boundedRetry.close(); + const movementRoot = fixtureRoot(); const movementWatchLog = path.join(movementRoot, "movement-watches.ndjson"); const movement = cppClient(movementRoot, [ @@ -1451,13 +1952,30 @@ async function assertClientFailures(root: string): Promise { } ).initialize(new AbortController().signal); const delayAbort = new AbortController(); - const delayed = delaying.refresh({ signal: delayAbort.signal }); - setTimeout(() => delayAbort.abort("delay cancellation"), 20).unref?.(); - await rejected( - "retry delay remains cancellable", - delayed, - "cancel", - ); + const originalSetTimeout = globalThis.setTimeout; + let enterDelay!: () => void; + const enteredDelay = new Promise((resolve) => { + enterDelay = resolve; + }); + globalThis.setTimeout = (( + callback: (...args: unknown[]) => void, + milliseconds?: number, + ...args: unknown[] + ) => { + const timer = originalSetTimeout(callback, milliseconds, ...args); + // The request timeout is 5,000 ms; 50 ms is the client's first retry + // backoff. Resolve only once that timer has actually been installed. + if (milliseconds === 50) enterDelay(); + return timer; + }) as typeof setTimeout; + try { + const delayed = delaying.refresh({ signal: delayAbort.signal }); + await enteredDelay; + delayAbort.abort("delay cancellation"); + await rejected("retry delay remains cancellable", delayed, "cancel"); + } finally { + globalThis.setTimeout = originalSetTimeout; + } await delaying.close(); const queued = cppClient(root, ["--hang"]); @@ -1552,7 +2070,9 @@ function cppClient( producerCommit: COMMIT, initializationOptions: options.initializationOptions, requestTimeoutMs: 5_000, - readyTimeoutMs: options.readyTimeoutMs ?? 10_000, + ...(options.readyTimeoutMs === undefined + ? {} + : { readyTimeoutMs: options.readyTimeoutMs }), ...(options.pieceBudgetBytes === undefined ? {} : { pieceBudgetBytes: options.pieceBudgetBytes }), @@ -1564,6 +2084,7 @@ function nodeShim( root: string, name: string, commit: string, + args: readonly string[] = [], ): string { const directory = path.join(root, "shims"); fs.mkdirSync(directory, { recursive: true }); @@ -1575,6 +2096,7 @@ function nodeShim( `"${process.execPath}"`, `"${GraphPaths.fakeCppGraphServer}"`, `--commit=${commit}`, + ...args.map((argument) => JSON.stringify(argument)), ].join(" "); fs.writeFileSync( file, @@ -1586,6 +2108,27 @@ function nodeShim( return file; } +async function beyondLegacyReadyDeadline(operation: () => Promise) { + const original = Object.getOwnPropertyDescriptor(performance, "now"); + let first = true; + Object.defineProperty(performance, "now", { + configurable: true, + value: () => { + if (first) { + first = false; + return 0; + } + return 300_001; + }, + }); + try { + return await operation(); + } finally { + if (original === undefined) delete (performance as { now?: unknown }).now; + else Object.defineProperty(performance, "now", original); + } +} + function readLines(file: string): Array> { return fs .readFileSync(file, "utf8") diff --git a/tests/test-graph/src/features/test_csharp_roslyn_client_is_atomic_and_resident.ts b/tests/test-graph/src/features/test_csharp_roslyn_client_is_atomic_and_resident.ts new file mode 100644 index 00000000..f729519c --- /dev/null +++ b/tests/test-graph/src/features/test_csharp_roslyn_client_is_atomic_and_resident.ts @@ -0,0 +1,476 @@ +import { TestValidator } from "@nestia/e2e"; +import { + CSHARP_ROSLYN_PRODUCER, + CSHARP_ROSLYN_PROVIDER, + CsharpGraphClient, + csharpGraphProvider, +} from "@samchon/graph"; +import fs from "node:fs"; +import path from "node:path"; + +import { GraphPaths } from "../internal/GraphPaths.js"; + +/** The Roslyn adapter keeps compiler generations atomic across every client boundary. */ +export const test_csharp_roslyn_client_is_atomic_and_resident = async () => { + const root = GraphPaths.createTempDirectory("samchon-graph-csharp-client-"); + fs.writeFileSync(path.join(root, "Program.cs"), "namespace Fixture; public class Program {}\n"); + fs.writeFileSync( + path.join(root, "Fixture.csproj"), + 'net10.0\n', + ); + + await assertResidentLifecycle(root); + await assertIncrementalLifecycle(root); + await assertFullTransactionModes(root); + await assertRetryAndCancellation(root); + await assertAtomicRefusals(root); + await assertProviderContract(root); +}; + +async function assertResidentLifecycle(root: string): Promise { + const marker = path.join(root, "closed.txt"); + const requestLog = path.join(root, "resident.ndjson"); + const client = directClient(root, [ + `--marker=${marker}`, + `--request-log=${requestLog}`, + "--expect-initialization-options", + ], () => undefined, { fixture: true }); + const initial = await withRoslynTrace(() => client.refresh()); + const unchanged = await client.refresh(); + const requests = readRequests(requestLog); + TestValidator.equals( + "a resident Roslyn client commits once and reuses the exact immutable snapshot", + [ + initial.changed, + initial.mode, + initial.generation, + unchanged.changed, + unchanged.mode, + unchanged.snapshot === initial.snapshot, + client.current === initial.snapshot, + client.generation, + initial.snapshot.provenance.provider, + initial.snapshot.provenance.tool, + requests.filter((message) => message.method === "initialize").length, + requests + .filter((message) => message.method === "workspace/executeCommand") + .map((message) => message.params?.arguments?.[0]?.knownGeneration === null), + ], + [ + true, + "initial", + 1, + false, + "unchanged", + true, + true, + 1, + CSHARP_ROSLYN_PROVIDER, + CSHARP_ROSLYN_PRODUCER, + 1, + [true, false], + ], + ); + await Promise.all([client.close(), client.close()]); + TestValidator.equals( + "the resident Roslyn process closes through the LSP handshake", + fs.readFileSync(marker, "utf8"), + "closed", + ); + await rejected("a closed Roslyn session refuses refresh", client.refresh(), "session is closed"); +} + +async function assertIncrementalLifecycle(root: string): Promise { + const client = directClient(root, ["--change"]); + const initial = await client.refresh(); + const edited = await client.refresh(); + const unchanged = await client.refresh(); + TestValidator.equals( + "same-universe Roslyn changes replace one shard and then become a no-op", + [ + edited.changed, + edited.mode, + edited.generation, + edited.snapshot.nodes.map((node) => node.name), + edited.snapshot.protocol?.baseGeneration, + edited.snapshot.protocol?.sequence, + unchanged.changed, + unchanged.snapshot === edited.snapshot, + ], + [ + true, + "incremental", + 2, + ["edited"], + initial.snapshot.protocol?.generation, + 2, + false, + true, + ], + ); + await client.close(); + + let reject = true; + const replayed = directClient(root, [], () => { + if (reject) { + reject = false; + throw "fixture validation rejection"; + } + }); + await rejected( + "consumer validation rejects a complete generation before publication", + replayed.refresh(), + "fixture validation rejection", + ); + TestValidator.predicate( + "a rejected Roslyn generation leaves no partial current state", + replayed.current === undefined && replayed.generation === 0, + ); + const recovered = await replayed.refresh(); + TestValidator.predicate( + "the producer replays a full frame transaction after consumer rejection", + recovered.mode === "initial" && recovered.generation === 1, + ); + await replayed.close(); +} + +async function assertFullTransactionModes(root: string): Promise { + for (const mode of ["reload", "rebuild"] as const) { + const client = directClient(root, [`--transition=${mode}`]); + const initial = await client.refresh(); + const transitioned = await client.refresh(); + const unchanged = await client.refresh(); + TestValidator.predicate( + `${mode} is accepted only as a full transaction with the expected universe relation`, + initial.mode === "initial" && + transitioned.mode === mode && + transitioned.snapshot.protocol?.baseGeneration === undefined && + unchanged.mode === "unchanged" && + unchanged.snapshot === transitioned.snapshot, + ); + await client.close(); + } + for (const transition of ["reload-nonfull", "stale-base"] as const) { + const client = directClient(root, [`--transition=${transition}`]); + await client.refresh(); + await rejected( + `${transition} is not a valid transaction mode`, + client.refresh(), + "response mode disagrees", + ); + await client.close(); + } +} + +async function assertRetryAndCancellation(root: string): Promise { + const retried = directClient(root, ["--content-modified=1"]); + TestValidator.equals( + "content movement is retried inside the Roslyn client", + (await retried.refresh()).mode, + "initial", + ); + await retried.close(); + + const boundedRetry = directClient( + root, + ["--content-modified=1"], + () => undefined, + undefined, + { readyTimeoutMs: 1_000 }, + ); + TestValidator.equals( + "a bounded retry can settle before its deadline", + (await boundedRetry.refresh()).mode, + "initial", + ); + await boundedRetry.close(); + + const bounded = directClient(root, ["--content-modified=20"], () => undefined, undefined, { + readyTimeoutMs: 1, + }); + await rejected( + "a caller may bound how long a moving Solution is retried", + bounded.refresh(), + "did not settle within 1 ms", + ); + await bounded.close(); + + const signaled = directClient(root); + const liveSignal = new AbortController(); + TestValidator.equals( + "a live caller signal races initialization without changing its result", + (await signaled.refresh({ signal: liveSignal.signal })).mode, + "initial", + ); + await signaled.close(); + + const failedInitialization = directClient(root, ["--initialize-error"]); + await rejected( + "a signal-wrapped initialization surfaces the producer rejection", + failedInitialization.refresh({ signal: new AbortController().signal }), + "fixture initialize failure", + ); + await failedInitialization.close(); + + const initializeLog = path.join(root, "cancel-initialize.ndjson"); + const hangingInitialization = directClient(root, [ + "--hang-initialize", + `--request-log=${initializeLog}`, + ]); + const initializeController = new AbortController(); + const initializing = hangingInitialization.refresh({ + signal: initializeController.signal, + }); + await waitForRequest(initializeLog, "initialize"); + initializeController.abort(new Error("initialize stop")); + await rejected( + "caller cancellation wins a pending Roslyn initialization race", + initializing, + "initialize stop", + ); + await hangingInitialization.close(); + + const retryLog = path.join(root, "cancel-retry.ndjson"); + const moving = directClient(root, [ + "--content-modified=20", + `--request-log=${retryLog}`, + ]); + const retryController = new AbortController(); + const retrying = moving.refresh({ signal: retryController.signal }); + await waitForRequest(retryLog, "workspace/executeCommand"); + await new Promise((resolve) => setTimeout(resolve, 5)); + retryController.abort(new Error("retry stop")); + await rejected( + "caller cancellation interrupts the content-movement retry delay", + retrying, + "retry stop", + ); + await moving.close(); + + const requestLog = path.join(root, "cancel.ndjson"); + const hanging = directClient(root, ["--hang", `--request-log=${requestLog}`]); + const active = hanging.refresh(); + const activeResult = active.catch((error: unknown) => error); + await waitForRequest(requestLog, "workspace/executeCommand"); + const queuedController = new AbortController(); + const queued = hanging.refresh({ signal: queuedController.signal }); + queuedController.abort(new Error("queued stop")); + await rejected("a queued Roslyn refresh observes caller cancellation", queued, "queued stop"); + await hanging.close(); + TestValidator.predicate( + "closing the session cancels its active Roslyn request", + String(await activeResult).includes("LSP request aborted"), + ); + + const aborted = new AbortController(); + aborted.abort(new Error("already stopped")); + const fresh = directClient(root); + await rejected( + "an already-cancelled caller never enters the resident queue", + fresh.refresh({ signal: aborted.signal }), + "already stopped", + ); + await fresh.close(); +} + +async function assertAtomicRefusals(root: string): Promise { + const cases = [ + ["envelope", "malformed producer envelope"], + ["mode", "response mode disagrees"], + ["initial-base", "response mode disagrees"], + ["sequence", "envelope disagrees"], + ["generation", "envelope disagrees"], + ["universe", "envelope disagrees"], + ["unchanged-frames", "unchanged envelope"], + ] as const; + for (const [fault, message] of cases) { + const client = directClient(root, [`--malformed=${fault}`]); + await rejected(`${fault} is refused`, client.refresh(), message); + TestValidator.predicate( + `${fault} leaves the Roslyn store unpublished`, + client.current === undefined && client.generation === 0, + ); + await client.close(); + } + + const internal = directClient(root, ["--internal-error"]); + await rejected( + "producer failures are surfaced without fallback inside an owned session", + internal.refresh(), + "fixture internal failure", + ); + TestValidator.predicate( + "an internal producer failure publishes nothing", + internal.current === undefined, + ); + await internal.close(); +} + +async function assertProviderContract(root: string): Promise { + const empty = GraphPaths.createTempDirectory("samchon-graph-no-csharp-"); + const nested = GraphPaths.createTempDirectory("samchon-graph-nested-csharp-"); + const nestedDeep = GraphPaths.createTempDirectory( + "samchon-graph-nested-deep-csharp-", + ); + const nestedEmpty = GraphPaths.createTempDirectory( + "samchon-graph-nested-empty-csharp-", + ); + fs.mkdirSync(path.join(nested, "src")); + fs.mkdirSync(path.join(nested, "src", "empty")); + fs.mkdirSync(path.join(nested, "src", "bin")); + fs.writeFileSync(path.join(nested, "src", "notes.txt"), "not a project\n"); + fs.writeFileSync( + path.join(nested, "src", "bin", "Ignored.csproj"), + "\n", + ); + fs.writeFileSync(path.join(nested, "src", "Nested.csproj"), "\n"); + fs.mkdirSync(path.join(nestedDeep, "src", "deeper"), { recursive: true }); + fs.writeFileSync( + path.join(nestedDeep, "src", "deeper", "Nested.csproj"), + "\n", + ); + fs.mkdirSync(path.join(nestedEmpty, "src", "deeper"), { recursive: true }); + fs.writeFileSync( + path.join(nestedEmpty, "src", "deeper", "notes.txt"), + "not a project\n", + ); + const unconfigured = csharpGraphProvider.configuration?.(root, {}); + const configured = csharpGraphProvider.configuration?.(root, { + SAMCHON_GRAPH_ROSLYN_WORKSPACE: process.execPath, + SAMCHON_GRAPH_DOTNET_TOOLCHAIN: process.execPath, + }); + const installed = csharpGraphProvider.resolve(root, { + SAMCHON_GRAPH_ROSLYN_WORKSPACE: process.execPath, + }); + const source = csharpGraphProvider.resolve(root, { + SAMCHON_GRAPH_DOTNET_TOOLCHAIN: process.execPath, + }); + const unavailable = csharpGraphProvider.resolve(root, { + PATH: "", + SystemRoot: process.env.SystemRoot, + }); + TestValidator.predicate( + "the C# owner resolves only gated solutions and records both tool choices", + csharpGraphProvider.resolve(empty, { + SAMCHON_GRAPH_ROSLYN_WORKSPACE: process.execPath, + }) === undefined && + csharpGraphProvider.resolve(nested, { + SAMCHON_GRAPH_ROSLYN_WORKSPACE: process.execPath, + })?.command === process.execPath && + csharpGraphProvider.resolve(nestedEmpty, { + SAMCHON_GRAPH_ROSLYN_WORKSPACE: process.execPath, + }) === undefined && + csharpGraphProvider.resolve(nestedDeep, { + SAMCHON_GRAPH_ROSLYN_WORKSPACE: process.execPath, + })?.command === process.execPath && + installed?.command === process.execPath && + source?.command === process.execPath && + unavailable === undefined && + source.args.includes("run") && + source.args.includes("--dotnet-host") && + source.args.includes(process.execPath) && + source.args.some((argument) => argument.endsWith("Samchon.Graph.CSharp.csproj")) && + unconfigured?.includes("SAMCHON_GRAPH_ROSLYN_WORKSPACE=unconfigured") === true && + configured?.includes(`SAMCHON_GRAPH_DOTNET_TOOLCHAIN=${process.execPath}`) === true, + ); + TestValidator.predicate( + "the C# owner is compiler-authoritative and keeps scip-dotnet only as fallback", + csharpGraphProvider.authority === "compiler" && + csharpGraphProvider.fallbacks?.map((provider) => provider.name).join() === "scip-dotnet" && + csharpGraphProvider.refuse({ cwd: root }) === undefined && + csharpGraphProvider + .refuse({ cwd: root, server: "csharp-ls", maxFiles: 1, lspReferenceLimit: 1 }) + ?.includes("server, maxFiles, lspReferenceLimit") === true, + ); + + const session = csharpGraphProvider.open({ + root, + command: { command: process.execPath, args: [GraphPaths.fakeCsharpGraphServer] }, + languages: ["csharp"], + options: { cwd: root }, + }); + try { + TestValidator.equals( + "the registered route validates its exact compiler contract", + (await session.refresh()).snapshot.provenance.provider, + CSHARP_ROSLYN_PROVIDER, + ); + } finally { + await session.close(); + } +} + +function directClient( + root: string, + flags: string[] = [], + validate: ConstructorParameters[0]["validate"] = () => + undefined, + initializationOptions?: unknown, + timing: Pick< + ConstructorParameters[0], + "readyTimeoutMs" | "requestTimeoutMs" + > = {}, +): CsharpGraphClient { + return new CsharpGraphClient({ + root, + command: process.execPath, + args: [GraphPaths.fakeCsharpGraphServer, ...flags], + initializationOptions, + validate, + ...timing, + }); +} + +function readRequests(file: string): Array<{ + method?: string; + params?: { arguments?: Array<{ knownGeneration?: string | null }> }; +}> { + const text = fs.readFileSync(file, "utf8"); + const lines = text.split(/\r?\n/u); + // The fake server appends one JSON line per request. Windows can expose the + // appended bytes before the terminating newline reaches a concurrent + // reader, so only parse records whose write is observably complete. + if (!text.endsWith("\n")) lines.pop(); + return lines + .filter((line) => line !== "") + .map((line) => JSON.parse(line)); +} + +async function waitForRequest(file: string, method: string): Promise { + const deadline = performance.now() + 5_000; + while (performance.now() < deadline) { + if ( + fs.existsSync(file) && + readRequests(file).some((message) => message.method === method) + ) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error(`fake C# server did not receive ${method}`); +} + +async function rejected( + name: string, + promise: Promise, + message: string, +): Promise { + try { + await promise; + } catch (error) { + TestValidator.predicate(name, String(error).includes(message)); + return; + } + throw new Error(`${name}: expected rejection`); +} + +async function withRoslynTrace(task: () => Promise): Promise { + const prior = process.env["SAMCHON_GRAPH_ROSLYN_TRACE"]; + process.env["SAMCHON_GRAPH_ROSLYN_TRACE"] = "1"; + try { + return await task(); + } finally { + if (prior === undefined) delete process.env["SAMCHON_GRAPH_ROSLYN_TRACE"]; + else process.env["SAMCHON_GRAPH_ROSLYN_TRACE"] = prior; + } +} diff --git a/tests/test-graph/src/features/test_csharp_roslyn_producer_covers_solution_semantics.ts b/tests/test-graph/src/features/test_csharp_roslyn_producer_covers_solution_semantics.ts new file mode 100644 index 00000000..c108c44a --- /dev/null +++ b/tests/test-graph/src/features/test_csharp_roslyn_producer_covers_solution_semantics.ts @@ -0,0 +1,649 @@ +import { TestValidator } from "@nestia/e2e"; +import { csharpGraphProvider } from "@samchon/graph"; +import childProcess from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { GraphPaths } from "../internal/GraphPaths.js"; + +/** The shipped Roslyn producer proves one real multi-project, multi-target compilation. */ +export const test_csharp_roslyn_producer_covers_solution_semantics = async () => { + const root = GraphPaths.createTempDirectory("samchon-graph-csharp-producer-"); + const dotnet = resolveDotnet(); + writeFixture(root); + runDotnet(dotnet, root, [ + "build", + "Fixture.slnx", + "--configuration", + "Release", + "--verbosity", + "quiet", + ]); + + const command = csharpGraphProvider.resolve(root, { + SystemRoot: process.env.SystemRoot, + SAMCHON_GRAPH_DOTNET_TOOLCHAIN: dotnet, + }); + if (command === undefined) { + throw new Error("the shipped Roslyn source fallback did not resolve"); + } + const buildInputs = + typeof csharpGraphProvider.buildInputs === "function" + ? csharpGraphProvider.buildInputs(root) + : (csharpGraphProvider.buildInputs ?? []); + TestValidator.predicate( + "the coordinator fences the solution entry point the Roslyn workspace loaded", + buildInputs.includes("Fixture.slnx"), + ); + TestValidator.predicate( + "the source fallback tells MSBuildLocator which SDK host launched it", + command.args.includes("--dotnet-host") && command.args.includes(dotnet), + ); + + const session = csharpGraphProvider.open({ + root, + command, + languages: ["csharp"], + options: { cwd: root, lspTimeoutMs: 180_000 }, + }); + try { + const started = Date.now(); + const initial = await session.refresh(); + const coldMs = Date.now() - started; + const snapshot = initial.snapshot; + const targets = snapshot.protocol?.targets ?? []; + const families = new Set(snapshot.coverage?.map((row) => row.family)); + const generated = snapshot.nodes.filter( + (node) => node.name === "GeneratedMarker", + ); + const buildGenerated = snapshot.nodes.filter( + (node) => node.name === "LegacyGenerated", + ); + const workMethods = snapshot.nodes.filter( + (node) => + node.qualifiedName === "Company.One.Shared.Worker.Work(string)", + ); + const workIds = new Set(workMethods.map((node) => node.id)); + const runIds = new Set( + snapshot.nodes + .filter( + (node) => node.qualifiedName === "Company.Two.Shared.Runner.Run()", + ) + .map((node) => node.id), + ); + const extraIds = new Set( + snapshot.nodes + .filter((node) => node.qualifiedName === "Company.One.Shared.Extra") + .map((node) => node.id), + ); + + TestValidator.predicate( + "Roslyn opens the whole solution and expands both library target frameworks", + initial.mode === "initial" && + initial.changed && + targets.length >= 4 && + snapshot.nodes.some((node) => + node.qualifiedName?.startsWith("Library,"), + ) && + snapshot.nodes.some((node) => + node.qualifiedName?.startsWith("Consumer,"), + ), + ); + TestValidator.predicate( + "source-generated documents, records, generic arity, and partial methods keep semantic identities", + generated.length >= 2 && + buildGenerated.length >= 2 && + buildGenerated.every((node) => + node.file.startsWith("bundled:///csharp/generated/"), + ) && + snapshot.nodes.some((node) => node.name === "Deconstruct") && + new Set( + snapshot.nodes + .filter((node) => node.name === "Foo") + .map((node) => node.qualifiedName), + ).size >= 2 && + snapshot.nodes.filter((node) => node.name === "Hook").length === 2 && + workIds.size === 2, + ); + TestValidator.predicate( + "the producer publishes the complete per-target fact-coverage matrix", + families.size === 15 && + snapshot.coverage?.length === targets.length * 15 && + snapshot.coverage.every((row) => + row.family === "renders" + ? row.state === "unsupported" + : row.state === "partial", + ) === true, + ); + TestValidator.predicate( + "compiler semantics cover inheritance, calls, access, construction, attributes, tests, and dispatch candidates", + [ + "accesses", + "calls", + "decorates", + "extends", + "implements", + "instantiates", + "overrides", + "references", + "tests", + "type_ref", + ].every((kind) => snapshot.edges.some((edge) => edge.kind === kind)) && + snapshot.unresolved?.some( + (site) => + site.family === "dispatches" && + site.reason === "dynamic" && + site.candidates.length !== 0, + ) === true, + ); + TestValidator.predicate( + "every compiler input has checker and disk identity in the initial generation", + snapshot.sources.size !== 0 && + [...snapshot.sources.values()].every( + (source) => + source.checkerDigest.length === 64 && + (source.diskDigest === "" || source.diskDigest.length === 64), + ) && + [...snapshot.sources.keys()].some((file) => + file.replaceAll("\\", "/").endsWith("/Generator.dll"), + ), + ); + + const noOpStarted = Date.now(); + const noOp = await session.refresh(); + const noOpMs = Date.now() - noOpStarted; + // V8 coverage instruments the JavaScript transport around the native + // producer. The ordinary test enforces no-op latency and the experiment + // enforces both no-op and edit latency; the coverage replay verifies + // behavior without treating instrumentation overhead as Roslyn latency. + const timingReliable = process.env.NODE_V8_COVERAGE === undefined; + TestValidator.predicate( + "an unchanged resident solution returns the identical generation under 250 ms", + noOp.mode === "unchanged" && + !noOp.changed && + noOp.snapshot === snapshot && + noOp.generation === initial.generation && + (!timingReliable || noOpMs < 250), + ); + + const runner = path.join(root, "Consumer", "Runner.cs"); + const beforeInvalidBody = session.current; + fs.writeFileSync( + runner, + consumerSource(true).replace( + "return worker.Work(value.ToString() + extra.Value + suffix);", + "return missingSymbol;", + ), + ); + await rejectedRefresh(session, "compiler errors"); + TestValidator.predicate( + "a document-local compiler error is rejected without running a full analyzer pass", + session.current === beforeInvalidBody, + ); + fs.writeFileSync(runner, consumerSource(true)); + const edited = await session.refresh(); + const bodyRunIds = new Set( + edited.snapshot.nodes + .filter( + (node) => + node.qualifiedName === "Company.Two.Shared.Runner.Run()", + ) + .map((node) => node.id), + ); + TestValidator.predicate( + "a body edit reuses the resident solution and preserves declaration identities", + edited.mode === "incremental" && + runIds.size === bodyRunIds.size && + [...runIds].every((id) => bodyRunIds.has(id)) && + edited.snapshot.diagnostics.some((diagnostic) => + diagnostic.message.includes("CSHARP_ACCEPTANCE_WARNING"), + ), + ); + + const api = path.join(root, "Library", "Api.cs"); + fs.writeFileSync(api, librarySource(false, true)); + const libraryEdited = await waitForChanged(session); + TestValidator.predicate( + "an unrelated body edit retains diagnostics from unchanged project shards", + libraryEdited.mode === "incremental" && + libraryEdited.snapshot.diagnostics.some((diagnostic) => + diagnostic.message.includes("CSHARP_ACCEPTANCE_WARNING"), + ), + ); + + fs.writeFileSync(api, librarySource(true, true)); + const overloaded = await waitForChanged(session); + const survivingWorkIds = new Set( + overloaded.snapshot.nodes + .filter( + (node) => + node.qualifiedName === "Company.One.Shared.Worker.Work(string)", + ) + .map((node) => node.id), + ); + TestValidator.predicate( + "inserting an overload rechecks dependents without renumbering an existing symbol", + overloaded.mode === "incremental" && + workIds.size === survivingWorkIds.size && + [...workIds].every((id) => survivingWorkIds.has(id)) && + overloaded.snapshot.nodes.filter( + (node) => + node.qualifiedName === "Company.One.Shared.Worker.Work(int)", + ).length === 2, + ); + + const extra = path.join(root, "Library", "Extra.cs"); + const moved = path.join(root, "Library", "Moved.cs"); + fs.renameSync(extra, moved); + const renamed = await waitForChanged(session); + const renamedExtraIds = new Set( + renamed.snapshot.nodes + .filter((node) => node.qualifiedName === "Company.One.Shared.Extra") + .map((node) => node.id), + ); + const oldSourcePresent = [...renamed.snapshot.sources.keys()].some((file) => + file.replaceAll("\\", "/").endsWith("/Extra.cs"), + ); + const movedSourcePresent = [...renamed.snapshot.sources.keys()].some((file) => + file.replaceAll("\\", "/").endsWith("/Moved.cs"), + ); + TestValidator.predicate( + "a file rename updates the manifest without renumbering its semantic declarations", + renamed.mode === "incremental" && + extraIds.size === renamedExtraIds.size && + [...extraIds].every((id) => renamedExtraIds.has(id)) && + !oldSourcePresent && + movedSourcePresent, + ); + + const movedSource = fs.readFileSync(moved, "utf8"); + const acceptedRename = session.current; + fs.unlinkSync(moved); + await rejectedRefresh(session, "compiler errors"); + TestValidator.predicate( + "deleting a referenced declaration rechecks dependents and retains the accepted generation", + session.current === acceptedRename, + ); + fs.writeFileSync(moved, movedSource); + const restored = await waitForSuccess(session); + TestValidator.predicate( + "restoring a rejected deletion returns the still-current semantic generation", + restored.mode === "unchanged" && + restored.snapshot === acceptedRename && + [...extraIds].every((id) => + restored.snapshot.nodes.some((node) => node.id === id), + ), + ); + + const project = path.join(root, "Consumer", "Consumer.csproj"); + const validProject = fs.readFileSync(project, "utf8"); + const accepted = session.current; + fs.writeFileSync(project, ""); + await rejectedRefresh(session); + TestValidator.predicate( + "an invalid project reload rejects atomically and retains the accepted generation", + session.current === accepted, + ); + fs.writeFileSync(project, validProject); + const repairedProject = await waitForSuccess(session); + TestValidator.predicate( + "repairing a compiler input reproduces and reuses the exact accepted generation", + repairedProject.mode === "unchanged" && + !repairedProject.changed && + repairedProject.snapshot === accepted, + ); + + const configuredProject = validProject.replace( + "enable", + "enable\n SAMCHON_ACCEPTANCE", + ); + fs.writeFileSync(project, configuredProject); + const reloaded = await waitForChanged(session); + TestValidator.predicate( + "a valid project configuration change reloads the whole solution", + reloaded.mode === "reload" && + reloaded.snapshot !== repairedProject.snapshot, + ); + fs.writeFileSync(project, validProject); + const configurationRestored = await waitForChanged(session); + TestValidator.predicate( + "restoring project configuration returns the accepted generation through a reload", + configurationRestored.mode === "reload" && + configurationRestored.snapshot !== reloaded.snapshot, + ); + + TestValidator.predicate( + "the cold acceptance run completed with a measured compiler load", + coldMs > 0, + ); + } finally { + await session.close(); + } +}; + +function resolveDotnet(): string { + const configured = process.env.SAMCHON_GRAPH_DOTNET_TOOLCHAIN; + if (configured !== undefined && path.isAbsolute(configured)) { + return configured; + } + const probe = childProcess.spawnSync( + process.platform === "win32" ? "where.exe" : "/bin/sh", + process.platform === "win32" + ? ["dotnet"] + : ["-c", 'command -v "$1"', "csharp-acceptance", "dotnet"], + { encoding: "utf8", windowsHide: true }, + ); + const executable = probe.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .find((line) => line !== ""); + if (probe.status !== 0 || executable === undefined) { + throw new Error("the C# producer acceptance test requires a .NET SDK"); + } + return path.resolve(executable); +} + +function runDotnet(dotnet: string, root: string, args: string[]): void { + const result = childProcess.spawnSync(dotnet, args, { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + DOTNET_CLI_TELEMETRY_OPTOUT: "1", + DOTNET_NOLOGO: "1", + NUGET_XMLDOC_MODE: "skip", + }, + timeout: 180_000, + windowsHide: true, + }); + if (result.status !== 0) { + const detail = [ + result.error?.message, + result.signal === null ? undefined : `signal=${result.signal}`, + result.stdout, + result.stderr, + ] + .filter((value): value is string => value !== undefined && value !== "") + .join("\n"); + throw new Error( + `dotnet ${args[0]} failed (${String(result.status)}):\n${detail}`, + ); + } +} + +async function waitForChanged( + session: ReturnType, +) { + for (let attempt = 0; attempt !== 40; ++attempt) { + const result = await session.refresh(); + if (result.changed) return result; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error("the resident Roslyn producer did not observe the file change"); +} + +async function rejectedRefresh( + session: ReturnType, + expected?: string, +): Promise { + for (let attempt = 0; attempt !== 40; ++attempt) { + try { + const result = await session.refresh(); + if (result.changed) { + throw new Error("an invalid project was published"); + } + } catch (error) { + if ( + error instanceof Error && + !error.message.includes("an invalid project was published") && + (expected === undefined || error.message.includes(expected)) + ) { + return; + } + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error("the resident Roslyn producer did not reject the invalid project"); +} + +async function waitForSuccess( + session: ReturnType, +) { + for (let attempt = 0; attempt !== 40; ++attempt) { + try { + return await session.refresh(); + } catch { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } + throw new Error("the resident Roslyn producer did not recover after repair"); +} + +function writeFixture(root: string): void { + write( + root, + "Fixture.slnx", + ` + + + + +`, + ); + write( + root, + "Generator/Generator.csproj", + ` + + netstandard2.0 + latest + enable + true + + + + + +`, + ); + write( + root, + "Library/IsExternalInit.cs", + `#if NETSTANDARD2_1 +namespace System.Runtime.CompilerServices; + +internal static class IsExternalInit { } +#endif +`, + ); + write( + root, + "Generator/MarkerGenerator.cs", + `using Microsoft.CodeAnalysis; + +namespace Fixture.Generator; + +[Generator] +public sealed class MarkerGenerator : IIncrementalGenerator +{ + public void Initialize(IncrementalGeneratorInitializationContext context) + { + context.RegisterPostInitializationOutput(output => output.AddSource( + "GeneratedMarker.g.cs", + "namespace Company.One.Shared; public static class GeneratedMarker { public static string Value => \\\"generated\\\"; }")); + } +} +`, + ); + write( + root, + "Library/Library.csproj", + ` + + net10.0;netstandard2.1 + latest + enable + enable + + + + + + + +`, + ); + write(root, "Library/Api.cs", librarySource(false)); + write( + root, + "Library/obj/Generated/LegacyGenerated.g.cs", + `namespace Company.One.Shared; + +public static class LegacyGenerated +{ + public static string Value => "legacy-generated"; +} +`, + ); + write( + root, + "Library/Extra.cs", + `namespace Company.One.Shared; + +public sealed class Extra +{ + public string Value => "extra"; +} +`, + ); + write( + root, + "Library/Partial.cs", + `namespace Company.One.Shared; + +public partial record Worker +{ + partial void Hook() => state = state.Trim(); +} +`, + ); + write( + root, + "Consumer/Consumer.csproj", + ` + + net10.0 + enable + enable + + + + + +`, + ); + write( + root, + "Consumer/Runner.cs", + consumerSource(false), + ); +} + +function consumerSource(changedBody: boolean): string { + return `#warning CSHARP_ACCEPTANCE_WARNING + +using Company.One.Shared; + +namespace Company.Two.Shared; + +public sealed class Runner +{ + private readonly IWorker worker = new Worker(">"); + private readonly Extra extra = new(); + + public string Run() + { + var value = new Foo(); + var suffix = "${changedBody ? "changed" : "initial"}"; + return worker.Work(value.ToString() + extra.Value + suffix); + } +} + +public sealed class RunnerTests +{ + [Xunit.Fact] + public void CallsRun() => _ = new Runner().Run(); +} +`; +} + +function librarySource( + withOverload: boolean, + changedBody: boolean = false, +): string { + return `using System; + +namespace Xunit +{ + [AttributeUsage(AttributeTargets.Method)] + public sealed class FactAttribute : Attribute { } +} + +namespace Company.One.Shared +{ + public interface IWorker + { + string Work(string input); + } + + public abstract record BaseWorker + { + public abstract string Work(string input); + } + + [Marker("🤷‍")] + [NumericMarker(1e-7)] + [NumericMarker(1e-6)] + [NumericMarker(1e20)] + [NumericMarker(1e21)] + [NumericMarker(1.2345678901234567)] + [NumericMarker(0.0000012345678901234567)] + public partial record Worker(string Prefix) : BaseWorker, IWorker + { + private string state = "${changedBody ? "!!" : "!"}"; + + partial void Hook(); +${withOverload ? "\n public string Work(int input) => input.ToString();\n" : ""} + public override string Work(string input) + { + Func normalize = value => value.Trim(); + Hook(); + return normalize(Prefix + input + GeneratedMarker.Value) + state; + } + } + + public sealed class Foo { } + + public sealed class Foo { } + + [AttributeUsage(AttributeTargets.Class)] + public sealed class MarkerAttribute(string name) : Attribute + { + public string Name { get; } = name; + } + + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + public sealed class NumericMarkerAttribute(double value) : Attribute + { + public double Value { get; } = value; + } +} +`; +} + +function write(root: string, relative: string, contents: string): void { + const file = path.join(root, ...relative.split("/")); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, contents); +} diff --git a/tests/test-graph/src/features/test_experiment_corpora_are_commit_pinned.ts b/tests/test-graph/src/features/test_experiment_corpora_are_commit_pinned.ts index 3bacbd68..0ae5c311 100644 --- a/tests/test-graph/src/features/test_experiment_corpora_are_commit_pinned.ts +++ b/tests/test-graph/src/features/test_experiment_corpora_are_commit_pinned.ts @@ -1,21 +1,46 @@ import { TestValidator } from "@nestia/e2e"; import { CPP_CLANG_PRODUCER_COMMIT, + JDT_GRAPH_PRODUCER_COMMIT, LANGUAGE_SPECS, RUST_GRAPH_PRODUCER_COMMIT, } from "@samchon/graph"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; +import { + measureLifecycleNoopPerformance, + nearestRankP95, +} from "../../../experiment/src/lifecycle-performance.mjs"; +import { findExperiment } from "../../../experiment/src/catalog.mjs"; +import { verifyGitTree } from "../../../experiment/src/git-tree.mjs"; +import { captureKotlinBuildReport } from "../../../experiment/src/kotlin-build-report.mjs"; +import { + RUST_GRAPH_PRODUCER_SLOW_TEST, + RUST_GRAPH_PRODUCER_UNIT_TEST, + verifyRustGraphProducer, +} from "../../../experiment/src/rust-producer.mjs"; +import { + measureClangBackgroundIndex, +} from "../../../experiment/src/clang-background-baseline.mjs"; +import { hasRepresentativeEdge } from "../../../experiment/src/representative-edges.mjs"; import { GraphPaths } from "../internal/GraphPaths"; /** Real-language experiments always check out one reviewable corpus revision. */ -export const test_experiment_corpora_are_commit_pinned = () => { +export const test_experiment_corpora_are_commit_pinned = async () => { + verifyGitTreeFixture(); + verifyKotlinBuildReportFixture(); const catalog = experimentSource("catalog.mjs"); const helpers = experimentSource("process.mjs"); const lifecycle = experimentSource("strict-lifecycle.mjs"); + const lifecyclePerformance = experimentSource("lifecycle-performance.mjs"); const runner = experimentSource("run-language.mjs"); const setup = experimentSource("setup-language.mjs"); + const gitTree = experimentSource("git-tree.mjs"); + const javaAgreement = experimentSource("java-producer-agreement.mjs"); + const clangProducer = experimentSource("clang-producer.mjs"); + const evidenceSummary = experimentSource("evidence-summary.mjs"); const repositories = [...catalog.matchAll(/repository:\s*"[^"]+"/g)]; const commits = [...catalog.matchAll(/commit:\s*"([0-9a-f]{40})"/g)]; @@ -50,13 +75,16 @@ export const test_experiment_corpora_are_commit_pinned = () => { const dart = region(catalog, 'language: "dart"', "\n];"); const luaSetup = region(setup, 'case "lua"', 'case "dart"'); const javaSetup = region(setup, 'case "java"', 'case "csharp"'); + const csharpSetup = region(setup, 'case "csharp"', 'case "kotlin"'); const kotlinSetup = region(setup, 'case "kotlin"', 'case "swift"'); + const swiftSetup = region(setup, 'case "swift"', 'case "scala"'); + const scalaSetup = region(setup, 'case "scala"', 'case "zig"'); const rustSetup = region(setup, 'case "rust"', 'case "cpp"'); const cppSetup = region(setup, 'case "cpp"', 'case "java"'); TestValidator.equals( "every registered strict-provider language has a lifecycle row", [...catalog.matchAll(/strictProvider:\s*"[^"]+"/g)].length, - 13, + 15, ); TestValidator.predicate( "Rust builds and records the exact native HIR producer declared by the catalog", @@ -65,14 +93,424 @@ export const test_experiment_corpora_are_commit_pinned = () => { rustSetup.includes("--default-toolchain 1.95.0") && rustSetup.includes('"rust-src"') && rustSetup.includes('["fetch", "--depth=1", "origin", experiment.producerCommit]') && + rustSetup.includes("verifyRustGraphProducer({ cargo, producerRoot, run })") && + rustSetup.indexOf("verifyRustGraphProducer({ cargo, producerRoot, run })") < + rustSetup.indexOf('["build", "--locked", "--release", "-p", "rust-analyzer"]') && rustSetup.includes('["build", "--locked", "--release", "-p", "rust-analyzer"]') && rustSetup.includes('for (const command of ["samchon-rust-analyzer", "rust-analyzer"])') && rustSetup.includes("fs.linkSync(producerBinary, link)") && !rustSetup.includes("rustup component add rust-analyzer"), ); + TestValidator.predicate( + "Rust separately measures its native baseline and resident p95 targets", + rust.includes( + 'nativeBaseline: "samchon-rust-analyzer prime-caches ."', + ) && + rust.includes("noopSamples: 20") && + rust.includes("editSamples: 20") && + rust.includes("noopP95MaxMs: 250") && + rust.includes("editP95MaxMs: 2_000") && + rust.includes('editFind: "broadcast::channel(1)"') && + rust.includes('"broadcast::channel(2)"') && + rust.includes('"broadcast::channel(3)"') && + lifecycle.includes('name: "native-baseline"') && + lifecycle.includes("measureLifecyclePerformance({") && + lifecyclePerformance.includes('name: "performance"') && + lifecyclePerformance.includes("performance no-op") && + lifecyclePerformance.includes("performance edit"), + ); + TestValidator.equals( + "nearest-rank p95 keeps singleton and exact twenty-sample boundaries", + [nearestRankP95([7]), nearestRankP95(Array.from({ length: 20 }, (_, i) => i + 1))], + [7, 19], + ); + TestValidator.error("nearest-rank p95 rejects an empty sample", () => + nearestRankP95([]), + ); + TestValidator.predicate( + "C and C++ measure twenty exact resident no-ops below 250 ms", + [c, cpp].every( + (row) => + row.includes('kind: "clang-background-index"') && + row.includes('command: "samchon-clangd"') && + row.includes("noopPerformance: {") && + row.includes("samples: 20") && + row.includes("p95MaxMs: 250"), + ) && lifecycle.includes("measureLifecycleNoopPerformance({"), + ); + const cExperiment = findExperiment("c") as { + repository: string; + commit: string; + representativeEdges: Array<{ kind: string; from: string; to: string }>; + }; + const cppExperiment = findExperiment("cpp") as typeof cExperiment; + TestValidator.equals( + "Redis and LevelDB smokes pin exact representative semantic edges", + [ + [ + cExperiment.repository, + cExperiment.commit, + cExperiment.representativeEdges, + ], + [ + cppExperiment.repository, + cppExperiment.commit, + cppExperiment.representativeEdges, + ], + [ + runner.includes("representativeEdges"), + runner.includes("hasRepresentativeEdge(dump, claim)"), + hasRepresentativeEdge( + { + nodes: [ + { id: "from", qualifiedName: "leveldb::DBImpl::Get" }, + { id: "to", qualifiedName: "leveldb::MemTable::Get" }, + ], + edges: [{ from: "from", to: "to", kind: "calls" }], + }, + { + from: "wrong::leveldb::DBImpl::Get", + to: "leveldb::MemTable::Get", + kind: "calls", + }, + ), + ], + ], + [ + [ + "https://github.com/samchon/graph-benchmark-redis.git", + "6bf6224c3dad518329ddc893ef9c5d58dcbabdeb", + [ + { kind: "calls", from: "processCommand", to: "lookupCommand" }, + { kind: "calls", from: "processCommand", to: "call" }, + { kind: "accesses", from: "processCommand", to: "server" }, + { kind: "type_ref", from: "processCommand", to: "client" }, + ], + ], + [ + "https://github.com/samchon/graph-benchmark-leveldb.git", + "7ee830d02b623e8ffe0b95d59a74db1e58da04c5", + [ + { + kind: "calls", + from: "leveldb::DBImpl::Get", + to: "leveldb::MemTable::Get", + }, + { + kind: "accesses", + from: "leveldb::DBImpl::Get", + to: "leveldb::DBImpl::mutex_", + }, + { + kind: "type_ref", + from: "leveldb::DBImpl::Get", + to: "leveldb::Slice", + }, + { kind: "extends", from: "leveldb::DBImpl", to: "leveldb::DB" }, + ], + ], + [true, true, false], + ], + ); + const residentDump = {}; + let noopSample = 0; + const noopPerformance = await measureLifecycleNoopPerformance({ + language: "cpp", + samples: 20, + p95MaxMs: 250, + currentDump: residentDump, + currentIdentity: "resident", + load: () => ({ + dump: residentDump, + mode: "unchanged", + identity: "resident", + elapsedMs: ++noopSample, + }), + }); + TestValidator.equals( + "no-op performance publishes every sample and nearest-rank p95", + noopPerformance, + { + name: "noop-performance", + status: "passed", + samples: Array.from({ length: 20 }, (_, index) => index + 1), + p95Ms: 19, + p95MaxMs: 250, + }, + ); + let boundaryError = ""; + try { + await measureLifecycleNoopPerformance({ + language: "cpp", + samples: 1, + p95MaxMs: 5, + currentDump: residentDump, + currentIdentity: "resident", + load: () => ({ + dump: residentDump, + mode: "unchanged", + identity: "resident", + elapsedMs: 5, + }), + }); + } catch (error) { + boundaryError = error instanceof Error ? error.message : String(error); + } + TestValidator.equals( + "no-op performance treats the strict ceiling as a miss", + boundaryError, + "cpp: lifecycle no-op performance missed its target: p95 5/5 ms", + ); + let progressListener: ((params: { + token: string; + value: { kind: string }; + }) => void) | undefined; + const baselineCalls: unknown[] = []; + const clock = [100, 175]; + const baselineRoot = path.resolve("baseline-fixture"); + const baselineCompilationDatabase = path.join( + baselineRoot, + "build", + "compile_commands.json", + ); + const baselineSource = path.join(baselineRoot, "db", "db_impl.cc"); + const baselineElapsed = await measureClangBackgroundIndex({ + command: "samchon-clangd", + compilationDatabase: baselineCompilationDatabase, + cwd: baselineRoot, + language: "cpp", + sourceFile: baselineSource, + timeoutMs: 1_000, + now: () => clock.shift()!, + readSource: () => "int baseline();\n", + createClient: (command, args) => ({ + onNotification: (method, listener) => { + baselineCalls.push(["notification", method]); + progressListener = listener; + }, + request: async (method, params) => { + baselineCalls.push(["request", method, params]); + return {}; + }, + notify: (method, params) => { + baselineCalls.push(["notify", method, params]); + if (method !== "textDocument/didOpen") return; + progressListener?.({ + token: "backgroundIndexProgress", + value: { kind: "begin" }, + }); + progressListener?.({ + token: "backgroundIndexProgress", + value: { kind: "end" }, + }); + }, + close: async () => { + baselineCalls.push(["close", command, args]); + }, + }), + }); + TestValidator.equals( + "native clang baseline waits for standard background-index progress", + [baselineElapsed, baselineCalls[0], baselineCalls.at(-1)], + [ + 75, + ["notification", "$/progress"], + [ + "close", + "samchon-clangd", + [ + "--background-index", + `--compile-commands-dir=${path.join(baselineRoot, "build")}`, + ], + ], + ], + ); + TestValidator.predicate( + "native clang baseline opens a real compilation-database source", + baselineCalls.some( + (entry) => + Array.isArray(entry) && + entry[0] === "notify" && + entry[1] === "textDocument/didOpen", + ), + ); + let initializationError = ""; + try { + await measureClangBackgroundIndex({ + command: "samchon-clangd", + compilationDatabase: baselineCompilationDatabase, + cwd: baselineRoot, + language: "cpp", + sourceFile: baselineSource, + timeoutMs: 10, + readSource: () => "int baseline();\n", + createClient: () => ({ + onNotification: () => undefined, + request: async () => { + throw new Error("fixture initialize failure"); + }, + notify: () => undefined, + close: async () => undefined, + }), + }); + } catch (error) { + initializationError = error instanceof Error ? error.message : String(error); + } + TestValidator.equals( + "native clang baseline preserves initialize failures without a stray timeout rejection", + initializationError, + "fixture initialize failure", + ); + const cargoCalls: Array<{ + command: string; + args: string[]; + options: { + cwd: string; + stdio: string; + check: boolean; + env?: Record; + }; + }> = []; + verifyRustGraphProducer({ + cargo: "cargo", + producerRoot: "producer", + run: (command, args, options) => { + cargoCalls.push({ command, args, options }); + return { + status: 0, + stdout: + "running 1 test\ntest fixture ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out\n", + stderr: "", + }; + }, + emit: () => undefined, + }); + TestValidator.equals( + "Rust producer verification runs exact unit and slow fixtures", + cargoCalls, + [ + { + command: "cargo", + args: [ + "test", + "--locked", + "--release", + "-p", + "ide", + "--lib", + RUST_GRAPH_PRODUCER_UNIT_TEST, + "--", + "--exact", + ], + options: { cwd: "producer", stdio: "pipe", check: false }, + }, + { + command: "cargo", + args: [ + "test", + "--locked", + "--release", + "-p", + "rust-analyzer", + "--test", + "slow-tests", + RUST_GRAPH_PRODUCER_SLOW_TEST, + "--", + "--exact", + ], + options: { + cwd: "producer", + stdio: "pipe", + check: false, + env: { RUN_SLOW_TESTS: "1" }, + }, + }, + ], + ); + let zeroTestError = ""; + try { + verifyRustGraphProducer({ + cargo: "cargo", + producerRoot: "producer", + run: () => ({ + status: 0, + stdout: + "running 0 tests\n\ntest result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out\n", + stderr: "", + }), + emit: () => undefined, + }); + } catch (error) { + zeroTestError = error instanceof Error ? error.message : String(error); + } + TestValidator.equals( + "Rust producer verification rejects Cargo's successful zero-test result exactly", + zeroTestError, + "Rust HIR unit fixture did not run exactly one passing test at the pinned producer commit", + ); + const emittedFailure: string[] = []; + let producerFailure = ""; + try { + verifyRustGraphProducer({ + cargo: "cargo", + producerRoot: "producer", + run: (_command, _args, options) => { + TestValidator.equals("failed Rust fixture stays captured", options, { + cwd: "producer", + stdio: "pipe", + check: false, + }); + return { + status: 101, + stdout: "actionable producer stdout\n", + stderr: "actionable producer stderr\n", + }; + }, + emit: (stdout, stderr) => emittedFailure.push(stdout, stderr), + }); + } catch (error) { + producerFailure = error instanceof Error ? error.message : String(error); + } + TestValidator.equals( + "failed Rust producer fixtures emit both streams before rejection", + emittedFailure, + ["actionable producer stdout\n", "actionable producer stderr\n"], + ); + TestValidator.equals( + "failed Rust producer fixture retains its exact exit code", + producerFailure, + "Rust HIR unit fixture failed at the pinned producer commit: exited with code 101", + ); + for (const failure of [ + { + result: { status: null, signal: null, error: new Error("spawn ENOENT") }, + detail: "could not start: spawn ENOENT", + }, + { + result: { status: null, signal: "SIGKILL" }, + detail: "terminated by signal SIGKILL", + }, + ]) { + let message = ""; + try { + verifyRustGraphProducer({ + cargo: "cargo", + producerRoot: "producer", + run: () => ({ ...failure.result, stdout: "", stderr: "" }), + emit: () => undefined, + }); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + TestValidator.equals( + `Rust producer failure preserves ${failure.detail}`, + message, + `Rust HIR unit fixture failed at the pinned producer commit: ${failure.detail}`, + ); + } TestValidator.predicate( "the remaining SCIP providers use isolated upstream lifecycle projects", - [kotlin, ruby, php, dart].every( + [ruby, php, dart].every( (row) => row.includes("projectRoot:") && row.includes('strictAuthority: "semantic-index"') && @@ -83,8 +521,8 @@ export const test_experiment_corpora_are_commit_pinned = () => { helpers.includes("fs.cpSync(source, root"), ); TestValidator.predicate( - "an unavailable compiler identity requires an explicit row limitation", - kotlin.includes("compilerLimitation:") && + "compiler identity and an explicit limitation remain mutually exclusive", + !kotlin.includes("compilerLimitation:") && runner.includes("experiment.compilerLimitation.trim()") && runner.includes('typeof provenance.producer.compiler === "string"') && runner.includes("provenance.producer.compiler.trim()") && @@ -102,17 +540,26 @@ export const test_experiment_corpora_are_commit_pinned = () => { setup.includes("await installGradle()"), ); TestValidator.predicate( - "Kotlin uses one checksum-pinned 2.3.20 producer and fixture generation", - kotlin.includes("e940c1889767a81347387067a375320dc6f5d83e") && - kotlin.includes("built with Kotlin 2.3.20") && - setup.includes("const SCIP_JAVA_KOTLIN_COMMIT") && - setup.includes('"e940c1889767a81347387067a375320dc6f5d83e"') && + "Kotlin pins a Koin-scale fixture and its resident K2 producer independently", + kotlin.includes("cca45c63d1088888f445304e13f9fbc310f62078") && + kotlin.includes("3a1565d0647d89a28880fa40ecbef0966a1a328c") && + kotlin.includes("3b5c24126b0670c9c9bd9369df71fcd112b34b67") && + kotlin.includes('strictProvider: "kotlinc-graph"') && + kotlin.includes('strictAuthority: "compiler"') && + kotlin.includes('strictTool: "scip-kotlinc-k2-graph"') && + kotlin.includes("strictMinimums: true") && + kotlin.includes('nativeBaseline: "gradle compileKotlin"') && + kotlin.includes("kotlinBuildReportRoot:") && + kotlin.includes("moduleName.set") && + kotlin.includes("minNodes: 1_000") && + kotlin.includes("minEdges: 1_000") && + kotlin.includes("noopP95MaxMs: 250") && + kotlin.includes("editP95MaxMs: 2000") && + !setup.includes("const SCIP_JAVA_KOTLIN_COMMIT") && + !setup.includes("const SCIP_JAVA_KOTLIN_TREE") && setup.includes('const SCIP_JAVA_KOTLIN_VERSION = "2.3.20"') && setup.includes( - "985eb03ef165864dbae3db4453d4566e699f78761bace3e4614bf67d38ce76cf", - ) && - setup.includes( - "`${SCIP_JAVA_KOTLIN_COMMIT}+kotlin-${SCIP_JAVA_KOTLIN_VERSION}`", + "`${experiment.producerCommit}+kotlin-${SCIP_JAVA_KOTLIN_VERSION}`", ) && setup.includes('":scip-java:installDist"') && // Two rows build the same launcher from two revisions, so the builder is @@ -125,8 +572,121 @@ export const test_experiment_corpora_are_commit_pinned = () => { !setup.includes("installScipJava = ") && kotlinSetup.includes("await installScipJavaKotlinSnapshot(gradle)") && setup.includes("const installScipJavaSource = async (gradle, pin)") && + setup.includes('run(link, ["index", "--help"]') && + setup.includes('run(link, ["kotlin-graph-server", "--help"]') && + setup.includes( + 'recordProvisionedEnvironment("SAMCHON_GRAPH_KOTLINC_GRAPH", link)', + ) && setup.includes('run(link, ["--version"])'), ); + TestValidator.predicate( + "Kotlin lifecycle rows publish compiler invalidation evidence", + lifecycle.includes("captureKotlinBuildReport(") && + lifecycle.includes("kotlinBuildReport:"), + ); + TestValidator.predicate( + "Scala pins both compiler lines and builds its BSP producer from shipped source", + scala.includes( + 'repository: "https://github.com/samchon/graph-benchmark-scala.git"', + ) && + scala.includes("b11f22758c902bffa29513c9fcda07863a2ad996") && + scala.includes('strictProvider: "scalac-graph"') && + scala.includes('strictAuthority: "compiler"') && + scala.includes('strictTool: "samchon-scala-graph"') && + scala.includes('prepare: "sbt bspConfig"') && + scala.includes("env -u SAMCHON_GRAPH_SCALA2_PLUGIN") && + scala.includes("noopP95MaxMs: 500") && + scala.includes("editP95MaxMs: 15_000") && + scala.includes("minNodes: 30") && + scala.includes("minEdges: 150") && + scalaSetup.includes('apt(["openjdk-21-jdk", "maven"])') && + scalaSetup.includes('path.join(repositoryRoot, "sidecars", "scala", "pom.xml")') && + scalaSetup.includes('`scala-graph-plugin_2.13.18-${version}.jar`') && + scalaSetup.includes('`scala-graph-plugin_3.9.0-${version}.jar`') && + scalaSetup.includes('path.join(binRoot, "samchon-scala-graph")') && + scalaSetup.includes("f92a2095ac75008764fe3b2b793ffe624c4fbef5bfd9b0022e4bc2daf668c651") && + scalaSetup.includes("SAMCHON_GRAPH_SCALA_GRAPH: producer") && + scalaSetup.includes("SAMCHON_GRAPH_SCALA2_PLUGIN: scala2Plugin") && + scalaSetup.includes("SAMCHON_GRAPH_SCALA3_PLUGIN: scala3Plugin") && + scalaSetup.includes("SAMCHON_GRAPH_SCALA_PLUGIN_VERSION: version"), + ); + TestValidator.predicate( + "Swift builds the pinned IndexStoreDB sidecar and measures native SwiftPM", + swift.includes( + 'repository: "https://github.com/apple/swift-argument-parser.git"', + ) && + swift.includes("2f77f2fccb6e84fecff338c37b199e33e7dfd119") && + swift.includes( + '"swift build --enable-index-store --build-tests -Xswiftc -index-include-locals"', + ) && + swift.includes('strictProvider: "swift-indexstore"') && + swift.includes('strictAuthority: "compiler"') && + swift.includes('strictTool: "samchon-swift-graph"') && + swift.includes('createdSymbol: "samchonGraphExperiment"') && + swift.includes('to: "mapEmpty"') && + swift.includes("noopP95MaxMs: 250") && + swift.includes("editP95MaxMs: 20_000") && + swiftSetup.includes('path.join(repositoryRoot, "sidecars", "swift")') && + swiftSetup.includes('"--configuration",') && + swiftSetup.includes('"release",') && + swiftSetup.includes('process.platform === "linux"') && + swiftSetup.includes('path.join(swiftRoot, "lib", "swift", "Block")') && + swiftSetup.includes('path.join(sidecarBin, "samchon-swift-graph")') && + swiftSetup.includes( + 'recordProvisionedEnvironment("SAMCHON_GRAPH_SWIFT_GRAPH", producer)', + ) && + swiftSetup.includes( + "indexstore-db-54212fce1aecb199070808bdb265e7f17e396015", + ), + ); + TestValidator.predicate( + "Java pins both producers, verifies their breadth and proves agreement", + java.includes('kind: "shell"') && + java.includes('command: "mvn -q test-compile"') && + java.includes("warmup: true") && + java.includes('clean: ["target"]') && + java.includes( + `jdtProducerCommit: "${JDT_GRAPH_PRODUCER_COMMIT}"`, + ) && + java.includes( + 'producerTree: "8cb3dd9b84fbbbb8dba22827b9d8e7dd21c3f46e"', + ) && + /jdtProducerTree:\s*"[0-9a-f]{40}"/u.test(java) && + !java.includes("regenerationLimitation:") && + setup.includes("if (pin.verify !== undefined) pin.verify({ gradle, source })") && + setup.includes("org.scip_code.scip_java.javac.JavaGraphShardTest") && + setup.includes("org.scip_code.scip_java.gradle.GraphGenerationStoreTest") && + setup.includes("tests.GradleGraphLifecycleTest") && + setup.includes("tests.MavenGraphLifecycleTest") && + setup.includes("tests.MavenGraphPluginTest") && + setup.includes("tests.GraphAggregateRunnerTest") && + setup.includes("tests.GradleBuildToolTest") && + setup.includes("const installJdtGraphProducer = async") && + setup.includes("verifyGitTree(source, pin.tree)") && + setup.includes("digest: `git-tree:${pin.tree}`") && + setup.includes("verifyGitTree(source, experiment.jdtProducerTree)") && + gitTree.includes('["add", "--all", "--force"]') && + gitTree.includes('["write-tree"]') && + setup.includes('run(maven, ["clean", "install", "-U", "-DskipTests=true"]') && + setup.includes("GraphSnapshotCommandTest") && + setup.includes("UnresolvedTypesQuickFixTest#testTypeInSealedTypeDeclaration") && + setup.includes("FileEventHandlerTest") && + setup.includes("CleanUpsTest") && + setup.includes('path.join(binRoot, "samchon-jdtls")') && + setup.includes( + 'recordProvisionedEnvironment("SAMCHON_GRAPH_JDT_WORKSPACE", dedicated)', + ) && + runner.includes("runJavaProducerAgreement(experiment, cwd)") && + javaAgreement.includes("const javac = await buildGraphDump(options)") && + javaAgreement.includes("delete process.env[JAVAC_OVERRIDE]") && + javaAgreement.includes("const jdt = await buildGraphDump(options)") && + /declaration\(\s*"constructor"/u.test(javaAgreement) && + javaAgreement.includes('declaration("method"') && + javaAgreement.includes('declarationsFor("GradleMainAgreement", ":compileJava")') && + javaAgreement.includes('declarationsFor("GradleTestAgreement", ":compileTestJava")') && + javaAgreement.includes('":module:compileJava"') && + javaAgreement.includes('["wrapper", "--gradle-version", "9.4.1", "--no-daemon"]'), + ); TestValidator.predicate( "a local start process reactivates the complete environment from setup", helpers.includes("export const activateProvisionedTools") && @@ -161,7 +721,7 @@ export const test_experiment_corpora_are_commit_pinned = () => { "isolated lifecycle edges can prove a pinned corpus relationship claim", [ [java, "instantiates"], - [kotlin, "references"], + [kotlin, "calls"], ].every( ([row, kind]) => row!.includes(`crossFileEdge: "${kind}"`) && @@ -189,10 +749,16 @@ export const test_experiment_corpora_are_commit_pinned = () => { runner.includes("semanticLimitation: experiment.semanticLimitation"), ); TestValidator.predicate( - "every producer with no grounded edge family states that limitation explicitly", - csharp.includes("semanticEdges: []") && - !csharp.includes("crossFileEdge:") && - declares(csharp, "semanticLimitation") && + "the C# experiment proves the resident compiler route and its edit bounds", + csharp.includes('strictProvider: "roslyn-workspace"') && + csharp.includes('strictAuthority: "compiler"') && + csharp.includes('strictTool: "samchon-roslyn"') && + csharp.includes('crossFileEdge: "accesses"') && + csharp.includes("noopP95MaxMs: 250") && + csharp.includes("editP95MaxMs: 2000") && + csharp.includes('failurePolicy: "reject"') && + csharpSetup.includes('"publish"') && + csharpSetup.includes("SAMCHON_GRAPH_ROSLYN_WORKSPACE") && runner.includes("experiment.semanticEdges.length === 0") && runner.includes("crossFileEdge !== undefined") && runner.includes("semanticLimitation.trim() ==="), @@ -202,10 +768,8 @@ export const test_experiment_corpora_are_commit_pinned = () => { [cpp, c].every( (row) => row.includes('strictProvider: "clangd-snapshot"') && - row.includes( - 'producerRepository: "https://github.com/samchon/llvm-project.git"', - ) && - row.includes(`producerCommit: "${CPP_CLANG_PRODUCER_COMMIT}"`) && + row.includes("producerRepository: CLANG_PRODUCER_REPOSITORY") && + row.includes("producerCommit: CLANG_PRODUCER_COMMIT") && row.includes('crossFileEdge: "references"') && row.includes('"contains"') && row.includes('"references"') && @@ -215,29 +779,40 @@ export const test_experiment_corpora_are_commit_pinned = () => { ) && !c.includes('"instantiates"') && !c.includes('"extends"') && - !c.includes('"overrides"'), + !c.includes('"overrides"') && + clangProducer.includes( + '"https://github.com/samchon/llvm-project.git"', + ) && + clangProducer.includes(`"${CPP_CLANG_PRODUCER_COMMIT}"`) && + clangProducer.includes("assertClangProducerAdapterPin()") && + setup.includes("installClangGraphProducer({"), ); TestValidator.predicate( "C and C++ build and record the exact campaign-owned native producer", - cppSetup.includes('apt(["clang", "cmake", "ninja-build", "bear"])') && - cppSetup.includes("installClangGraphProducer()") && - setup.includes( - '["fetch", "--depth=1", "origin", experiment.producerCommit]', + cppSetup.includes("CLANG_PRODUCER_BUILD_PACKAGES") && + cppSetup.includes("installClangGraphProducer({") && + clangProducer.includes( + '["fetch", "--depth=1", "origin", CLANG_PRODUCER_COMMIT]', + ) && + clangProducer.includes('["checkout", "--detach", "FETCH_HEAD"]') && + clangProducer.includes('["rev-parse", "HEAD"]') && + clangProducer.includes( + '"-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra"', + ) && + clangProducer.includes('"--target",') && + clangProducer.includes('"clangd",') && + clangProducer.includes( + 'for (const command of ["samchon-clangd", "clangd"])', ) && - setup.includes('["checkout", "--detach", "FETCH_HEAD"]') && - setup.includes('["rev-parse", "HEAD"]') && - setup.includes('"-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra"') && - setup.includes('"--target",') && - setup.includes('"clangd",') && - setup.includes('for (const command of ["samchon-clangd", "clangd"])') && - setup.includes("fs.linkSync(binary, link)") && - setup.includes('path.join(build, "lib", "clang")') && - setup.includes("fs.cpSync(builtResources, installedResources") && - setup.includes('"include",') && - setup.includes('"stddef.h",') && - setup.includes("version.includes(experiment.producerCommit)") && - setup.includes("installedVersion.includes(experiment.producerCommit)") && - setup.includes('tool: "samchon-clangd"') && + clangProducer.includes("fs.linkSync(binary, link)") && + clangProducer.includes('path.join(build, "lib", "clang")') && + clangProducer.includes( + "fs.cpSync(builtResources, installedResources", + ) && + clangProducer.includes('"include",') && + clangProducer.includes('"stddef.h",') && + clangProducer.includes("version.includes(CLANG_PRODUCER_COMMIT)") && + clangProducer.includes('tool: "samchon-clangd"') && !cppSetup.includes('apt(["clangd"'), ); // A fixed parallelism here already cost CI lanes, and the size of this build @@ -260,7 +835,11 @@ export const test_experiment_corpora_are_commit_pinned = () => { // actually happened here and the nearest spellings of them; it cannot // enumerate every way to reintroduce a constant. const clangBuild = withoutLineComments( - region(setup, "const installClangGraphProducer", "const installScipPython"), + region( + clangProducer, + "export function installClangGraphProducer", + "function installedClangGraphProducer", + ), ); TestValidator.equals( "the native Clang build is sized by the machine and bounded by its memory", @@ -289,18 +868,20 @@ export const test_experiment_corpora_are_commit_pinned = () => { // it, so the region is what makes a deletion visible. const restoredProducer = withoutLineComments( region( - setup, - "const installedClangGraphProducer", - "const installClangGraphProducer", + clangProducer, + "function installedClangGraphProducer", + "function assertVersion", ), ); TestValidator.equals( "a restored native Clang producer is re-proved against the pin before reuse", [ - setup.includes("if (installedClangGraphProducer()) return;"), - cppSetup.includes("installClangGraphProducer()"), - /String\(reported\.stdout\)\.includes\(\s*experiment\.producerCommit,?\s*\)/u.test( - restoredProducer, + /const installed = installedClangGraphProducer\(\{\s*toolsRoot,\s*binRoot,\s*platform,\s*\}\);/u.test( + clangProducer, + ), + cppSetup.includes("installClangGraphProducer({"), + restoredProducer.includes( + 'for (const binary of [installed, alias]) assertVersion("cache", binary)', ), restoredProducer.includes('"stddef.h"'), restoredProducer.includes("versions.length !== 1"), @@ -332,10 +913,8 @@ export const test_experiment_corpora_are_commit_pinned = () => { // fact plane and the source manifest come back byte-identical and only the // build universe moves. The claim, not the check, was what had to change. TestValidator.predicate( - "a degraded publication is distinct from an unchanged tolerated one", - csharp.includes('failurePolicy: "published"') && - declares(csharp, "failureLimitation") && - [lua, python].every( + "the lifecycle keeps degraded and tolerated publication policies distinct", + [lua, python].every( (row) => row.includes('failurePolicy: "tolerated"') && declares(row, "failureLimitation"), @@ -364,17 +943,12 @@ export const test_experiment_corpora_are_commit_pinned = () => { TestValidator.predicate( "native C and C++ regeneration stays reproducible", [cpp, c].every((row) => !declares(row, "regenerationLimitation")) && - // Counted over the whole catalog so any reproduction exemption requires - // a reviewed contract change here. There is exactly one, and this is the - // review: scip-java digests its build universe from the raw javac - // invocation, and that invocation names the per-run temporary directory - // its embedded plugin jar is unpacked into. An unchanged checkout comes - // back with identical facts under a different universe — five nodes and - // eight edges both times — so the exemption is about the producer's - // universe rather than about its facts, and it says so. - [...catalog.matchAll(/regenerationLimitation:/g)].length === 1 && - declares(java, "regenerationLimitation") && - java.includes("temporary directory its embedded plugin jar") && + // Counted over the whole catalog so a future exemption requires a + // reviewed contract change here. The pinned Java producer now hashes + // plugin bytes and tags only its transient scratch path, so it returns to + // the same strongest assertion as C/C++. + [...catalog.matchAll(/regenerationLimitation:/g)].length === 0 && + !declares(java, "regenerationLimitation") && runner.includes("experiment.regenerationLimitation !== undefined") && runner.includes("regenerationLimitation.trim() === \"\"") && runner.includes( @@ -384,15 +958,14 @@ export const test_experiment_corpora_are_commit_pinned = () => { lifecycle.includes("!reproduced && limitation === undefined") && lifecycle.includes('name: "regeneration"'), ); - // Sixteen product languages, thirteen strict rows. The other three are - // decisions with evidence behind them, not lanes nobody reached, and a + // Sixteen product languages, fifteen strict rows. The remaining one is a + // decision with evidence behind it, not a lane nobody reached, and a // bounded generic row that passes on node counts cannot tell a reader which // it is. So the absence of a strict provider is itself a declaration. TestValidator.predicate( "every language without a strict provider states what blocks one", - [swift, scala, zig].every((row) => - declares(row, "feasibilityBlocked"), - ) && + declares(zig, "feasibilityBlocked") && + !declares(swift, "feasibilityBlocked") && runner.includes('typeof experiment.feasibilityBlocked !== "string"') && runner.includes("feasibilityBlocked: experiment.feasibilityBlocked"), ); @@ -548,6 +1121,21 @@ export const test_experiment_corpora_are_commit_pinned = () => { runner.includes("provenance.facts.includes(crossFileEdge)") && runner.includes("tools: toolManifest(experiment.language)"), ); + TestValidator.predicate( + "real-provider artifacts compact complete coverage and stable unresolved reasons", + runner.includes("coverageSummary: summarizeCoverage(dump, provenance?.provider)") && + runner.includes( + "unresolvedSummary: summarizeUnresolved(dump, provenance?.provider)", + ) && + evidenceSummary.includes("GRAPH_EDGE_KINDS.map") && + evidenceSummary.includes('row.state === "complete"') && + evidenceSummary.includes('row.state === "partial"') && + evidenceSummary.includes('row.state === "unsupported"') && + evidenceSummary.includes('"identity-unstable"') && + evidenceSummary.includes('"provider-gap"') && + !evidenceSummary.includes("site.evidence") && + !evidenceSummary.includes("site.candidates"), + ); // A digest over the root archive proves nothing while installation still // resolves that archive's dependency ranges against whatever the registry @@ -588,6 +1176,118 @@ export const test_experiment_corpora_are_commit_pinned = () => { ); }; +/** Exercise both the accepting and rejecting cleanup paths of the tree pin. */ +function verifyGitTreeFixture(): void { + const source = fs.mkdtempSync( + path.join(os.tmpdir(), "samchon-graph-git-tree-"), + ); + const repository = path.join(source, ".git"); + try { + fs.writeFileSync(path.join(source, "fixture.txt"), "graph snapshot\n"); + verifyGitTree(source, "40c24dc91a696208881f6616618948ca18f05a92"); + TestValidator.equals( + "successful Git tree verification removes its temporary repository", + fs.existsSync(repository), + false, + ); + TestValidator.error("a mismatched Git tree is rejected", () => + verifyGitTree(source, "0000000000000000000000000000000000000000"), + ); + TestValidator.equals( + "failed Git tree verification removes its temporary repository", + fs.existsSync(repository), + false, + ); + } finally { + fs.rmSync(source, { force: true, recursive: true }); + } +} + +/** Exercise latest-report selection and path-free invalidation evidence. */ +function verifyKotlinBuildReportFixture(): void { + const root = GraphPaths.createTempDirectory( + "samchon-graph-kotlin-build-report-", + ); + const reports = path.join(root, "reports"); + fs.mkdirSync(reports); + const older = path.join(reports, "older.json"); + const latest = path.join(reports, "latest.json"); + fs.writeFileSync(older, JSON.stringify({ buildOperationRecord: [] })); + fs.writeFileSync( + latest, + JSON.stringify({ + buildOperationRecord: [ + { + path: ":compileKotlin", + didWork: true, + totalTimeMs: 12, + changedFiles: { + modifiedFiles: [ + path.join(root, "src", "Main.kt"), + path.resolve(root, "..", "Secret.kt"), + ], + removedFiles: [path.join(root, "src", "Old.kt")], + }, + buildMetrics: { + buildAttributes: { + myAttributes: { + CLASSPATH_SNAPSHOT_NOT_FOUND: 1, + UNUSED: 0, + }, + }, + }, + icLogLines: [ + "Non-incremental compilation will be performed: CLASSPATH_SNAPSHOT_NOT_FOUND", + "Classpath changes info passed from Gradle task: ToBeComputedByIncrementalCompiler", + "Finished executing kotlin compiler using DAEMON strategy", + ], + }, + { + path: ":compileTestKotlin", + didWork: true, + icLogLines: ["Incremental compilation completed"], + }, + ], + }), + ); + fs.utimesSync(older, new Date(1), new Date(1)); + fs.utimesSync(latest, new Date(2), new Date(2)); + + TestValidator.equals( + "Kotlin build reports retain exact compiler invalidation decisions", + captureKotlinBuildReport(root, "reports"), + { + tasks: [ + { + task: ":compileKotlin", + didWork: true, + elapsedMs: 12, + incremental: false, + invalidation: + "Non-incremental compilation will be performed: CLASSPATH_SNAPSHOT_NOT_FOUND", + classpath: + "Classpath changes info passed from Gradle task: ToBeComputedByIncrementalCompiler", + changedFiles: { + modified: ["", "src/Main.kt"], + removed: ["src/Old.kt"], + }, + buildAttributes: ["CLASSPATH_SNAPSHOT_NOT_FOUND"], + daemon: true, + }, + { + task: ":compileTestKotlin", + didWork: true, + incremental: true, + daemon: false, + }, + ], + }, + ); + TestValidator.error("a Kotlin report root cannot escape its project", () => + captureKotlinBuildReport(root, "../outside"), + ); +} + /** * One source region with its line comments removed. * diff --git a/tests/test-graph/src/features/test_graph_snapshot_protocol_commits_atomic_shard_generations.ts b/tests/test-graph/src/features/test_graph_snapshot_protocol_commits_atomic_shard_generations.ts index c49f75b2..51a77b95 100644 --- a/tests/test-graph/src/features/test_graph_snapshot_protocol_commits_atomic_shard_generations.ts +++ b/tests/test-graph/src/features/test_graph_snapshot_protocol_commits_atomic_shard_generations.ts @@ -433,6 +433,159 @@ export const test_graph_snapshot_protocol_commits_atomic_shard_generations = "universe movement disguised as a byte-identical shard upsert", ); + const factStore = new GraphSnapshotProtocol.Store(process.cwd()); + let factValidations = 0; + const validateFacts = () => { + factValidations += 1; + }; + const factInitial = factStore.apply(transaction("fact-generation-1"), { + warnings: ["stable warning"], + validate: validateFacts, + reuseValidatedFacts: true, + }); + const factDelta = transaction("fact-generation-2", { + baseGeneration: "fact-generation-1", + baseSequence: 1, + sourceDigest: "d", + }); + await rejectedWithoutMovement( + factStore, + mutate(factDelta, (frames) => { + commit(frames).factDigest = digest("f"); + }), + "a fact-equivalent delta with a false fact proof", + undefined, + "commit fact digest mismatch", + ); + await rejectedWithoutMovement( + factStore, + mutate(factDelta, (frames) => { + (frames[1] as GraphSnapshotProtocol.IBegin).manifest = digest("f"); + }), + "a fact-equivalent delta with a false input manifest", + undefined, + "input manifest digest mismatch", + ); + const factEdited = factStore.apply(factDelta, { + warnings: ["stable warning"], + validate: validateFacts, + reuseValidatedFacts: true, + }); + TestValidator.predicate( + "a source-only delta advances its protocol and manifest while reusing validated graph facts", + factEdited !== factInitial && + factEdited.nodes === factInitial.nodes && + factEdited.edges === factInitial.edges && + factEdited.coverage === factInitial.coverage && + factEdited.unresolved === factInitial.unresolved && + factEdited.protocol?.generation === "fact-generation-2" && + factEdited.sources.get(path.resolve("src/main.ts"))?.diskDigest === + digest("d") && + factValidations === 1, + ); + + const expandedSourceDelta = transaction("fact-generation-3", { + baseGeneration: "fact-generation-2", + baseSequence: 2, + sourceDigest: "e", + }); + const expandedSource = upsert(expandedSourceDelta, "source"); + expandedSource.shard.sources.push({ + file: path.resolve("tsconfig.json"), + checkerDigest: digest("e"), + diskDigest: digest("e"), + }); + expandedSource.digest = GraphSnapshotProtocol.shardDigest( + expandedSource.shard, + ); + const expandedBegin = expandedSourceDelta[1] as GraphSnapshotProtocol.IBegin; + expandedBegin.manifest = GraphSnapshotProtocol.manifestDigest( + expandedSource.shard.sources, + ); + const expandedCommit = commit(expandedSourceDelta); + expandedCommit.shards = factEdited.protocol!.shards.map((entry) => + entry.key === expandedSource.shard.key + ? { key: entry.key, digest: expandedSource.digest } + : { ...entry }, + ); + expandedCommit.factDigest = factEdited.protocol!.factDigest; + const expanded = factStore.apply(expandedSourceDelta, { + warnings: ["stable warning"], + validate: validateFacts, + reuseValidatedFacts: true, + }); + TestValidator.predicate( + "source membership movement takes the ordinary validation path", + factValidations === 2 && + expanded.nodes !== factEdited.nodes && + expanded.sources.has(path.resolve("tsconfig.json")), + ); + + const warningDelta = transaction("fact-generation-4", { + baseGeneration: "fact-generation-3", + baseSequence: 3, + sourceDigest: "f", + }); + const warningEdited = factStore.apply(warningDelta, { + warnings: ["moved warning"], + validate: validateFacts, + reuseValidatedFacts: true, + }); + TestValidator.predicate( + "warning movement takes the ordinary validation path", + factValidations === 3 && warningEdited.nodes !== expanded.nodes, + ); + factStore.apply( + transaction("fact-generation-5", { + baseGeneration: "fact-generation-4", + baseSequence: 4, + nodeName: "changed fact", + sourceDigest: "a", + }), + { + warnings: ["moved warning"], + validate: validateFacts, + reuseValidatedFacts: true, + }, + ); + TestValidator.equals( + "fact movement takes the ordinary validation path", + factValidations, + 4, + ); + + const conflictStore = new GraphSnapshotProtocol.Store(process.cwd()); + const conflictInitial = transaction("conflict-generation-1"); + coverageShard(conflictInitial).shard.sources.push({ + file: path.resolve("src/main.ts"), + checkerDigest: digest("c"), + diskDigest: digest("c"), + }); + refreshDigests(conflictInitial); + conflictStore.apply(conflictInitial, { reuseValidatedFacts: true }); + const conflictingDelta = transaction("conflict-generation-2", { + baseGeneration: "conflict-generation-1", + baseSequence: 1, + sourceDigest: "d", + }); + const conflictingSource = upsert(conflictingDelta, "source"); + commit(conflictingDelta).shards = conflictStore.current!.protocol!.shards.map( + (entry) => + entry.key === "source" + ? { key: entry.key, digest: conflictingSource.digest } + : { ...entry }, + ); + commit(conflictingDelta).factDigest = + conflictStore.current!.protocol!.factDigest; + TestValidator.predicate( + "fact-equivalent shards may not disagree about the bytes of one source", + errorMessage(() => + conflictStore.apply(conflictingDelta, { + reuseValidatedFacts: true, + }), + ).includes("shards disagree about source"), + ); + const boundedGenerationStore = new GraphSnapshotProtocol.Store(process.cwd()); boundedGenerationStore.apply(transaction("bounded-generation-1")); boundedGenerationStore.apply( @@ -1526,14 +1679,33 @@ async function rejectedWithoutMovement( frames: readonly GraphSnapshotProtocol.Frame[], label: string, signal?: AbortSignal, + expected?: string, ): Promise { const before = store.current; - await TestValidator.error(`${label} rejects`, () => - store.apply(frames, { signal }), + const message = errorMessage(() => + store.apply(frames, { + signal, + ...(expected === undefined + ? {} + : { reuseValidatedFacts: true, warnings: ["stable warning"] }), + }), + ); + TestValidator.predicate( + `${label} rejects`, + message.length !== 0 && (expected === undefined || message.includes(expected)), ); TestValidator.predicate(`${label} retains the prior generation`, store.current === before); } +function errorMessage(task: () => unknown): string { + try { + task(); + return ""; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } +} + function record(value: object): Record { return value as Record; } diff --git a/tests/test-graph/src/features/test_java_regeneration_failure_names_compiler_input.ts b/tests/test-graph/src/features/test_java_regeneration_failure_names_compiler_input.ts new file mode 100644 index 00000000..d93d802b --- /dev/null +++ b/tests/test-graph/src/features/test_java_regeneration_failure_names_compiler_input.ts @@ -0,0 +1,224 @@ +import { TestValidator } from "@nestia/e2e"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + captureGenerationEvidence, + firstEvidenceDifference, +} from "../../../experiment/src/regeneration-evidence.mjs"; + +/** Java regeneration diagnostics must expose the exact producer input row. */ +export const test_java_regeneration_failure_names_compiler_input = () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "graph-java-universe-")); + try { + const store = path.join(root, "target", "scip-targetroot"); + const coldTarget = path.join( + store, + "META-INF", + "scip-graph-store", + "targets", + "a".repeat(64), + ); + const coldCurrent = path.join(coldTarget, "CURRENT"); + const coldGeneration = "b".repeat(64); + const coldCommitted = path.join( + coldTarget, + "generations", + coldGeneration, + ); + const coldInvocation = path.join( + coldCommitted, + ".universe", + `${"c".repeat(64)}.args.d`, + `${"d".repeat(64)}.args`, + ); + fs.mkdirSync(path.dirname(coldInvocation), { recursive: true }); + fs.writeFileSync(coldCurrent, `${coldGeneration}\n`); + fs.writeFileSync( + path.join(coldCommitted, "UNIVERSE"), + "java.version=21\ncompilerTarget=first\n", + ); + fs.writeFileSync( + coldInvocation, + [ + "@invocation", + "@plugin", + Buffer.from("e".repeat(64)).toString("base64url"), + encodedIdentity("-classpath", `${"x".repeat(520)}first.jar`), + "", + ].join("\n"), + ); + + const cold = captureGenerationEvidence(root, "target/scip-targetroot"); + TestValidator.predicate( + "normalized compiler input is decoded", + cold.some((row: string) => row.includes("first.jar")), + ); + + fs.rmSync(coldTarget, { force: true, recursive: true }); + const retryTarget = path.join( + store, + "META-INF", + "scip-graph-store", + "targets", + "7".repeat(64), + ); + const retryCurrent = path.join(retryTarget, "CURRENT"); + const retryGeneration = "f".repeat(64); + const retryCommitted = path.join( + retryTarget, + "generations", + retryGeneration, + ); + const retryInvocation = path.join( + retryCommitted, + ".universe", + `${"8".repeat(64)}.args.d`, + `${"9".repeat(64)}.args`, + ); + fs.mkdirSync(path.dirname(retryInvocation), { recursive: true }); + fs.writeFileSync(retryCurrent, `${retryGeneration}\n`); + fs.writeFileSync( + path.join(retryCommitted, "UNIVERSE"), + "java.version=21\ncompilerTarget=retry\n", + ); + fs.writeFileSync( + retryInvocation, + [ + "@invocation", + "@plugin", + Buffer.from("e".repeat(64)).toString("base64url"), + encodedIdentity("-classpath", `${"x".repeat(520)}retry.jar`), + "", + ].join("\n"), + ); + const retry = captureGenerationEvidence(root, "target/scip-targetroot"); + const difference = firstEvidenceDifference(cold, retry); + TestValidator.predicate( + "first moved compiler input is actionable", + difference.includes("first.jar") && difference.includes("retry.jar"), + ); + TestValidator.equals( + "added compiler input is named", + firstEvidenceDifference(["row-a"], ["row-a", "row-b"]), + "missing -> row-b", + ); + TestValidator.equals( + "duplicate compiler input count is preserved", + firstEvidenceDifference(["row-a", "row-a"], ["row-a"]), + "row-a -> missing", + ); + const longPrefix = "x".repeat(500); + const longDifference = firstEvidenceDifference( + [`${longPrefix}OLD`], + [`${longPrefix}NEW`], + ); + TestValidator.predicate( + "difference after the diagnostic bound stays visible", + longDifference.includes("OLD") && + longDifference.includes("NEW") && + longDifference.includes("...") && + longDifference.length <= 964, + ); + const unicodePrefix = "😀".repeat(250); + const unicodeDifference = firstEvidenceDifference( + [`${unicodePrefix}OLD!${unicodePrefix}`], + [`${unicodePrefix}NEW!${unicodePrefix}`], + ); + TestValidator.predicate( + "bounded diagnostics preserve Unicode code points", + unicodeDifference.includes("OLD") && + unicodeDifference.includes("NEW") && + unicodeDifference.includes("...") && + unicodeDifference.length <= 964 && + hasNoLoneSurrogate(unicodeDifference), + ); + const addedUnicode = firstEvidenceDifference( + [], + [`row:${"😀".repeat(300)}`], + ); + TestValidator.predicate( + "bounded added input preserves Unicode code points", + addedUnicode.startsWith("missing -> ") && + addedUnicode.includes("...") && + addedUnicode.length <= 491 && + hasNoLoneSurrogate(addedUnicode), + ); + const addedInvocation = `compiler invocation:${"y".repeat(700)}retry.jar`; + const addedWithUniverse = firstEvidenceDifference( + ["generation universe:compilerTarget=old"], + [addedInvocation, "generation universe:compilerTarget=new"], + ); + TestValidator.predicate( + "added invocation takes precedence over its opaque universe digest", + addedWithUniverse.startsWith("missing -> ") && + addedWithUniverse.includes("retry.jar"), + ); + const removedWithUniverse = firstEvidenceDifference( + [addedInvocation, "generation universe:compilerTarget=old"], + ["generation universe:compilerTarget=new"], + ); + TestValidator.predicate( + "removed invocation takes precedence over its opaque universe digest", + removedWithUniverse.endsWith(" -> missing") && + removedWithUniverse.includes("retry.jar"), + ); + TestValidator.error( + "store root cannot escape the isolated corpus", + () => captureGenerationEvidence(root, ".."), + Error, + ); + const foreign = fs.mkdtempSync( + path.join(os.tmpdir(), "graph-java-universe-foreign-"), + ); + try { + const compiler = path.join(retryCommitted, ".universe"); + fs.rmSync(compiler, { force: true, recursive: true }); + if (process.platform === "win32") { + fs.symlinkSync(foreign, compiler, "junction"); + } else { + fs.symlinkSync(foreign, compiler, "dir"); + } + TestValidator.error( + "compiler universe cannot escape through a link", + () => captureGenerationEvidence(root, "target/scip-targetroot"), + Error, + ); + } finally { + fs.rmSync(foreign, { force: true, recursive: true }); + } + fs.writeFileSync(retryCurrent, "not-a-generation\n"); + TestValidator.error( + "malformed current generation is rejected", + () => captureGenerationEvidence(root, "target/scip-targetroot"), + Error, + ); + } finally { + fs.rmSync(root, { force: true, recursive: true }); + } +}; + +function encodedIdentity(prefix: string, suffix: string): string { + const literal = (value: string): string => + Buffer.from(value, "utf8").toString("base64url"); + return Buffer.from( + `v1|literal:${literal(prefix)}|tool|literal:${literal(suffix)}`, + "utf8", + ).toString("base64url"); +} + +function hasNoLoneSurrogate(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + if (index + 1 >= value.length) return false; + const next = value.charCodeAt(index + 1); + if (next < 0xdc00 || next > 0xdfff) return false; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return false; + } + } + return true; +} diff --git a/tests/test-graph/src/features/test_jdt_workspace_graph_is_bulk_atomic_and_fails_closed.ts b/tests/test-graph/src/features/test_jdt_workspace_graph_is_bulk_atomic_and_fails_closed.ts new file mode 100644 index 00000000..8036f0ea --- /dev/null +++ b/tests/test-graph/src/features/test_jdt_workspace_graph_is_bulk_atomic_and_fails_closed.ts @@ -0,0 +1,818 @@ +import { TestValidator } from "@nestia/e2e"; +import { + JDT_GRAPH_PRODUCER_COMMIT, + IJdtGraphSnapshot, + JDT_GRAPH_PROVIDER, + JdtGraphClient, + JdtGraphSnapshotAdapter, + assertGraphSnapshotContract, + jdtGraphProvider, + javaDeclarationSymbol, + semanticGraphNodeId, +} from "@samchon/graph"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { GraphPaths } from "../internal/GraphPaths.js"; + +export const test_jdt_workspace_graph_is_bulk_atomic_and_fails_closed = + async (): Promise => { + const root = GraphPaths.createTempDirectory("samchon-graph-jdt-"); + const source = path.join(root, "src", "Example.java"); + fs.mkdirSync(path.dirname(source), { recursive: true }); + fs.writeFileSync( + source, + "package example; public final class Example { void run() {} }\n", + ); + fs.writeFileSync(path.join(root, "src", "Zed.java"), "class Zed {}\n"); + + await assertClientLifecycle(root, source); + assertAdapterBoundaries(root, source); + }; + +async function assertClientLifecycle( + root: string, + source: string, +): Promise { + const requestLog = path.join(root, "requests.ndjson"); + const marker = path.join(root, "closed.txt"); + let validations = 0; + const client = new JdtGraphClient({ + root, + command: process.execPath, + args: [ + GraphPaths.fakeJdtGraphServer, + `--request-log=${requestLog}`, + `--marker=${marker}`, + ], + initializationOptions: { fixture: true }, + requestTimeoutMs: 10_000, + maxMessageBytes: 16 * 1024 * 1024, + validate: (snapshot) => { + validations += 1; + assertGraphSnapshotContract( + snapshot, + jdtGraphProvider, + ["java"], + root, + ); + }, + }); + const active = new AbortController(); + const initial = await client.refresh({ signal: active.signal }); + const unchanged = await client.refresh(); + TestValidator.equals( + "the JDT client publishes and reuses one compiler-owned workspace generation", + [ + initial.changed, + initial.mode, + initial.generation, + unchanged.changed, + unchanged.mode, + unchanged.snapshot === initial.snapshot, + client.current === initial.snapshot, + client.generation, + initial.snapshot.provenance.provider, + initial.snapshot.provenance.tool, + initial.snapshot.coverage?.length, + initial.snapshot.edges.map((edge) => edge.kind), + initial.snapshot.diagnostics[0]?.severity, + validations, + ], + [ + true, + "initial", + 1, + false, + "unchanged", + true, + true, + 1, + JDT_GRAPH_PROVIDER, + IJdtGraphSnapshot.PRODUCER, + 15, + ["contains", "contains"], + "info", + 1, + ], + ); + TestValidator.predicate( + "JDT declaration modifiers and structural ownership survive adaptation", + initial.snapshot.nodes.some( + (node) => + node.kind === "class" && + node.modifiers?.includes("readonly") === true, + ) && initial.snapshot.nodes.some((node) => node.closure === true), + ); + + fs.writeFileSync(path.join(root, "src", "A.java"), "class A {}\n"); + fs.appendFileSync(source, "class Added {}\n"); + const incremental = await client.refresh(); + TestValidator.predicate( + "a saved Java edit advances the resident generation incrementally", + incremental.changed && + incremental.mode === "incremental" && + incremental.generation === 2 && + incremental.snapshot !== initial.snapshot, + ); + fs.unlinkSync(source); + const deleted = await client.refresh(); + TestValidator.predicate( + "a deleted Java source leaves the resident generation without stale declarations", + deleted.changed && + deleted.mode === "incremental" && + deleted.generation === 3 && + deleted.snapshot.nodes.length === 0, + ); + fs.writeFileSync( + source, + "package example; public final class Example { void run() {} }\n", + ); + const messages = fs + .readFileSync(requestLog, "utf8") + .trim() + .split(/\r?\n/u) + .map((line) => JSON.parse(line) as { method?: string; params?: { command?: string } }); + TestValidator.equals( + "each refresh asks for one bulk snapshot and never fans out by declaration", + [ + messages.filter( + (message) => + message.method === "workspace/executeCommand" && + message.params?.command === "java.graph.snapshot", + ).length, + messages.some((message) => message.method === "textDocument/references"), + messages.some((message) => message.method === "textDocument/documentSymbol"), + messages.filter( + (message) => message.method === "workspace/didChangeWatchedFiles", + ).length, + ], + [4, false, false, 3], + ); + + await Promise.all([client.close(), client.close()]); + TestValidator.equals( + "the JDT process closes through the LSP handshake", + fs.readFileSync(marker, "utf8"), + "closed", + ); + await rejected( + "a closed JDT session refuses refresh", + client.refresh(), + "session is closed", + ); + + const cancelled = new JdtGraphClient({ + root, + command: process.execPath, + args: [GraphPaths.fakeJdtGraphServer], + validate: () => undefined, + }); + const controller = new AbortController(); + controller.abort(new Error("caller stopped")); + await rejected( + "a pre-cancelled JDT refresh never enters the queue", + cancelled.refresh({ signal: controller.signal }), + "caller stopped", + ); + await cancelled.close(); + + await assertQueueCancellation(root, source); + await assertInFlightCancellation(root); + await assertInitializationFailure(root); + await assertCommandCancellationRecovery(root); + await assertInputMovementFence(root, source); + await assertNonErrorValidationFailure(root); + await assertRegisteredProvider(root); +} + +async function assertQueueCancellation(root: string, source: string): Promise { + const client = directClient(root, ["--delay-command=100"]); + const first = client.refresh(); + const controller = new AbortController(); + const queued = client.refresh({ signal: controller.signal }); + controller.abort(new Error("queued stop")); + await rejected("a queued JDT refresh cancels before it starts", queued, "queued stop"); + await first; + await client.close(); + TestValidator.predicate("the queue-cancellation fixture preserves its source", fs.existsSync(source)); +} + +async function assertInFlightCancellation(root: string): Promise { + const client = directClient(root, ["--delay-initialize=100"]); + const controller = new AbortController(); + const pending = client.refresh({ signal: controller.signal }); + setTimeout(() => controller.abort(new Error("in-flight stop")), 10); + await rejected("an in-flight JDT initialization observes caller cancellation", pending, "in-flight stop"); + await new Promise((resolve) => setTimeout(resolve, 120)); + await client.close(); +} + +async function assertInitializationFailure(root: string): Promise { + const client = directClient(root, ["--fail-initialize"]); + const controller = new AbortController(); + await rejected( + "an underlying JDT initialization failure wins the live caller signal race", + client.refresh({ signal: controller.signal }), + "fixture initialize failure", + ); + await client.close(); +} + +async function assertCommandCancellationRecovery(root: string): Promise { + const requestLog = path.join(root, "command-cancel-requests.ndjson"); + const client = directClient(root, [ + "--delay-command=100", + `--request-log=${requestLog}`, + ]); + const controller = new AbortController(); + const pending = client.refresh({ signal: controller.signal }); + await waitForRequest(requestLog, "workspace/executeCommand"); + controller.abort(new Error("command stop")); + await rejected( + "an in-flight JDT command observes caller cancellation", + pending, + "LSP request aborted: workspace/executeCommand", + ); + await new Promise((resolve) => setTimeout(resolve, 120)); + const recovered = await client.refresh(); + TestValidator.equals( + "a full unchanged producer snapshot resynchronizes after command cancellation", + [ + recovered.changed, + recovered.mode, + recovered.generation, + recovered.snapshot.protocol?.sequence, + requestCount(requestLog, "workspace/didChangeWatchedFiles"), + ], + [true, "initial", 1, 1, 2], + ); + await client.close(); +} + +async function assertInputMovementFence(root: string, source: string): Promise { + const requestLog = path.join(root, "input-fence-requests.ndjson"); + const client = directClient(root, [ + "--reuse-after-change", + `--request-log=${requestLog}`, + ]); + await client.refresh(); + fs.appendFileSync(source, "// moved behind producer\n"); + await rejected( + "a producer cannot reuse a generation after a watched source moves", + client.refresh(), + "watched Java inputs moved", + ); + await rejected( + "a stale producer remains fenced until it accepts the moved input", + client.refresh(), + "watched Java inputs moved", + ); + TestValidator.equals( + "the stale input notification is repeated until a generation accepts it", + requestCount(requestLog, "workspace/didChangeWatchedFiles"), + 3, + ); + fs.writeFileSync( + source, + "package example; public final class Example { void run() {} }\n", + ); + await client.close(); +} + +async function assertNonErrorValidationFailure(root: string): Promise { + const requestLog = path.join(root, "validation-retry-requests.ndjson"); + let first = true; + const client = directClient(root, [`--request-log=${requestLog}`], () => { + if (!first) return; + first = false; + throw "fixture string failure"; + }); + await rejected( + "a non-Error validation failure is still surfaced as an Error", + client.refresh(), + "fixture string failure", + ); + const recovered = await client.refresh(); + TestValidator.equals( + "a full unchanged producer snapshot resynchronizes after validation rejection", + [ + recovered.changed, + recovered.mode, + recovered.generation, + recovered.snapshot.protocol?.sequence, + requestCount(requestLog, "workspace/didChangeWatchedFiles"), + ], + [true, "initial", 1, 1, 2], + ); + await client.close(); +} + +async function assertRegisteredProvider(root: string): Promise { + const unconfigured = jdtGraphProvider.configuration?.(root, {}); + const configuration = jdtGraphProvider.configuration?.(root, { + SAMCHON_GRAPH_JDT_WORKSPACE: process.execPath, + }); + const resolved = jdtGraphProvider.resolve(root, { + SAMCHON_GRAPH_JDT_WORKSPACE: process.execPath, + }); + TestValidator.predicate( + "the registered JDT route publishes its exact producer and override inputs", + configuration?.[0] === `producer-commit=${JDT_GRAPH_PRODUCER_COMMIT}` && + configuration[1] === `SAMCHON_GRAPH_JDT_WORKSPACE=${process.execPath}` && + unconfigured?.[1] === "SAMCHON_GRAPH_JDT_WORKSPACE=unconfigured" && + resolved?.command === process.execPath, + ); + TestValidator.predicate( + "the registered JDT route accepts only whole-workspace requests", + jdtGraphProvider.refuse({ cwd: root }) === undefined && + jdtGraphProvider + .refuse({ + cwd: root, + server: "jdtls", + maxFiles: 1, + lspReferenceLimit: 1, + }) + ?.includes("server, maxFiles, lspReferenceLimit") === true, + ); + const session = jdtGraphProvider.open({ + root, + command: { + command: process.execPath, + args: [GraphPaths.fakeJdtGraphServer], + }, + languages: ["java"], + options: { cwd: root }, + }); + try { + TestValidator.equals( + "the registered JDT route enforces its compiler contract", + (await session.refresh()).snapshot.provenance.provider, + JDT_GRAPH_PROVIDER, + ); + } finally { + await session.close(); + } +} + +function directClient( + root: string, + flags: string[], + validate: ConstructorParameters[0]["validate"] = () => + undefined, +): JdtGraphClient { + return new JdtGraphClient({ + root, + command: process.execPath, + args: [GraphPaths.fakeJdtGraphServer, ...flags], + validate, + }); +} + +function assertAdapterBoundaries(root: string, source: string): void { + fs.writeFileSync( + path.join(root, "pom.xml"), + "4.0.0\n", + ); + const adapter = new JdtGraphSnapshotAdapter(root); + const initial = rawSnapshot(root, source); + const published = adapter.apply(initial, { + validate: (snapshot) => + assertGraphSnapshotContract( + snapshot, + jdtGraphProvider, + ["java"], + root, + ), + }); + const singleSlash = rawSnapshot(root, source); + const projectUri = singleSlashFileUri(root); + const sourceUri = singleSlashFileUri(source); + singleSlash.projects[0]!.location = projectUri; + singleSlash.sources[0]!.uri = sourceUri; + for (const node of singleSlash.nodes) { + node.uri = sourceUri; + node.evidence.uri = sourceUri; + } + for (const edge of singleSlash.edges) edge.evidence.uri = sourceUri; + for (const diagnostic of singleSlash.diagnostics) { + diagnostic.uri = sourceUri; + diagnostic.evidence.uri = sourceUri; + } + TestValidator.predicate( + "JDT single-slash file URIs resolve to the project files they name", + new JdtGraphSnapshotAdapter(root) + .apply(singleSlash) + .snapshot.sources.has(path.normalize(source)), + ); + const unchanged = structuredClone(initial); + unchanged.mode = "unchanged"; + TestValidator.predicate( + "the adapter reuses the exact object for an unchanged producer generation", + adapter.apply(unchanged).snapshot === published.snapshot, + ); + TestValidator.predicate( + "the shared Java declaration key is signature-aware without using positions", + javaDeclarationSymbol({ + kind: "method", + name: "run", + qualifiedName: "example.Example.run", + signature: "(int):void", + }).endsWith("|int") && + javaDeclarationSymbol({ + kind: "method", + name: "run", + qualifiedName: "example.Example.run(int)", + displayName: "run(java.lang.String)", + }).endsWith("|java.lang.String") && + javaDeclarationSymbol({ + kind: "class", + name: "Example", + }).endsWith("|Example|") && + javaDeclarationSymbol({ + kind: "method", + name: "run", + displayName: "run", + }).endsWith("|run|") && + javaDeclarationSymbol({ + kind: "method", + name: "run", + signature: "run(", + }).endsWith("|run|"), + ); + + const signed = rawSnapshot(root, source); + signed.nodes[1]!.signature = "class Example"; + const signedSnapshot = new JdtGraphSnapshotAdapter(root).apply(signed).snapshot; + const moduleRoot = path.join(root, "module"); + const moduleSource = path.join(moduleRoot, "src", "Example.java"); + fs.mkdirSync(path.dirname(moduleSource), { recursive: true }); + fs.writeFileSync(moduleSource, "package example; class Example {}\n"); + fs.writeFileSync( + path.join(moduleRoot, "pom.xml"), + "4.0.0\n", + ); + const modular = rawSnapshot(root, moduleSource); + modular.projects[0]!.location = pathToFileURL(moduleRoot).href; + const moduleSnapshot = new JdtGraphSnapshotAdapter(root).apply(modular).snapshot; + TestValidator.predicate( + "persistent signatures and nested Maven project coordinates enter JDT identity", + signedSnapshot.nodes.some( + (node) => node.kind === "class" && node.signature === "class Example", + ) && + moduleSnapshot.nodes.find((node) => node.kind === "class")?.id !== + published.snapshot.nodes.find((node) => node.kind === "class")?.id, + ); + + const gradleRoot = path.join(root, "gradle"); + fs.mkdirSync(gradleRoot, { recursive: true }); + fs.writeFileSync( + path.join(gradleRoot, "settings.gradle"), + "rootProject.name = 'fixture'\ninclude 'module'\n", + ); + fs.writeFileSync(path.join(gradleRoot, "build.gradle"), "plugins { id 'java' }\n"); + const gradleMain = path.join( + gradleRoot, + "src", + "main", + "java", + "Example.java", + ); + const gradleTest = path.join( + gradleRoot, + "src", + "test", + "java", + "Example.java", + ); + const gradleModule = path.join( + gradleRoot, + "module", + "src", + "main", + "java", + "Example.java", + ); + for (const file of [gradleMain, gradleTest, gradleModule]) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, "package example; class Example {}\n"); + } + fs.writeFileSync( + path.join(gradleRoot, "module", "build.gradle"), + "plugins { id 'java' }\n", + ); + const gradleCases = [ + [gradleMain, ":compileJava"], + [gradleTest, ":compileTestJava"], + [gradleModule, ":module:compileJava"], + ] as const; + const gradleIds = gradleCases.map(([file]) => { + const raw = rawSnapshot(gradleRoot, file); + if (file === gradleModule) { + raw.projects[0]!.location = pathToFileURL( + path.join(gradleRoot, "module"), + ).href; + } + const snapshot = new JdtGraphSnapshotAdapter(gradleRoot).apply(raw).snapshot; + return snapshot.nodes.find((node) => node.kind === "class")?.id; + }); + TestValidator.equals( + "JDT derives the exact standard Gradle main, test and subproject task scopes", + gradleIds, + gradleCases.map(([, target]) => gradleJavaId(target)), + ); + const gradleVariants = [ + ["settings-kts", "settings.gradle.kts", "src/main/java", ":compileJava"], + ["build-only", "build.gradle", "src/main/java", ":compileJava"], + ["build-kts-only", "build.gradle.kts", "src/test/java", ":compileTestJava"], + ["custom-source", "build.gradle", "generated/java", "jdt:fixture"], + ] as const; + const variantIds = gradleVariants.map( + ([name, buildFile, sourceDirectory]) => { + const variantRoot = path.join(root, name); + const variantSource = path.join( + variantRoot, + sourceDirectory, + "Example.java", + ); + fs.mkdirSync(path.dirname(variantSource), { recursive: true }); + fs.writeFileSync(variantSource, "package example; class Example {}\n"); + fs.writeFileSync(path.join(variantRoot, buildFile), "// fixture\n"); + const snapshot = new JdtGraphSnapshotAdapter(variantRoot).apply( + rawSnapshot(variantRoot, variantSource), + ).snapshot; + return snapshot.nodes.find((node) => node.kind === "class")?.id; + }, + ); + TestValidator.equals( + "JDT recognizes both Gradle DSLs and declines custom task inference", + variantIds, + gradleVariants.map(([, , , target]) => gradleJavaId(target)), + ); + + const sameButIncremental = structuredClone(initial); + sameButIncremental.mode = "incremental"; + TestValidator.predicate( + "consumer history makes a repeated generation unchanged after producer cursor drift", + adapter.apply(sameButIncremental).snapshot === published.snapshot, + ); + const movedButUnchanged = movedSnapshot(initial); + movedButUnchanged.mode = "unchanged"; + const resynchronized = adapter.apply(movedButUnchanged); + TestValidator.predicate( + "a full producer snapshot resynchronizes an unseen same-universe generation", + resynchronized.changed && resynchronized.mode === "incremental", + ); + + const broken = structuredClone(initial); + broken.complete = false; + broken.mode = "error"; + broken.diagnostics = [ + { + uri: pathToFileURL(source).href, + severity: "error", + code: "broken", + message: "broken resident buffer", + evidence: evidence(source), + }, + ]; + refused( + "an erroneous resident model retains the prior strict generation", + () => adapter.apply(broken), + "retained the prior strict generation", + ); + TestValidator.predicate( + "a refused JDT generation leaves current untouched", + adapter.current === resynchronized.snapshot, + ); + + const moved = movedSnapshot(initial, "generation-three"); + moved.universe = digest("universe-two"); + const next = adapter.apply(moved); + TestValidator.predicate( + "a valid moved producer generation replaces current atomically", + next.changed && next.mode === "reload" && adapter.current === next.snapshot, + ); + + const cases: Array<[string, (raw: IJdtGraphSnapshot) => unknown, string]> = [ + ["wrong schema", (raw) => (raw.schemaVersion = 2), "malformed producer snapshot"], + ["wrong capability", (raw) => (raw.capabilities.resident = false), "incompatible capabilities"], + ["duplicate project", (raw) => raw.projects.push(structuredClone(raw.projects[0]!)), "project universe"], + ["bad source encoding", (raw) => (raw.sources[0]!.checkerEncoding = "utf8"), "source manifest"], + ["duplicate symbol", (raw) => raw.nodes.push(structuredClone(raw.nodes[0]!)), "declaration"], + ["absent edge endpoint", (raw) => (raw.edges[0]!.to = "missing"), "containment edge"], + ["bad diagnostic", (raw) => (raw.diagnostics[0]!.severity = "error"), "completion state"], + ["malformed diagnostic", (raw) => (raw.diagnostics[0]!.code = ""), "malformed diagnostic"], + ["malformed evidence", (raw) => (raw.nodes[0]!.evidence = null as never), "malformed declaration"], + ["foreign evidence", (raw) => { + raw.nodes[0]!.evidence.uri = pathToFileURL(path.join(root, "src", "foreign.java")).href; + }, "malformed declaration"], + ["unresolved row", (raw) => raw.unresolved.push({}), "malformed producer snapshot"], + ["escaped source", (raw) => { + const outside = pathToFileURL(path.join(path.dirname(root), "outside.java")).href; + raw.sources[0]!.uri = outside; + }, "escaped the project root"], + ]; + refused( + "a non-object JDT response is refused", + () => new JdtGraphSnapshotAdapter(root).apply(null), + "not an object", + ); + for (const [name, mutate, message] of cases) { + const raw = rawSnapshot(root, source); + mutate(raw); + refused(name, () => new JdtGraphSnapshotAdapter(root).apply(raw), message); + } +} + +function rawSnapshot(root: string, source: string): IJdtGraphSnapshot { + const uri = pathToFileURL(source).href; + const file = "java/fixture/file/example"; + const type = "java/fixture/type/example.Example"; + const method = `${type}/method/run()`; + const variable = `${method}/variable/value:int`; + const location = evidence(source); + return { + schemaVersion: 1, + protocolVersion: 1, + producer: { + name: IJdtGraphSnapshot.PRODUCER, + version: "1.50.0.fixture", + compilerVersion: "21", + }, + capabilities: { + atomicGenerations: true, + resident: true, + sourceDigests: true, + diskDigests: true, + unsavedBuffers: true, + diagnostics: true, + facts: ["contains"], + }, + universe: digest("universe"), + generation: digest("generation-one"), + complete: true, + mode: "initial", + sequence: 1, + projects: [ + { + name: "fixture", + location: pathToFileURL(root).href, + output: "/fixture/bin", + compilerVersion: "21", + options: {}, + classpath: [], + }, + ], + sources: [ + { + project: "fixture", + uri, + checkerDigest: digest("checker"), + checkerEncoding: IJdtGraphSnapshot.CHECKER_ENCODING, + diskDigest: digest(fs.readFileSync(source)), + }, + ], + nodes: [ + rawNode(file, file, "persistent", uri, "Example.java", "", "file", "", "file", [], location), + rawNode(type, "Lexample/Example;", "persistent", uri, "Example", "example.Example", "class", "", "type", ["public", "final"], location), + rawNode(method, "Lexample/Example;.run()V", "structural", uri, "run", "example.Example.run", "method", "():void", "method", ["public"], location), + rawNode(variable, "local#value", "generation", uri, "value", "", "variable", "int", "variable", [], location), + ], + edges: [ + { from: file, to: type, kind: "contains", evidence: location }, + { from: type, to: method, kind: "contains", evidence: location }, + { from: method, to: variable, kind: "contains", evidence: location }, + ], + diagnostics: [ + { + uri, + severity: "warning", + code: "fixture", + message: "fixture warning", + evidence: location, + }, + ], + coverage: { contains: "complete" }, + unresolved: [], + }; +} + +function movedSnapshot( + raw: IJdtGraphSnapshot, + generation = "generation-two", +): IJdtGraphSnapshot { + const moved = structuredClone(raw); + moved.generation = digest(generation); + moved.mode = "incremental"; + moved.sequence = 2; + return moved; +} + +function gradleJavaId(target: string): string { + const symbol = javaDeclarationSymbol({ + kind: "class", + name: "Example", + qualifiedName: "example.Example", + }); + return semanticGraphNodeId( + { + version: 2, + language: "java", + symbol, + role: "class", + native: { key: symbol, stability: "semantic" }, + scope: { target }, + stability: "persistent", + }, + "example.Example", + ); +} + +function rawNode( + symbol: string, + nativeKey: string, + stability: IJdtGraphSnapshot.INode["stability"], + uri: string, + name: string, + qualifiedName: string, + kind: string, + signature: string, + declarationKind: string, + modifiers: string[], + location: IJdtGraphSnapshot.IEvidence, +): IJdtGraphSnapshot.INode { + return { + project: "fixture", + symbol, + nativeKey, + stability, + uri, + name, + qualifiedName, + kind, + signature, + declarationKind, + exported: modifiers.includes("public"), + modifiers, + evidence: location, + }; +} + +function evidence(source: string): IJdtGraphSnapshot.IEvidence { + return { + uri: pathToFileURL(source).href, + startLine: 1, + startColumn: 1, + endLine: 1, + endColumn: 20, + }; +} + +function singleSlashFileUri(file: string): string { + return pathToFileURL(file).href.replace(/^file:\/\/\//u, "file:/"); +} + +function requestCount(file: string, method: string): number { + return fs + .readFileSync(file, "utf8") + .trim() + .split(/\r?\n/u) + .map((line) => JSON.parse(line) as { method?: string }) + .filter((message) => message.method === method).length; +} + +async function waitForRequest(file: string, method: string): Promise { + const deadline = performance.now() + 5_000; + while (performance.now() < deadline) { + if (fs.existsSync(file) && requestCount(file, method) !== 0) return; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error(`fake JDT server did not receive ${method}`); +} + +function digest(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function refused(name: string, closure: () => unknown, message: string): void { + TestValidator.error(name, closure, (error) => + TestValidator.predicate(name, String(error).includes(message)), + ); +} + +async function rejected( + name: string, + promise: Promise, + message: string, +): Promise { + try { + await promise; + } catch (error) { + TestValidator.predicate(name, String(error).includes(message)); + return; + } + throw new Error(`${name}: expected rejection`); +} diff --git a/tests/test-graph/src/features/test_kotlinc_graph_declines_a_launcher_without_graph_output.ts b/tests/test-graph/src/features/test_kotlinc_graph_declines_a_launcher_without_graph_output.ts new file mode 100644 index 00000000..64b312bf --- /dev/null +++ b/tests/test-graph/src/features/test_kotlinc_graph_declines_a_launcher_without_graph_output.ts @@ -0,0 +1,287 @@ +import { TestValidator } from "@nestia/e2e"; +import { + buildGraphDump, + KOTLIN_GRAPH_PROVIDER, + kotlinGraphProvider, + selectGraphProviders, +} from "@samchon/graph"; +import fs from "node:fs"; +import path from "node:path"; + +import { GraphPaths } from "../internal/GraphPaths"; + +/** + * A launcher's name is not its capability. + * + * Every released `scip-java` answers `index`, and none of them writes this graph: + * `--kotlin-graph-output` arrived with the kotlinc plugin. A route that resolved on the + * command name alone would run a whole Gradle build and only then + * find nothing to read—and, worse, a reader would have watched a + * compiler-authority provider get selected. So the capability is asked for + * before anything is compiled, and a launcher that lacks it declines the way + * an uninstalled one does. + * + * 1. A launcher whose `index --help` omits the option is not selected. + * 2. The decline names the provider and the authority the build gave up. + * 3. The same launcher with the option is selected, and a bounded request + * still refuses it with a sentence rather than silently weakening it. + */ +export const test_kotlinc_graph_declines_a_launcher_without_graph_output = + async (): Promise => { + const root = GraphPaths.createTempDirectory("samchon-graph-kotlinc-select-"); + fs.writeFileSync( + path.join(root, "settings.gradle.kts"), + 'rootProject.name = "fixture"\n', + ); + // The two compilation units the fake producer reports having compiled. + // The coordinator hashes them itself before it will publish anything, so + // a fixture that named sources it does not have would fail that fence for + // the fixture reason rather than the route one. + fs.mkdirSync(path.join(root, "src", "main", "kotlin", "com"), { + recursive: true, + }); + fs.writeFileSync( + path.join(root, "src", "main", "kotlin", "com", "Example.kt"), + "package com;\npublic class Example {}\n", + ); + fs.writeFileSync( + path.join(root, "src", "main", "kotlin", "com", "Caller.kt"), + "package com;\npublic class Caller {\n public static Example make() {\n return new Example();\n }\n}\n", + ); + + const windows = process.platform === "win32"; + const script = (name: string, body: string): string => { + const file = path.join(root, windows ? `${name}.cmd` : name); + fs.writeFileSync( + file, + windows ? `@echo off\r\n${body}\r\n` : `#!/bin/sh\n${body}\n`, + ); + if (!windows) fs.chmodSync(file, 0o755); + return file; + }; + const shim = (name: string, flags: readonly string[]): string => + script( + name, + `"${process.execPath}" "${GraphPaths.fakeKotlinGraph}" ${flags.join(" ")} ${windows ? "%*" : '"$@"'}`, + ); + // Every JDK since 9 answers `--version` on standard output, which is + // where the shared toolchain probe reads. + const jdk = script("java", "echo openjdk 21.0.12 2026-10-21"); + + const select = (launcher: string, options = {}) => + selectGraphProviders(root, ["kotlin"], options, { + ...process.env, + SAMCHON_GRAPH_KOTLINC_GRAPH: launcher, + SAMCHON_GRAPH_JAVA_TOOLCHAIN: jdk, + }); + + const legacy = select(shim("legacy", ["--fake-legacy-launcher"])); + TestValidator.predicate( + "a launcher without --kotlin-graph-output is not selected", + legacy.candidates.every( + (candidate) => candidate.provider.name !== KOTLIN_GRAPH_PROVIDER, + ), + ); + TestValidator.predicate( + "the decline names the provider and the authority given up", + legacy.warnings.some( + (warning) => + warning.includes(KOTLIN_GRAPH_PROVIDER) && + warning.includes("compiler provider") && + warning.includes("was not found"), + ), + ); + + const legacyServer = select( + shim("legacy-server", ["--fake-legacy-server"]), + ); + TestValidator.predicate( + "a launcher without the resident protocol is not selected", + legacyServer.candidates.every( + (candidate) => candidate.provider.name !== KOTLIN_GRAPH_PROVIDER, + ), + ); + + const current = shim("current", []); + const selected = select(current); + TestValidator.predicate( + "a launcher that publishes the option owns the language", + selected.candidates.some( + (candidate) => + candidate.provider.name === KOTLIN_GRAPH_PROVIDER && + candidate.provider.authority === "compiler" && + candidate.languages.join() === "kotlin", + ), + ); + + // A whole-target producer has no bounded mode. Refusing is a sentence, not + // a silence: a capped request that quietly fell through to the generic lane + // would read exactly like the compiler-owned result it replaced. + const bounded = select(current, { + server: "jdtls", + maxFiles: 10, + lspReferenceLimit: 250, + }); + TestValidator.predicate( + "a bounded request refuses the route instead of weakening it", + bounded.candidates.every( + (candidate) => candidate.provider.name !== KOTLIN_GRAPH_PROVIDER, + ) && + bounded.warnings.some( + (warning) => + warning.includes(KOTLIN_GRAPH_PROVIDER) && + warning.includes("server, maxFiles, lspReferenceLimit"), + ), + ); + + // With no launcher anywhere—no override, and a PATH with nothing on it— + // the route is simply absent. A machine that happens to have `scip-java` + // installed must not turn this case into a different one. + const empty = path.join(root, "empty-path"); + fs.mkdirSync(empty, { recursive: true }); + const absent = selectGraphProviders( + root, + ["kotlin"], + {}, + { PATH: empty, Path: empty }, + ); + TestValidator.predicate( + "no launcher at all declines the same way", + absent.candidates.every( + (candidate) => candidate.provider.name !== KOTLIN_GRAPH_PROVIDER, + ), + ); + + // The build universe this route reuses facts against. A JDK swap + // recompiles against a different JDK and a launcher upgrade can + // move the shard schema, so both are identity rather than decoration. + const environment = { + ...process.env, + SAMCHON_GRAPH_KOTLINC_GRAPH: current, + SAMCHON_GRAPH_JAVA_TOOLCHAIN: jdk, + }; + const configuration = kotlinGraphProvider.configuration?.(root, environment); + TestValidator.predicate( + "the build universe names the JDK and the launcher that will run", + configuration?.length === 2 && + configuration[0]!.startsWith("java=") && + configuration[0]!.includes("21.0.12") && + configuration[1]!.startsWith("scip-java=") && + kotlinGraphProvider + .configurationDerivation?.(root, environment) + .inconclusive.length === 0, + ); + + const command = kotlinGraphProvider.resolve(root, environment); + TestValidator.predicate( + "the registered route resolves its launcher", + command !== undefined, + ); + if (command === undefined) { + throw new Error("kotlinc-graph: the fixture launcher did not resolve"); + } + + const mavenRoot = GraphPaths.createTempDirectory( + "samchon-graph-kotlinc-maven-", + ); + fs.writeFileSync( + path.join(mavenRoot, "pom.xml"), + "4.0.0\n", + ); + const unsupportedBuild = selectGraphProviders( + mavenRoot, + ["kotlin"], + {}, + environment, + ); + TestValidator.predicate( + "the Gradle-only compiler route declines a Maven project", + unsupportedBuild.candidates.every( + (candidate) => candidate.provider.name !== KOTLIN_GRAPH_PROVIDER, + ), + ); + // A session reads the environment from the process, not from the object + // its provider was resolved with—every registry entry does, because a + // session outlives the selection that opened it. The fixture therefore has + // to put its toolchain where the session will look, or the row it derives + // depends on whatever JDK the host happens to have. + const previous = new Map(); + for (const key of [ + "SAMCHON_GRAPH_KOTLINC_GRAPH", + "SAMCHON_GRAPH_JAVA_TOOLCHAIN", + ]) { + previous.set(key, process.env[key]); + process.env[key] = environment[key]; + } + try { + const session = kotlinGraphProvider.open({ + root, + command, + languages: ["kotlin"], + options: { cwd: root }, + }); + try { + const generation = await session.refresh(); + TestValidator.predicate( + "the registered route publishes its own compiler-owned generation", + generation.mode === "initial" && + generation.snapshot.provenance.provider === KOTLIN_GRAPH_PROVIDER && + generation.snapshot.provenance.authority === "compiler" && + generation.snapshot.provenance.compilerVersion === "2.3.20" && + generation.snapshot.protocol !== undefined, + ); + } finally { + await session.close(); + } + + // The whole route, through the coordinator that owns the project input + // generation. Opening the session directly proves the producer contract + // and nothing about the fence around it: a snapshot only publishes if + // every source it names binds to bytes the coordinator hashed itself, + // and a provider that omitted a disk digest or named a file in a form + // the coordinator cannot compare fails there rather than here. + const dump = await buildGraphDump({ + cwd: root, + mode: "lsp", + languages: ["kotlin"], + }); + TestValidator.predicate( + "the coordinator publishes the route's generation, fence and all", + (dump.provenance ?? []).some( + (row) => + row.provider === KOTLIN_GRAPH_PROVIDER && + row.authority === "compiler", + ) && + dump.nodes.some( + (node) => node.file === "src/main/kotlin/com/Example.kt", + ), + ); + + // An ordinary source edit, through the provider's own inputs rather than + // a fixture's list of them. This is the case a session that fingerprints + // only build files cannot pass: it would reuse the snapshot taken before + // the edit, and the coordinator would refuse it for describing bytes the + // file no longer has. A route that will not notice a source edit is not + // an incremental route, it is a stale one. + fs.writeFileSync( + path.join(root, "src", "main", "kotlin", "com", "Example.kt"), + "package com;\npublic class Example {}\n// edited\n", + ); + const edited = await buildGraphDump({ + cwd: root, + mode: "lsp", + languages: ["kotlin"], + }); + TestValidator.predicate( + "a source edit republishes rather than reusing a stale generation", + (edited.provenance ?? []).some( + (row) => row.provider === KOTLIN_GRAPH_PROVIDER, + ), + ); + } finally { + for (const [key, value] of previous) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + }; diff --git a/tests/test-graph/src/features/test_kotlinc_graph_publishes_atomic_target_generations.ts b/tests/test-graph/src/features/test_kotlinc_graph_publishes_atomic_target_generations.ts new file mode 100644 index 00000000..029c797e --- /dev/null +++ b/tests/test-graph/src/features/test_kotlinc_graph_publishes_atomic_target_generations.ts @@ -0,0 +1,557 @@ +import { TestValidator } from "@nestia/e2e"; +import { + assertGraphSnapshotContract, + KOTLIN_GRAPH_PROVIDER, + KotlinGraphSession, + kotlinGraphProvider, +} from "@samchon/graph"; +import fs from "node:fs"; +import path from "node:path"; + +import { GraphPaths } from "../internal/GraphPaths"; + +const SOURCES = { + "src/main/kotlin/com/Example.kt": "package com;\npublic class Example {}\n", + "src/main/kotlin/com/Caller.kt": + "package com;\npublic class Caller {\n public static Example make() {\n return new Example();\n }\n}\n", +}; + +/** + * The strict Kotlin route publishes what one build committed, and nothing else. + * + * The producer commits per target: each carries its own content-addressed + * generation and the universe it compiled against, and an incremental build + * rewrites only the sources kotlinc recompiled. So the consumer's whole job is + * to prove the transaction it was handed rather than to assemble one—and + * every case below is a way that proof can fail while the payload still parses. + * + * 1. A cold generation publishes compiler authority, target-scoped identities, + * a complete coverage matrix and one node per external endpoint. + * 2. An unchanged build reuses the exact snapshot; a recompiled source moves + * one shard and keeps the rest, which is what makes it incremental; a moved + * universe reloads instead; a dropped source leaves the generation. + * 3. A producer that cannot be trusted—wrong schema, wrong protocol, wrong + * project, an incomplete matrix, an edge with nothing on one end, one + * symbol declared or named twice—is refused with the prior generation + * intact. An endpoint the target merely does not declare is not one of + * those: that is an ordinary external node, and case 1 requires it. + */ +export const test_kotlinc_graph_publishes_atomic_target_generations = + async (): Promise => { + const root = GraphPaths.createTempDirectory("samchon-graph-kotlinc-"); + for (const [file, text] of Object.entries(SOURCES)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + } + + const open = ( + options: { + maxArtifactBytes?: number; + configuration?: () => readonly string[]; + } = {}, + ...flags: string[] + ): KotlinGraphSession => + new KotlinGraphSession({ + root, + languages: ["kotlin"], + provider: KOTLIN_GRAPH_PROVIDER, + command: { + command: process.execPath, + args: [GraphPaths.fakeKotlinGraph, ...flags], + }, + inputs: () => Object.keys(SOURCES), + configuration: () => ["kotlin=2.3.20"], + validate: (snapshot) => + assertGraphSnapshotContract( + snapshot, + kotlinGraphProvider, + ["kotlin"], + root, + ), + ...options, + }); + + // A byte ceiling that cannot bound anything is refused where it is stated, + // not where a build would later exceed it. + TestValidator.error("a ceiling that admits no artifact", () => { + open({ maxArtifactBytes: 0 }); + }); + + let configurationCalls = 0; + const session = open({ + configuration: () => { + configurationCalls += 1; + return ["kotlin=2.3.20"]; + }, + }); + try { + TestValidator.equals( + "the resident producer owns its selected command until close", + session.ownsProviderTopology, + true, + ); + const cold = await session.refresh(); + TestValidator.equals( + "the first published generation is generation one", + session.generation, + 1, + ); + TestValidator.equals( + "a cold build is the first generation", + cold.mode, + "initial", + ); + const snapshot = cold.snapshot; + TestValidator.equals( + "the route publishes compiler authority", + snapshot.provenance.authority, + "compiler", + ); + TestValidator.equals( + "the route names itself, not its launcher", + snapshot.provenance.provider, + KOTLIN_GRAPH_PROVIDER, + ); + TestValidator.equals( + "the producer names the plugin that wrote it", + snapshot.provenance.tool, + "scip-kotlinc-k2-graph", + ); + TestValidator.equals( + "one Kotlin compiler compiled every shard", + snapshot.provenance.compilerVersion, + "2.3.20", + ); + TestValidator.predicate( + "FIR diagnostics are carried with their source shard", + snapshot.diagnostics.some( + (diagnostic) => + diagnostic.file === "src/main/kotlin/com/Caller.kt" && + diagnostic.code === "kotlinc" && + diagnostic.severity === "warning" && + diagnostic.message === "fixture warning", + ), + ); + + // Coverage is what makes an absent edge meaningful, so it has to be + // complete for the target and appear exactly once per family. + const coverage = snapshot.coverage ?? []; + TestValidator.equals( + "the target states every relationship family once", + coverage.length, + 15, + ); + TestValidator.predicate( + "renders is stated unsupported rather than left out", + coverage.some( + (row) => row.family === "renders" && row.state === "unsupported", + ), + ); + TestValidator.predicate( + "every coverage row belongs to this provider and target", + coverage.every( + (row) => + row.provider === KOTLIN_GRAPH_PROVIDER && + row.language === "kotlin" && + row.target === ":|jvm|main", + ), + ); + + // The producer wrote the same relationship at two call sites. The graph's + // triple is unique and keeps the first source-order evidence. + const instantiates = snapshot.edges.filter( + (edge) => edge.kind === "instantiates", + ); + TestValidator.equals( + "one relationship written twice becomes one edge", + instantiates.length, + 1, + ); + const nodes = new Map(snapshot.nodes.map((node) => [node.id, node])); + const crossFile = instantiates[0]!; + TestValidator.predicate( + "the instantiation crosses a compilation unit", + nodes.get(crossFile.from)?.file === "src/main/kotlin/com/Caller.kt" && + nodes.get(crossFile.to)?.file === "src/main/kotlin/com/Example.kt", + ); + + // An endpoint no shard declares is still an endpoint. It becomes one + // external node scoped to the target that referenced it, so the edge has + // somewhere to land without the graph inventing a declaration for it. + const external = snapshot.nodes.filter((node) => node.external); + TestValidator.equals( + "a symbol reached twice from outside is still one external node", + external.length, + 3, + ); + // An edge can also originate outside the compilation, and the producer + // names what an edge points at rather than where it came from. Such an + // endpoint has only its own symbol to be displayed by, which is a + // display and not a name the compiler gave it. + TestValidator.predicate( + "an endpoint the producer never named displays its own symbol", + external.some( + (node) => + node.qualifiedName === undefined && + node.name.endsWith("$anon1#run()."), + ), + ); + // The producer named this one at a `type_ref` site and named nothing at + // the `decorates` site that reached it first. The description that says + // something wins, and it wins regardless of which shard was adapted + // first, because the endpoint set is settled before any shard is built. + TestValidator.predicate( + "a site that names an endpoint outranks one that cannot", + external.some( + (node) => node.qualifiedName === "kotlin.lang.Deprecated", + ), + ); + // The producer displays an executable with its parameter list, because + // that is what tells two overloads apart on sight. The graph's name is + // the simple declared name a reader types and a lookup matches, and a + // route that published the display as the name would leave every Java + // method unfindable by the name it is written with. + TestValidator.predicate( + "a method is named the way it is declared, not the way it displays", + snapshot.nodes.some( + (node) => + node.kind === "method" && + node.name === "make" && + node.qualifiedName === "com.Caller.make", + ), + ); + // Where the producer formats no signature, the display it came from is + // the only statement of the declaration's shape there is, so cutting the + // list out of the name has to put it somewhere rather than lose it. + TestValidator.predicate( + "a display with no signature beside it becomes the signature", + snapshot.nodes.some( + (node) => + node.kind === "constructor" && + node.name === "" && + node.signature === "constructor()", + ), + ); + TestValidator.predicate( + "the external node keeps the producer's naming, list cut away", + external.some( + (node) => node.qualifiedName === "kotlin.lang.Object.toString", + ), + ); + // A producer that could not name the endpoint leaves the symbol as the + // only thing there is to display it by, rather than inventing one. + TestValidator.predicate( + "an external symbol carries no file it does not have", + external.every( + (node) => node.file === "" && node.kind === "external_symbol", + ), + ); + + // Identity is target-scoped, which is what lets one source compiled into + // two targets be two facts rather than a collision. + TestValidator.predicate( + "declarations carry a target-scoped semantic identity", + snapshot.nodes.every((node) => node.id.startsWith("@v2/kotlin/")), + ); + + const sites = snapshot.unresolved ?? []; + // A candidate the target declares becomes that declaration's identity; a + // candidate it does not stays the producer's own symbol, because minting + // a declaration for an unresolved possibility is what the site exists to + // avoid. + TestValidator.predicate( + "unresolved sites are published with their proven candidates", + sites.some( + (site) => + site.family === "dispatches" && + site.reason === "dynamic" && + site.provider === KOTLIN_GRAPH_PROVIDER && + (site.candidates ?? []).length === 2 && + site.candidates!.some((candidate) => + candidate.startsWith("@v2/kotlin/"), + ) && + site.candidates!.some((candidate) => + candidate.startsWith("semanticdb maven"), + ), + ), + ); + TestValidator.predicate( + "a site with nothing to name carries no candidate list", + sites.some( + (site) => + site.reason === "analysis-error" && site.candidates === undefined, + ), + ); + // A family the producer calls partial with no site of its own still has + // to say so somewhere, or "some sites are unproven" reads exactly like + // "every site is proven". + TestValidator.predicate( + "a partial family with no located site publishes its gap", + sites.some( + (site) => + site.reason === "provider-gap" && + site.evidence.file.startsWith("bundled:///kotlin/target/"), + ), + ); + + TestValidator.predicate( + "the generation is content addressed and immutable", + typeof snapshot.protocol?.generation === "string" && + /^[0-9a-f]{64}$/u.test(snapshot.protocol.generation) && + snapshot.protocol.targets.join() === ":|jvm|main", + ); + // Two compilation units and the target's own coordinate. The first two + // are absolute host paths a reader can hash for itself; the third is the + // bundled identity the target-level facts hang off, which has no file. + const manifest = [...snapshot.sources.keys()]; + TestValidator.equals( + "a source manifest binds every fact to the bytes kotlinc read", + manifest.filter((file) => path.isAbsolute(file)).length, + 2, + ); + TestValidator.predicate( + "the target's own facts carry a bundled coordinate", + manifest.length === 3 && + manifest.some((file) => file.startsWith("bundled:///kotlin/target/")), + ); + + // Nothing moved, so nothing is rebuilt: the input fingerprint answers + // before the build tool is asked. + const unchanged = await session.refresh(); + TestValidator.equals( + "an unchanged project reuses its generation", + unchanged.mode, + "unchanged", + ); + TestValidator.predicate( + "the reused snapshot is the same object, not an equal one", + unchanged.snapshot === snapshot, + ); + TestValidator.equals( + "a resident producer establishes its fixed toolchain once", + configurationCalls, + 1, + ); + } finally { + await session.close(); + } + + // One source recompiled. The producer rewrites that shard and leaves the + // other byte-identical, so the consumer carries the unchanged one forward + // and the generation says incremental rather than rebuild. + const marker = path.join(root, "build", "invocations"); + const incremental = open( + {}, + "--fake-incremental", + `--fake-marker=${marker}`, + ); + try { + await incremental.refresh(); + fs.writeFileSync( + path.join(root, "src/main/kotlin/com/Caller.kt"), + `${SOURCES["src/main/kotlin/com/Caller.kt"]}// edit\n`, + ); + const second = await incremental.refresh(); + TestValidator.equals( + "a recompiled source is an incremental generation", + second.mode, + "incremental", + ); + TestValidator.predicate( + "the new generation names the one before it as its base", + second.snapshot.protocol?.baseGeneration !== undefined && + second.snapshot.protocol.sequence === 2, + ); + } finally { + await incremental.close(); + } + + const refuses = async ( + label: string, + flags: readonly string[], + ): Promise => { + const rejecting = open({}, ...flags); + try { + let failed = false; + try { + await rejecting.refresh(); + } catch { + failed = true; + } + TestValidator.predicate(`${label} is refused`, failed); + TestValidator.predicate( + `${label} publishes no partial generation`, + rejecting.current === undefined, + ); + } finally { + await rejecting.close(); + } + }; + + await refuses("a future artifact schema", ["--fake-future-schema"]); + await refuses("a future producer protocol", ["--fake-future-protocol"]); + await refuses("a foreign producer", ["--fake-foreign-producer"]); + await refuses("a producer without atomic generations", [ + "--fake-no-atomic-generations", + ]); + await refuses("an artifact produced for another project", [ + "--fake-foreign-root", + ]); + await refuses("a generation that committed no target", ["--fake-no-target"]); + await refuses("a coverage matrix with a family missing", [ + "--fake-hole-in-coverage", + ]); + await refuses("a target claiming a family this route cannot prove", [ + "--fake-claims-unsupported", + ]); + await refuses("an edge with nothing on one end", [ + "--fake-empty-endpoint", + ]); + await refuses("one external symbol named two ways", [ + "--fake-two-named-externals", + ]); + await refuses("an edge of an unregistered family", [ + "--fake-unclaimed-family", + ]); + await refuses("one symbol declared by two compilation units", [ + "--fake-duplicate-symbol", + ]); + await refuses("a shard committed under another target", [ + "--fake-foreign-shard-target", + ]); + await refuses("evidence with no source position", ["--fake-bad-evidence"]); + await refuses("a build that printed where its graph belongs", [ + "--fake-not-json", + ]); + await refuses("a build that failed", ["--fake-build-failure"]); + + // A ceiling a real generation exceeds is refused before the artifact is + // parsed, because the parse is the cost the ceiling exists to bound. + const bounded = open({ maxArtifactBytes: 1 }); + try { + let message = ""; + try { + await bounded.refresh(); + } catch (error) { + message = (error as Error).message; + } + TestValidator.predicate( + "an artifact past the ceiling is refused by size, not by parsing", + message.includes("exceeded the 1 byte limit"), + ); + } finally { + await bounded.close(); + } + + // A build that wrote nothing where its graph belongs. The sentence has to + // say the file was empty rather than quote four hundred characters of it. + const empty = open({}, "--fake-empty-artifact"); + try { + let message = ""; + try { + await empty.refresh(); + } catch (error) { + message = (error as Error).message; + } + TestValidator.predicate( + "an empty artifact is reported as empty", + message.includes("(the file is empty)"), + ); + } finally { + await empty.close(); + } + + // Two JDKs in one build is a thing a Gradle toolchain per source set does, + // and no single version is then the build's. The field says so by being + // empty rather than by reporting the first shard's reading. + const mixed = open({}, "--fake-two-compilers"); + try { + const generation = await mixed.refresh(); + TestValidator.equals( + "a build with two JDKs names both rather than picking one", + generation.snapshot.provenance.compilerVersion, + "2.3.10; 2.3.20", + ); + } finally { + await mixed.close(); + } + + // A universe that moved invalidates every shard the last generation held, + // whether or not its bytes are identical, so the route reloads instead of + // sending a delta that would invalidate everything anyway. + const moved = path.join(root, "build", "moving"); + const reloading = open( + {}, + "--fake-moving-universe", + `--fake-marker=${moved}`, + ); + try { + await reloading.refresh(); + fs.writeFileSync( + path.join(root, "src/main/kotlin/com/Example.kt"), + `${SOURCES["src/main/kotlin/com/Example.kt"]}// classpath moved +`, + ); + const second = await reloading.refresh(); + TestValidator.equals( + "a moved build universe reloads rather than deltas", + second.mode, + "reload", + ); + } finally { + await reloading.close(); + } + + // A source the build no longer compiles is deleted from the generation + // rather than left behind as a fact nothing refreshes. + const dropped = path.join(root, "build", "dropping"); + const deleting = open( + {}, + "--fake-deleted-source", + `--fake-marker=${dropped}`, + ); + try { + const before = await deleting.refresh(); + fs.writeFileSync( + path.join(root, "src/main/kotlin/com/Caller.kt"), + `${SOURCES["src/main/kotlin/com/Caller.kt"]}// dropped +`, + ); + const after = await deleting.refresh(); + TestValidator.predicate( + "a source the build dropped leaves the generation", + before.snapshot.protocol!.shards.length - + after.snapshot.protocol!.shards.length === + 1, + ); + } finally { + await deleting.close(); + } + + // Two targets compiling the same source is the case target-scoped identity + // exists for: one file, two universes, and no node that belongs to both. + const multi = open({}, "--fake-two-targets"); + try { + const generation = await multi.refresh(); + const targets = generation.snapshot.protocol?.targets ?? []; + TestValidator.equals( + "each committed target is published", + targets.join(), + ":module-a|jvm|main,:module-b|jvm|main", + ); + const declarations = generation.snapshot.nodes.filter( + (node) => !node.external, + ); + TestValidator.equals( + "one source in two targets is two declarations", + declarations.length, + 2, + ); + TestValidator.predicate( + "the two declarations do not share an identity", + declarations[0]!.id !== declarations[1]!.id, + ); + } finally { + await multi.close(); + } + }; diff --git a/tests/test-graph/src/features/test_kotlinc_graph_refuses_an_artifact_it_cannot_prove.ts b/tests/test-graph/src/features/test_kotlinc_graph_refuses_an_artifact_it_cannot_prove.ts new file mode 100644 index 00000000..1b7aec3e --- /dev/null +++ b/tests/test-graph/src/features/test_kotlinc_graph_refuses_an_artifact_it_cannot_prove.ts @@ -0,0 +1,364 @@ +import { TestValidator } from "@nestia/e2e"; +import { IKotlinGraphSnapshot, KotlinGraphSnapshotAdapter } from "@samchon/graph"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { GraphPaths } from "../internal/GraphPaths"; + +/** + * Every field the adapter reads is a field a producer can get wrong. + * + * The lifecycle case drives the whole session and proves the route works. This + * one holds the artifact still and moves one thing at a time, because that is + * the only way to tell a validator that checks a field from one that mentions + * it: a malformed generation has to be refused for the reason it is malformed, + * and the prior generation has to survive every one of them. + * + * The baseline is written by the fake producer rather than typed here, so the + * shape these cases mutate is the shape the route actually receives. + */ +export const test_kotlinc_graph_refuses_an_artifact_it_cannot_prove = + (): void => { + const root = GraphPaths.createTempDirectory("samchon-graph-kotlinc-wire-"); + fs.mkdirSync(path.join(root, "src", "main", "kotlin", "com"), { + recursive: true, + }); + fs.writeFileSync( + path.join(root, "src", "main", "kotlin", "com", "Example.kt"), + "package com;\npublic class Example {}\n", + ); + fs.writeFileSync( + path.join(root, "src", "main", "kotlin", "com", "Caller.kt"), + "package com;\npublic class Caller {}\n", + ); + const artifact = path.join(root, "graph.json"); + const produced = spawnSync( + process.execPath, + [ + GraphPaths.fakeKotlinGraph, + "index", + "--output", + path.join(root, "index.scip"), + "--kotlin-graph-output", + artifact, + ], + { cwd: root, encoding: "utf8" }, + ); + TestValidator.equals( + "the fake producer writes a baseline artifact", + produced.status, + 0, + ); + const valid = JSON.parse( + fs.readFileSync(artifact, "utf8"), + ) as IKotlinGraphSnapshot; + + // The baseline itself must publish, or every refusal below would pass for + // the wrong reason. + const accepting = new KotlinGraphSnapshotAdapter(root); + TestValidator.predicate( + "the unmutated artifact publishes a generation", + accepting.current === undefined && + accepting.apply(structuredClone(valid)).protocol !== undefined && + accepting.current !== undefined, + ); + const published = accepting.current; + + const rejects = ( + label: string, + mutate: (value: IKotlinGraphSnapshot) => void, + ): void => { + const candidate = structuredClone(valid); + mutate(candidate); + TestValidator.error(label, () => { + accepting.apply(candidate); + }); + TestValidator.predicate( + `${label} leaves the prior generation standing`, + accepting.current === published, + ); + }; + + // Not a mutation of the baseline: a build that printed a line where its + // graph belongs parses to a string, and the first thing the adapter has to + // establish is that it was handed an object at all. + TestValidator.error("an artifact that is not an object", () => { + accepting.apply("BUILD SUCCESSFUL"); + }); + + rejects("an artifact from a future schema", (value) => { + value.schemaVersion = 2; + }); + rejects("an artifact from a foreign producer", (value) => { + value.producer.name = "some-other-graph"; + }); + rejects("a producer speaking a future protocol", (value) => { + value.producer.protocolVersion = 2; + }); + rejects("a producer that cannot commit atomic generations", (value) => { + value.producer.capabilities.atomicGenerations = false; + }); + rejects("a producer that cannot preserve incremental generations", (value) => { + value.producer.capabilities.incremental = false; + }); + rejects("a producer that cannot publish diagnostics", (value) => { + value.producer.capabilities.diagnostics = false; + }); + rejects("an artifact produced for another project", (value) => { + value.projectRoot = path.join(value.projectRoot, "elsewhere"); + }); + rejects("an artifact that committed no target", (value) => { + value.targets = []; + }); + + rejects("an artifact with no producer block", (value) => { + (value as { producer?: unknown }).producer = "not-a-producer-block"; + }); + rejects("a producer that states no version", (value) => { + value.producer.version = ""; + }); + rejects("a producer with no capability block", (value) => { + (value.producer as { capabilities?: unknown }).capabilities = true; + }); + rejects("a capability block missing incremental", (value) => { + (value.producer.capabilities as { incremental?: unknown }).incremental = + "yes"; + }); + rejects("a capability block missing diagnostics", (value) => { + (value.producer.capabilities as { diagnostics?: unknown }).diagnostics = + 1; + }); + rejects("an artifact that names no project root", (value) => { + value.projectRoot = ""; + }); + rejects("an artifact whose targets are not a list", (value) => { + (value as { targets?: unknown }).targets = {}; + }); + + rejects("a target with no name", (value) => { + value.targets[0]!.name = ""; + }); + rejects("a target generation that is not a digest", (value) => { + value.targets[0]!.generation = "0"; + }); + rejects("a target universe that is not a digest", (value) => { + value.targets[0]!.universe = "not-a-digest"; + }); + rejects("a target whose coverage is not a matrix", (value) => { + (value.targets[0] as { coverage?: unknown }).coverage = []; + }); + rejects("a target that committed no shard", (value) => { + value.targets[0]!.shards = []; + }); + rejects("a coverage state the protocol does not define", (value) => { + value.targets[0]!.coverage.calls = "probably"; + }); + rejects("two targets committed under one name", (value) => { + value.targets.push(structuredClone(value.targets[0]!)); + }); + + rejects("a shard from a future schema", (value) => { + value.targets[0]!.shards[0]!.schemaVersion = 2; + }); + rejects("a shard of another language", (value) => { + value.targets[0]!.shards[0]!.language = "java"; + }); + rejects("a shard with no source", (value) => { + value.targets[0]!.shards[0]!.source = ""; + }); + // A source identity is compared twice against the same edge endpoints: + // once as the producer spelled it, once as the normalized path a node + // carries. `./a/B.kt` and `a/B.kt` are one file and two strings, so a + // producer that spells it either second way makes those two walks disagree + // about what an endpoint is—and the route has to say that, rather than + // dereference the lookup that came back empty. One case per spelling, + // because each is a different way of being non-canonical and a check that + // caught only the first would let the rest through. + for (const [label, source] of [ + ["a source spelled with a leading dot segment", "./src/main/kotlin/com/Example.kt"], + ["a source spelled with an interior dot segment", "src/main/./kotlin/com/Example.kt"], + ["a source that climbs out of the project", "../Example.kt"], + ["a source spelled with Windows separators", "src\\main\\kotlin\\com\\Example.kt"], + ["a source given as an absolute POSIX path", "/src/main/kotlin/com/Example.kt"], + ["a source given as an absolute Windows path", "C:/src/Example.kt"], + ["a source carrying a NUL", "src/main/kotlin/com/Exa\0mple.kt"], + ["a source with an empty segment", "src//Example.kt"], + ["a source that is only a dot segment", "."], + ["a source spelled as a directory", "src/main/kotlin/com/"], + ] as const) { + rejects(label, (value) => { + value.targets[0]!.shards[0]!.source = source; + }); + } + rejects("a checker digest that is not a digest", (value) => { + value.targets[0]!.shards[0]!.checkerDigest = "nope"; + }); + rejects("a disk digest that is not a digest", (value) => { + value.targets[0]!.shards[0]!.diskDigest = "nope"; + }); + // The coordinator will not publish a generation whose sources it cannot + // hash for itself, so an absent disk digest is refused here rather than + // three refreshes later by a fence that cannot name the producer. + rejects("a source the producer could not read from disk", (value) => { + value.targets[0]!.shards[0]!.diskDigest = ""; + }); + rejects("a shard that states no compiler", (value) => { + (value.targets[0]!.shards[0] as { compilerVersion?: unknown }) + .compilerVersion = 21; + }); + // A route that publishes compiler authority has to name the compiler. An + // empty reading is the producer failing to read `kotlin.version`, not a + // build without one. + rejects("a shard whose compiler reading is empty", (value) => { + value.targets[0]!.shards[0]!.compilerVersion = ""; + }); + rejects("a shard whose nodes are not a list", (value) => { + (value.targets[0]!.shards[0] as { nodes?: unknown }).nodes = {}; + }); + rejects("one source committed twice in one target", (value) => { + value.targets[0]!.shards.push( + structuredClone(value.targets[0]!.shards[0]!), + ); + }); + + rejects("a declaration with no symbol", (value) => { + value.targets[0]!.shards[0]!.nodes[0]!.symbol = ""; + }); + rejects("a declaration of a kind the graph has no node for", (value) => { + value.targets[0]!.shards[0]!.nodes[0]!.kind = "annotation"; + }); + rejects("a declaration with no name", (value) => { + value.targets[0]!.shards[0]!.nodes[0]!.name = ""; + }); + rejects("a declaration whose qualified name is absent", (value) => { + (value.targets[0]!.shards[0]!.nodes[0] as { qualifiedName?: unknown }) + .qualifiedName = null; + }); + rejects("a declaration with no file", (value) => { + value.targets[0]!.shards[0]!.nodes[0]!.file = ""; + }); + rejects("a declaration attributed to another source", (value) => { + value.targets[0]!.shards[0]!.nodes[0]!.file = "../Outside.kt"; + }); + rejects("a declaration that does not say whether it is exported", (value) => { + (value.targets[0]!.shards[0]!.nodes[0] as { exported?: unknown }) + .exported = "yes"; + }); + rejects("a modifier outside the shared vocabulary", (value) => { + value.targets[0]!.shards[0]!.nodes[0]!.modifiers = ["sealed"]; + }); + rejects("a modifier repeated on one declaration", (value) => { + value.targets[0]!.shards[0]!.nodes[0]!.modifiers = ["public", "public"]; + }); + rejects("a declaration with no signature field", (value) => { + (value.targets[0]!.shards[0]!.nodes[0] as { signature?: unknown }) + .signature = null; + }); + rejects("a declaration with no compiler origin", (value) => { + value.targets[0]!.shards[0]!.nodes[0]!.origin = ""; + }); + rejects("one symbol declared twice in one compilation unit", (value) => { + value.targets[0]!.shards[0]!.nodes.push( + structuredClone(value.targets[0]!.shards[0]!.nodes[0]!), + ); + }); + + rejects("an edge with no source endpoint", (value) => { + value.targets[0]!.shards[0]!.edges[0]!.from = ""; + }); + rejects("an edge of a family the graph has no name for", (value) => { + value.targets[0]!.shards[0]!.edges[0]!.kind = "inherits"; + }); + rejects("an access mode that is neither a string nor absent", (value) => { + (value.targets[0]!.shards[0]!.edges[0] as { access?: unknown }).access = + 7; + }); + rejects("a provenance that is neither a string nor absent", (value) => { + (value.targets[0]!.shards[0]!.edges[0] as { provenance?: unknown }) + .provenance = 7; + }); + rejects("a target name that is neither a string nor absent", (value) => { + (value.targets[0]!.shards[0]!.edges[0] as { targetName?: unknown }) + .targetName = 7; + }); + rejects( + "a target qualified name that is neither a string nor absent", + (value) => { + ( + value.targets[0]!.shards[0]!.edges[0] as { + targetQualifiedName?: unknown; + } + ).targetQualifiedName = 7; + }, + ); + rejects("an endpoint kind the graph has no node for", (value) => { + value.targets[0]!.shards[0]!.edges[0]!.targetKind = "annotation"; + }); + + rejects("an unresolved family the graph has no name for", (value) => { + unresolvedShard(value).unresolved[0]!.family = "inherits"; + }); + rejects("an unresolved reason outside the closed set", (value) => { + unresolvedShard(value).unresolved[0]!.reason = "unlucky"; + }); + rejects("unresolved candidates that are not a list", (value) => { + (unresolvedShard(value).unresolved[0] as { candidates?: unknown }) + .candidates = "one"; + }); + rejects("one candidate named twice", (value) => { + const site = unresolvedShard(value).unresolved[0]!; + site.candidates = [...site.candidates, ...site.candidates]; + }); + rejects("diagnostics that are not a list", (value) => { + (value.targets[0]!.shards[0] as { diagnostics?: unknown }).diagnostics = {}; + }); + rejects("a diagnostic severity outside the shared vocabulary", (value) => { + const shard = value.targets[0]!.shards.find( + (entry) => entry.diagnostics.length > 0, + )!; + shard.diagnostics[0]!.severity = "fatal"; + }); + rejects("a diagnostic with no message", (value) => { + const shard = value.targets[0]!.shards.find( + (entry) => entry.diagnostics.length > 0, + )!; + shard.diagnostics[0]!.message = ""; + }); + rejects("evidence with no file", (value) => { + unresolvedShard(value).unresolved[0]!.evidence.file = ""; + }); + rejects("evidence attributed to another source", (value) => { + unresolvedShard(value).unresolved[0]!.evidence.file = "../Outside.kt"; + }); + rejects("evidence with a fractional position", (value) => { + value.targets[0]!.shards[0]!.nodes[0]!.evidence.endColumn = 1.5; + }); + + // An endpoint outside the compilation is one node however many sources + // name it, so two references that describe it differently are a producer + // contradiction rather than two nodes. + rejects("one external symbol described two ways", (value) => { + const shard = value.targets[0]!.shards.find( + (entry) => entry.edges.length > 3, + )!; + const external = shard.edges.find( + (edge) => edge.targetQualifiedName === "kotlin.lang.Object.toString", + )!; + shard.edges.push({ + ...structuredClone(external), + kind: "references", + targetName: "somethingElse", + targetQualifiedName: "kotlin.lang.Object.somethingElse", + }); + }); + }; + +/** The shard the fake producer publishes its unresolved site on. */ +function unresolvedShard( + value: IKotlinGraphSnapshot, +): IKotlinGraphSnapshot.IShard { + return value.targets[0]!.shards.find( + (shard) => shard.unresolved.length > 0, + )!; +} diff --git a/tests/test-graph/src/features/test_kotlinc_graph_resident_transport_recovers_and_is_bounded.ts b/tests/test-graph/src/features/test_kotlinc_graph_resident_transport_recovers_and_is_bounded.ts new file mode 100644 index 00000000..3e86860b --- /dev/null +++ b/tests/test-graph/src/features/test_kotlinc_graph_resident_transport_recovers_and_is_bounded.ts @@ -0,0 +1,333 @@ +import { TestValidator } from "@nestia/e2e"; +import { KotlinGraphProducerClient } from "@samchon/graph"; +import fs from "node:fs"; +import path from "node:path"; + +import { GraphPaths } from "../internal/GraphPaths"; + +/** One failed resident request retires only its owned producer generation. */ +export const test_kotlinc_graph_resident_transport_recovers_and_is_bounded = + async (): Promise => { + const root = GraphPaths.createTempDirectory( + "samchon-graph-kotlinc-resident-", + ); + + for (const value of [0, -1, 1.5, Number.NaN, 2_147_483_648]) { + TestValidator.error(`unsafe timeout ${String(value)} is refused`, () => + create(root, [], { requestTimeoutMs: value }), + ); + } + for (const value of [0, -1, 1.5, Number.NaN]) { + TestValidator.error( + `unsafe response bound ${String(value)} is refused`, + () => create(root, [], { maxResponseBytes: value }), + ); + } + + const reuseLog = path.join(root, "reuse.log"); + const reused = create(root, [ + `--fake-server-log=${reuseLog}`, + "--fake-server-blank-prefix", + "--fake-server-split-response", + ]); + try { + const liveController = new AbortController(); + await reused.produce( + path.join(root, "reuse-one.json"), + liveController.signal, + ); + await reused.produce(path.join(root, "reuse-two.json"), undefined); + const processIds = lines(reuseLog); + TestValidator.predicate( + "successful requests reuse one resident producer", + processIds.length === 2 && processIds[0] === processIds[1], + ); + } finally { + await reused.close(); + } + + const errorMarker = path.join(root, "error-once.marker"); + const errorLog = path.join(root, "error.log"); + const recoverable = create(root, [ + `--fake-server-error-once=${errorMarker}`, + `--fake-server-log=${errorLog}`, + "--fake-server-stderr", + ]); + try { + const failure = await rejectionOf( + recoverable.produce(path.join(root, "error.json"), undefined), + ); + TestValidator.predicate( + "a producer-declared failure reaches the caller", + failure.message.includes("deliberate resident failure") && + failure.message.includes("resident fixture stderr"), + ); + await recoverable.produce( + path.join(root, "error-recovered.json"), + undefined, + ); + const processIds = lines(errorLog); + TestValidator.predicate( + "a normal error does not discard a healthy resident producer", + processIds.length === 2 && processIds[0] === processIds[1], + ); + } finally { + await recoverable.close(); + } + + await assertRestart(root, "crash", "server-crash-once", "exited"); + await assertRestart( + root, + "malformed", + "server-malformed-once", + "invalid Kotlin graph server response", + ); + for (const [name, flag, message] of [ + ["string", "server-string-once", "must be an object"], + ["null", "server-null-once", "must be an object"], + ["array", "server-non-object-once", "must be an object"], + ["identity", "server-bad-identity-once", "invalid Kotlin graph response identity"], + ["protocol", "server-bad-protocol-once", "invalid Kotlin graph response identity"], + ["result", "server-bad-result-once", "invalid Kotlin graph response result"], + ["error-type", "server-non-string-error-once", "invalid Kotlin graph response result"], + ["error-empty", "server-empty-error-once", "invalid Kotlin graph response result"], + ["response-id", "server-unexpected-id-once", "unexpected Kotlin graph response id"], + ] as const) { + await assertRestart(root, name, flag, message); + } + await assertRestart( + root, + "oversized", + "server-oversized-once", + "byte limit", + { maxResponseBytes: 128 }, + ); + await assertRestart( + root, + "timeout", + "server-stall-once", + "timed out after 5000 ms", + { requestTimeoutMs: 5_000 }, + ); + + const alreadyAborted = new AbortController(); + alreadyAborted.abort(); + const preflightClient = create(root, []); + try { + const failure = await rejectionOf( + preflightClient.produce( + path.join(root, "preflight-aborted.json"), + alreadyAborted.signal, + ), + ); + TestValidator.predicate( + "an already-cancelled request starts no producer", + failure.name === "AbortError" && failure.message.includes("aborted"), + ); + } finally { + await preflightClient.close(); + } + + let abortReads = 0; + const racingSignal = { + get aborted() { + abortReads += 1; + return abortReads > 1; + }, + addEventListener() {}, + removeEventListener() {}, + } as unknown as AbortSignal; + const racingClient = create(root, []); + try { + const failure = await rejectionOf( + racingClient.produce( + path.join(root, "racing-abort.json"), + racingSignal, + ), + ); + TestValidator.predicate( + "cancellation between preflight and listener registration wins the race", + failure.name === "AbortError" && abortReads === 2, + ); + } finally { + await racingClient.close(); + } + + const missingClient = new KotlinGraphProducerClient({ + root, + provider: "kotlinc-graph", + command: { + command: path.join(root, "missing-kotlin-graph-server"), + args: [], + }, + requestTimeoutMs: 5_000, + }); + try { + const failure = await rejectionOf( + missingClient.produce(path.join(root, "missing.json"), undefined), + ); + TestValidator.predicate( + "a launcher that cannot spawn fails its owned request", + (failure.message.includes("Kotlin graph server failed") || + failure.message.includes("process launch failed")) && + failure.message.includes("missing-kotlin-graph-server"), + ); + } finally { + await missingClient.close(); + } + + const closedInputClient = create( + root, + ["--fake-server-close-stdin"], + { requestTimeoutMs: 5_000 }, + ); + try { + const failure = await rejectionOf( + closedInputClient.produce("x".repeat(8 * 1024 * 1024), undefined), + ); + TestValidator.predicate( + "a producer that closes its input pipe remains deadline-bounded", + failure.message.includes("stdin failed") || + failure.message.includes("timed out after 5000 ms"), + ); + } finally { + await closedInputClient.close(); + } + + const abortMarker = path.join(root, "abort.marker"); + const abortLog = path.join(root, "abort.log"); + const abortedClient = create(root, [ + `--fake-server-stall-once=${abortMarker}`, + `--fake-server-log=${abortLog}`, + ]); + try { + const controller = new AbortController(); + const aborted = abortedClient.produce( + path.join(root, "aborted.json"), + controller.signal, + ); + await waitForLines(abortLog, 1); + controller.abort(); + const failure = await rejectionOf(aborted); + TestValidator.predicate( + "cancellation rejects and retires its exact producer", + failure.name === "AbortError" && failure.message.includes("aborted"), + ); + await abortedClient.produce( + path.join(root, "abort-recovered.json"), + undefined, + ); + const processIds = lines(abortLog); + TestValidator.predicate( + "the request after cancellation starts a fresh producer", + processIds.length === 2 && processIds[0] !== processIds[1], + ); + } finally { + await abortedClient.close(); + } + + const closeLog = path.join(root, "close.log"); + const closingClient = create(root, [ + "--fake-server-stall", + `--fake-server-log=${closeLog}`, + ]); + const pending = closingClient.produce( + path.join(root, "closing.json"), + undefined, + ); + await waitForLines(closeLog, 1); + const firstClose = closingClient.close(); + const secondClose = closingClient.close(); + TestValidator.equals( + "close is idempotent while termination is in flight", + firstClose, + secondClose, + ); + const closeFailure = await rejectionOf(pending); + await firstClose; + TestValidator.predicate( + "close rejects an active request without waiting for its deadline", + closeFailure.message.includes("session is closed"), + ); + const afterClose = await rejectionOf( + closingClient.produce(path.join(root, "after-close.json"), undefined), + ); + TestValidator.predicate( + "a closed client cannot spawn another producer", + afterClose.message.includes("session is closed") && + lines(closeLog).length === 1, + ); + }; + +async function assertRestart( + root: string, + name: string, + flag: string, + message: string, + options: Partial = {}, +): Promise { + const marker = path.join(root, `${name}.marker`); + const log = path.join(root, `${name}.log`); + const client = create( + root, + [`--fake-${flag}=${marker}`, `--fake-server-log=${log}`], + options, + ); + try { + const failure = await rejectionOf( + client.produce(path.join(root, `${name}-failed.json`), undefined), + ); + TestValidator.predicate( + `${name} fails its owned request precisely`, + failure.message.includes(message), + ); + await client.produce(path.join(root, `${name}-recovered.json`), undefined); + const processIds = lines(log); + TestValidator.predicate( + `${name} recovers on a fresh resident producer`, + processIds.length === 2 && processIds[0] !== processIds[1], + ); + } finally { + await client.close(); + } +} + +function create( + root: string, + flags: readonly string[], + options: Partial = {}, +): KotlinGraphProducerClient { + return new KotlinGraphProducerClient({ + root, + provider: "kotlinc-graph", + command: { + command: process.execPath, + args: [GraphPaths.fakeKotlinGraph, ...flags], + }, + ...options, + }); +} + +async function rejectionOf(promise: Promise): Promise { + try { + await promise; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } + throw new Error("expected resident request to reject"); +} + +function lines(file: string): string[] { + return fs.existsSync(file) + ? fs.readFileSync(file, "utf8").trim().split(/\r?\n/u).filter(Boolean) + : []; +} + +async function waitForLines(file: string, count: number): Promise { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + if (lines(file).length >= count) return; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error(`resident fixture did not write ${file}`); +} diff --git a/tests/test-graph/src/features/test_lsp_client_closes_servers_that_break_the_shutdown_handshake.ts b/tests/test-graph/src/features/test_lsp_client_closes_servers_that_break_the_shutdown_handshake.ts index 65e2c2ea..1cd8175a 100644 --- a/tests/test-graph/src/features/test_lsp_client_closes_servers_that_break_the_shutdown_handshake.ts +++ b/tests/test-graph/src/features/test_lsp_client_closes_servers_that_break_the_shutdown_handshake.ts @@ -6,6 +6,7 @@ import { pathToFileURL } from "node:url"; import { Worker } from "node:worker_threads"; import { GraphPaths } from "../internal/GraphPaths"; +import { waitForProcessId } from "../internal/waitForProcessId"; interface ILspClient { request( @@ -798,8 +799,7 @@ const assertExitedLeaderDoesNotLeakItsProcessGroup = async ( let pid: number | undefined; try { await client.request("initialize", {}); - await waitForFile(pidFile); - pid = Number(fs.readFileSync(pidFile, "utf8")); + pid = await waitForProcessId(pidFile); await settleWithin(client.close(), 5_000, () => terminate(pid!)); TestValidator.equals( "close waits for a process group after its cooperative leader exits", @@ -904,8 +904,7 @@ const assertStubbornProcessTreeIsOwned = async ( let pid: number | undefined; try { await client.request("initialize", {}); - await waitForFile(pidFile); - pid = Number(fs.readFileSync(pidFile, "utf8")); + pid = await waitForProcessId(pidFile); await settleWithin(client.close(), 5_000, () => terminate(pid!)); TestValidator.equals( "close returns only after a signal-resistant LSP child exits", diff --git a/tests/test-graph/src/features/test_mcp_topology_fences_file_joins_by_code_generation.ts b/tests/test-graph/src/features/test_mcp_topology_fences_file_joins_by_code_generation.ts index 0aa7021c..bbd4ad6a 100644 --- a/tests/test-graph/src/features/test_mcp_topology_fences_file_joins_by_code_generation.ts +++ b/tests/test-graph/src/features/test_mcp_topology_fences_file_joins_by_code_generation.ts @@ -36,17 +36,19 @@ export const test_mcp_topology_fences_file_joins_by_code_generation = topologyDump(fixture.dump.project), ); const application = new SamchonGraphApplication(graph, () => topology); - const compatible = await application.inspect_code_graph({ - question: "show repository packages and their source files", - draft: { reason: "repository orientation", type: "topology" }, - review: "topology is the typed repository plane", - request: { - type: "topology", - query: "source", - relations: ["joins-file"], - limit: 10, - }, - }); + const compatible = await tracedTopology(() => + application.inspect_code_graph({ + question: "show repository packages and their source files", + draft: { reason: "repository orientation", type: "topology" }, + review: "topology is the typed repository plane", + request: { + type: "topology", + query: "source", + relations: ["joins-file"], + limit: 10, + }, + }), + ); TestValidator.equals( "a stable code generation admits only joins to indexed code files", [ @@ -312,6 +314,17 @@ export const test_mcp_topology_fences_file_joins_by_code_generation = } }; +async function tracedTopology(closure: () => Promise): Promise { + const previous = process.env.SAMCHON_GRAPH_TOPOLOGY_TRACE; + process.env.SAMCHON_GRAPH_TOPOLOGY_TRACE = "1"; + try { + return await closure(); + } finally { + if (previous === undefined) delete process.env.SAMCHON_GRAPH_TOPOLOGY_TRACE; + else process.env.SAMCHON_GRAPH_TOPOLOGY_TRACE = previous; + } +} + function topologyDump(project: string): ISamchonRepositoryContextDump { const workspace = repositoryContextId("fixture", "workspace", "."); const sourceHelper = repositoryContextId( diff --git a/tests/test-graph/src/features/test_process_id_fixture_waits_for_complete_publication.ts b/tests/test-graph/src/features/test_process_id_fixture_waits_for_complete_publication.ts new file mode 100644 index 00000000..61cf4e80 --- /dev/null +++ b/tests/test-graph/src/features/test_process_id_fixture_waits_for_complete_publication.ts @@ -0,0 +1,53 @@ +import { TestValidator } from "@nestia/e2e"; +import fs from "node:fs"; +import path from "node:path"; + +import { GraphPaths } from "../internal/GraphPaths"; +import { waitForProcessId } from "../internal/waitForProcessId"; + +/** + * PID fixtures expose a truncation window before their complete write lands. + * Cleanup must accept only a newline-terminated positive integer and must stop + * waiting at its deadline when no publication boundary ever arrives. + */ +export const test_process_id_fixture_waits_for_complete_publication = + async () => { + const root = GraphPaths.createTempDirectory("samchon-graph-pid-publication-"); + const file = path.join(root, "child.pid"); + fs.writeFileSync(file, ""); + + const waiting = waitForProcessId(file); + await delay(20); + fs.writeFileSync(file, "0"); + await delay(20); + fs.writeFileSync(file, "123x"); + await delay(20); + fs.writeFileSync(file, "4"); + await delay(30); + fs.writeFileSync(file, "424242"); + await delay(20); + fs.writeFileSync(file, "424242\n"); + + TestValidator.equals( + "an incomplete or unsafe pid is never published to cleanup", + await waiting, + 424242, + ); + + const incomplete = path.join(root, "incomplete.pid"); + fs.writeFileSync(incomplete, "7"); + let rejection: unknown; + try { + await waitForProcessId(incomplete, 20); + } catch (error) { + rejection = error; + } + TestValidator.predicate( + "a pid without its publication boundary times out", + rejection instanceof Error && + rejection.message.includes("timed out waiting for a complete process id"), + ); + }; + +const delay = (milliseconds: number): Promise => + new Promise((resolve) => setTimeout(resolve, milliseconds)); diff --git a/tests/test-graph/src/features/test_provider_registry_selects_one_owner_per_language.ts b/tests/test-graph/src/features/test_provider_registry_selects_one_owner_per_language.ts index 76c5bd42..9c412e5d 100644 --- a/tests/test-graph/src/features/test_provider_registry_selects_one_owner_per_language.ts +++ b/tests/test-graph/src/features/test_provider_registry_selects_one_owner_per_language.ts @@ -152,7 +152,6 @@ async function assertSelection(): Promise { GRAPH_PROVIDERS.filter((provider) => [ "scip-java", - "scip-dotnet", "scip-python", "scip-ruby", "scip-dart", @@ -162,7 +161,6 @@ async function assertSelection(): Promise { ), { "scip-java": ["contains", "references"], - "scip-dotnet": [], "scip-python": ["references"], "scip-ruby": [], "scip-dart": [], @@ -509,25 +507,23 @@ async function assertSelection(): Promise { TestValidator.equals( "only languages without a truthful strict producer remain unowned", publicLanguages.filter((language) => !owners.has(language)), - ["swift", "scala", "zig"], + ["zig"], ); TestValidator.equals( "C and C++ share one compilation-universe provider", owners.get("c"), owners.get("cpp"), ); - // One launcher, two compiler plugins, two owners. They used to be one - // registry entry claiming both languages, which stopped being viable the - // moment a Java-only producer existed: a fallback owns the same atomic - // languages as the route it backs, so a strict Java route could only take - // the SCIP entry as its fallback by taking Kotlin's ownership with it. + // One JVM, three compiler integrations, three owners. Each preferred + // compiler route owns only its language and carries its ordinary route as + // fallback. TestValidator.equals( - "Java and Kotlin own their producers independently, and Scala neither", + "Java, Kotlin, and Scala own their producers independently", [owners.get("java"), owners.get("kotlin"), owners.get("scala")], [ ["javac-graph"], - ["scip-kotlinc"], - undefined, + ["kotlinc-graph"], + ["scalac-graph"], ], ); } diff --git a/tests/test-graph/src/features/test_repository_context_adapters_preserve_authoritative_models.ts b/tests/test-graph/src/features/test_repository_context_adapters_preserve_authoritative_models.ts index 4f316e52..a9e4dc62 100644 --- a/tests/test-graph/src/features/test_repository_context_adapters_preserve_authoritative_models.ts +++ b/tests/test-graph/src/features/test_repository_context_adapters_preserve_authoritative_models.ts @@ -6,12 +6,16 @@ import { cmakeRepositoryContextProvider, gradleRepositoryContextProvider, pnpmRepositoryContextProvider, + resolveCargoCommand, } from "@samchon/graph"; +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; +import { pathToFileURL } from "node:url"; import { GraphPaths } from "../internal/GraphPaths"; import { parseGradleRepositoryContextModel } from "../../../../packages/graph/src/repository/parseGradleRepositoryContextModel"; +import { isSubPath } from "../../../../packages/graph/src/utils/isSubPath"; /** * Each adapter reads a different owning tool, and the tempting failure is the @@ -37,6 +41,45 @@ export const test_repository_context_adapters_preserve_authoritative_models = const cargo = cargoFixture(root); const gradle = gradleFixture(root); const cmake = cmakeFixture(root); + assertTopologyPhaseTrace(); + + const providerSources = [ + "cargoRepositoryContextProvider.ts", + "gradleRepositoryContextProvider.ts", + "cmakeRepositoryContextProvider.ts", + ].map((file) => + fs.readFileSync( + path.join( + GraphPaths.repositoryRoot, + "packages", + "graph", + "src", + "repository", + file, + ), + "utf8", + ), + ); + TestValidator.predicate( + "Cargo, Gradle and CMake share the canonical containment rule", + providerSources.every( + (source) => + source.includes('import { isSubPath } from "../utils/isSubPath"') && + !source.includes("function isInside"), + ), + ); + if (process.platform === "win32") { + const drive = path.parse(root).root.toUpperCase(); + const otherDrive = drive.startsWith("C:") ? "D:\\" : "C:\\"; + TestValidator.equals( + "canonical containment rejects Windows cross-drive paths and accepts case-only root spelling", + [ + isSubPath(root, path.join(otherDrive, "foreign", "file")), + isSubPath(root.toUpperCase(), path.join(root, "inside")), + ], + [false, true], + ); + } TestValidator.equals( "repository-context adapters detect only their owning manifests", @@ -342,6 +385,50 @@ export const test_repository_context_adapters_preserve_authoritative_models = (node) => node.kind === "entrypoint" && node.name.startsWith("exports"), ), ); + const pnpmBoundary = pnpmBoundaryFixture(root); + TestValidator.equals( + "pnpm publishes only canonical package directories as roots and marks escaping entrypoints external", + { + roots: pnpmBoundary.shards[0]!.nodes + .filter( + (node) => + node.kind === "source-root" || + node.kind === "generated-root", + ) + .map((node) => [node.kind, node.root, node.external]) + .sort((left, right) => + String(left[1]) < String(right[1]) ? -1 : 1, + ), + entrypoints: pnpmBoundary.shards[0]!.nodes + .filter((node) => node.kind === "entrypoint") + .map((node) => [node.file, node.external]), + }, + { + roots: [ + ["generated-root", "pkg/dist", false], + ["source-root", "pkg/src", false], + ], + entrypoints: [["../outside-entry.js", true]], + }, + ); + for (const [field, value] of [ + ["files", "src"], + ["scripts", "build"], + ["bin", ["cli.js"]], + ["exports", 1], + ["main", 1], + ] as const) { + TestValidator.error( + `pnpm rejects a malformed ${field} field with its manifest identity`, + () => pnpmMalformedManifestFixture(root, field, value), + ); + } + TestValidator.error("pnpm names an invalid JSON manifest", () => + pnpmInvalidManifestFixture(root, "{"), + ); + TestValidator.error("pnpm rejects a non-object manifest root", () => + pnpmInvalidManifestFixture(root, "[]"), + ); TestValidator.equals( "pnpm falls back to package.json evidence when no workspace manifest is present", pnpmNoWorkspaceFixture(root).shards[0]!.nodes[0]!.evidence?.file, @@ -404,12 +491,46 @@ export const test_repository_context_adapters_preserve_authoritative_models = exerciseGradleModelParser(root); const toolDirectory = path.join(root, "tools"); + const missingToolEnv = { + ...process.env, + PATH: "", + SAMCHON_GRAPH_CARGO: undefined, + }; + TestValidator.error("Cargo refuses an absent native executable", () => + cargoRepositoryContextProvider.collect({ + root: path.join(root, "cargo"), + env: missingToolEnv, + }), + ); + TestValidator.equals( + "Cargo reports an absent version probe when model collection is injected", + cargoRepositoryContextProvider.collect( + { root: path.join(root, "cargo"), env: missingToolEnv }, + () => cargoModel(root), + ).toolVersion, + "", + ); installFakeRepositoryTool(toolDirectory, "pnpm"); installFakeRepositoryTool(toolDirectory, "cargo"); const toolEnv = { ...process.env, PATH: `${toolDirectory}${path.delimiter}${process.env.PATH ?? ""}`, + SAMCHON_GRAPH_CARGO: path.join( + toolDirectory, + process.platform === "win32" ? "cargo.cmd" : "cargo", + ), }; + const tracedSession = pnpmRepositoryContextProvider.open({ + root, + env: { + ...toolEnv, + FIXTURE_TOOL_MODEL: JSON.stringify(pnpmModel(root)), + SAMCHON_GRAPH_TOPOLOGY_TRACE: "1", + }, + }); + await tracedSession.refresh(); + await tracedSession.close(); + assertNativeCargoResolution(root); TestValidator.predicate( "the pnpm process boundary accepts a valid resolved workspace model", pnpmRepositoryContextProvider.collect({ @@ -460,7 +581,19 @@ export const test_repository_context_adapters_preserve_authoritative_models = for (const [provider, invalidModels] of [ [ pnpmRepositoryContextProvider, - [JSON.stringify([{ path: "" }])], + [ + JSON.stringify([1]), + JSON.stringify([{ path: "" }]), + JSON.stringify([{ path: root, name: 1 }]), + JSON.stringify([{ path: root, private: "invalid" }]), + JSON.stringify([{ path: root, dependencies: "invalid" }]), + JSON.stringify([ + { + path: root, + dependencies: { invalid: { path: 1 } }, + }, + ]), + ], ], [ cargoRepositoryContextProvider, @@ -516,6 +649,8 @@ export const test_repository_context_adapters_preserve_authoritative_models = ); } + await exerciseNestedWorkspaceDiscovery(root, toolEnv); + exerciseCmakeRefusals(root); const aborted = new AbortController(); @@ -613,6 +748,7 @@ function pnpmModel(root: string) { { name: "@fixture/app", path: path.join(root, "apps", "app"), + private: true, dependencies: { "@fixture/lib": { path: path.join(root, "packages", "lib") }, }, @@ -630,7 +766,7 @@ function pnpmEdgeFixture(root: string) { bin: "cli.js", exports: { ".": { - import: "esm.js", + import: ["esm.js", null], ignored: null, }, "./feature": "feature.js", @@ -652,6 +788,66 @@ function pnpmEdgeFixture(root: string) { ); } +function pnpmBoundaryFixture(root: string) { + const workspace = path.join(root, "pnpm-boundary"); + const pkg = path.join(workspace, "pkg"); + write(path.join(workspace, "pnpm-workspace.yaml"), "packages:\n - pkg\n"); + writeJson(path.join(workspace, "package.json"), { + name: "boundary-workspace", + private: true, + }); + write(path.join(root, "outside-entry.js"), "export {};\n"); + write(path.join(workspace, "outside-root", "index.ts"), "export {};\n"); + write(path.join(pkg, "README.md"), "fixture\n"); + write(path.join(pkg, "lint.config.ts"), "export {};\n"); + write(path.join(pkg, "src", "index.ts"), "export {};\n"); + writeJson(path.join(pkg, "package.json"), { + name: "boundary-package", + files: [ + "README.md", + "lint.config.ts", + "src", + "dist", + "../outside-root", + "C:drive-relative-root", + ], + main: path.join(root, "outside-entry.js"), + }); + return pnpmRepositoryContextProvider.collect( + { root: workspace, env: process.env }, + () => [{ name: "boundary-package", path: pkg }], + ); +} + +function pnpmMalformedManifestFixture( + root: string, + field: "files" | "scripts" | "bin" | "exports" | "main", + value: unknown, +): void { + const workspace = path.join(root, `pnpm-malformed-${field}`); + const pkg = path.join(workspace, "pkg"); + write(path.join(workspace, "pnpm-workspace.yaml"), "packages:\n - pkg\n"); + writeJson(path.join(pkg, "package.json"), { + name: `malformed-${field}`, + [field]: value, + }); + pnpmRepositoryContextProvider.collect( + { root: workspace, env: process.env }, + () => [{ name: `malformed-${field}`, path: pkg }], + ); +} + +function pnpmInvalidManifestFixture(root: string, content: string): void { + const workspace = path.join(root, `pnpm-invalid-${content.length}`); + const pkg = path.join(workspace, "pkg"); + write(path.join(workspace, "pnpm-workspace.yaml"), "packages:\n - pkg\n"); + write(path.join(pkg, "package.json"), content); + pnpmRepositoryContextProvider.collect( + { root: workspace, env: process.env }, + () => [{ name: "invalid", path: pkg }], + ); +} + function pnpmNoWorkspaceFixture(root: string) { const workspace = path.join(root, "pnpm-no-workspace"); writeJson(path.join(workspace, "package.json"), { @@ -1453,6 +1649,223 @@ function cmakeScenario( return reply; } +async function exerciseNestedWorkspaceDiscovery( + root: string, + toolEnv: NodeJS.ProcessEnv, +): Promise { + await exerciseNestedPnpmDiscovery(root, toolEnv); + await exerciseNestedCargoDiscovery(root, toolEnv); +} + +async function exerciseNestedPnpmDiscovery( + root: string, + toolEnv: NodeJS.ProcessEnv, +): Promise { + const workspace = path.join(root, "nested-pnpm"); + const first = path.join(workspace, "groups", "a", "one"); + const created = path.join(workspace, "groups", "b", "two"); + const renamed = path.join(workspace, "groups", "c", "two"); + write( + path.join(workspace, "pnpm-workspace.yaml"), + "packages:\n - groups/*/*\n", + ); + write(path.join(workspace, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n"); + writeJson(path.join(first, "package.json"), { name: "one" }); + const env = { + ...toolEnv, + FIXTURE_TOOL_MODEL: JSON.stringify([{ name: "one", path: first }]), + }; + const session = pnpmRepositoryContextProvider.open({ + root: workspace, + env, + }); + const initial = await session.refresh(); + const unchanged = await session.refresh(); + writeJson(path.join(created, "package.json"), { name: "two" }); + env.FIXTURE_TOOL_MODEL = JSON.stringify([ + { name: "one", path: first }, + { name: "two", path: created }, + ]); + const afterCreate = await session.refresh(); + fs.renameSync(path.dirname(created), path.dirname(renamed)); + env.FIXTURE_TOOL_MODEL = JSON.stringify([ + { name: "one", path: first }, + { name: "two", path: renamed }, + ]); + const afterRename = await session.refresh(); + fs.rmSync(path.dirname(renamed), { recursive: true, force: true }); + env.FIXTURE_TOOL_MODEL = JSON.stringify([{ name: "one", path: first }]); + const afterDelete = await session.refresh(); + write(path.join(workspace, "pnpm-lock.yaml"), "lockfileVersion: '9.1'\n"); + env.FIXTURE_TOOL_MODE = "failed"; + await TestValidator.error("a failed nested pnpm refresh rejects", () => + session.refresh(), + ); + const retained = session.current; + delete env.FIXTURE_TOOL_MODE; + const recovered = await session.refresh(); + TestValidator.equals( + "pnpm observes deep member create, rename, delete, no-op, failure and recovery", + [ + initial.generation, + unchanged.changed, + afterCreate.generation, + packageNames(afterCreate.snapshot), + afterRename.generation, + packageNames(afterRename.snapshot), + afterDelete.generation, + packageNames(afterDelete.snapshot), + retained?.generation.sequence, + recovered.generation, + ], + [ + 1, + false, + 2, + ["one", "two"], + 3, + ["one", "two"], + 4, + ["one"], + 4, + 5, + ], + ); + await session.close(); +} + +async function exerciseNestedCargoDiscovery( + root: string, + toolEnv: NodeJS.ProcessEnv, +): Promise { + const workspace = path.join(root, "nested-cargo"); + const first = path.join(workspace, "groups", "a", "one"); + const created = path.join(workspace, "groups", "b", "two"); + const renamed = path.join(workspace, "groups", "c", "two"); + write( + path.join(workspace, "Cargo.toml"), + "[workspace]\nmembers=['groups/*/*']\nresolver='2'\n", + ); + write(path.join(workspace, "Cargo.lock"), ""); + writeCargoMember(first, "one"); + const env = { + ...toolEnv, + FIXTURE_TOOL_MODEL: JSON.stringify( + nestedCargoModel(workspace, [["one", first]]), + ), + }; + const session = cargoRepositoryContextProvider.open({ + root: workspace, + env, + }); + const initial = await session.refresh(); + const unchanged = await session.refresh(); + writeCargoMember(created, "two"); + env.FIXTURE_TOOL_MODEL = JSON.stringify( + nestedCargoModel(workspace, [ + ["one", first], + ["two", created], + ]), + ); + const afterCreate = await session.refresh(); + fs.renameSync(path.dirname(created), path.dirname(renamed)); + env.FIXTURE_TOOL_MODEL = JSON.stringify( + nestedCargoModel(workspace, [ + ["one", first], + ["two", renamed], + ]), + ); + const afterRename = await session.refresh(); + fs.rmSync(path.dirname(renamed), { recursive: true, force: true }); + env.FIXTURE_TOOL_MODEL = JSON.stringify( + nestedCargoModel(workspace, [["one", first]]), + ); + const afterDelete = await session.refresh(); + write(path.join(workspace, "Cargo.lock"), "# moved\n"); + env.FIXTURE_TOOL_MODE = "failed"; + await TestValidator.error("a failed nested Cargo refresh rejects", () => + session.refresh(), + ); + const retained = session.current; + delete env.FIXTURE_TOOL_MODE; + const recovered = await session.refresh(); + TestValidator.equals( + "Cargo observes deep member create, rename, delete, no-op, failure and recovery", + [ + initial.generation, + unchanged.changed, + afterCreate.generation, + packageNames(afterCreate.snapshot), + afterRename.generation, + packageNames(afterRename.snapshot), + afterDelete.generation, + packageNames(afterDelete.snapshot), + retained?.generation.sequence, + recovered.generation, + ], + [ + 1, + false, + 2, + ["one", "two"], + 3, + ["one", "two"], + 4, + ["one"], + 4, + 5, + ], + ); + await session.close(); +} + +function writeCargoMember(directory: string, name: string): void { + write( + path.join(directory, "Cargo.toml"), + `[package]\nname='${name}'\nversion='1.0.0'\n`, + ); + write(path.join(directory, "src", "lib.rs"), "pub fn fixture() {}\n"); +} + +function nestedCargoModel( + workspace: string, + members: ReadonlyArray, +) { + return { + workspace_root: workspace, + workspace_members: members.map(([name]) => `${name} 1`), + packages: members.map(([name, directory]) => ({ + id: `${name} 1`, + name, + version: "1.0.0", + manifest_path: path.join(directory, "Cargo.toml"), + targets: [ + { + name, + kind: ["lib"], + crate_types: ["lib"], + src_path: path.join(directory, "src", "lib.rs"), + }, + ], + })), + resolve: { + nodes: members.map(([name]) => ({ + id: `${name} 1`, + dependencies: [], + })), + }, + }; +} + +function packageNames(snapshot: { + nodes: readonly { kind: string; name: string; external: boolean }[]; +}): string[] { + return snapshot.nodes + .filter((node) => node.kind === "package" && !node.external) + .map((node) => node.name) + .sort(); +} + function installFakeRepositoryTool(directory: string, name: string): void { fs.mkdirSync(directory, { recursive: true }); const source = [ @@ -1477,6 +1890,76 @@ function installFakeRepositoryTool(directory: string, name: string): void { } } +function assertNativeCargoResolution(root: string): void { + const workspace = path.join(root, "cargo-native-resolution"); + const privateBin = path.join(workspace, ".samchon-graph", "bin"); + fs.mkdirSync(privateBin, { recursive: true }); + const executable = path.join( + privateBin, + process.platform === "win32" ? "cargo.exe" : "cargo", + ); + if (process.platform === "win32") fs.copyFileSync(process.execPath, executable); + else { + write(executable, "#!/bin/sh\nexit 0\n"); + fs.chmodSync(executable, 0o755); + } + const resolved = resolveCargoCommand( + workspace, + { ...process.env, PATH: "" }, + ["--version"], + ); + TestValidator.predicate( + "Cargo repository context resolves the platform-native executable without inventing cargo.cmd", + resolved !== undefined && + path.resolve(resolved.command) === path.resolve(executable) && + resolved.args.includes("--version") && + !resolved.command.toLowerCase().endsWith("cargo.cmd"), + ); +} + +function assertTopologyPhaseTrace(): void { + const module = pathToFileURL( + path.join( + GraphPaths.repositoryRoot, + "packages", + "graph", + "lib", + "repository", + "topologyPhaseTrace.js", + ), + ).href; + const traced = spawnSync( + process.execPath, + [ + "--input-type=module", + "-e", + `import { topologyPhaseTrace } from ${JSON.stringify(module)}; topologyPhaseTrace("fixture", "join", performance.now() - 5, { nodes: 2 });`, + ], + { + encoding: "utf8", + env: process.env, + windowsHide: true, + }, + ); + const prefix = "@samchon/graph: topology-phase="; + const line = traced.stderr + .split(/\r?\n/) + .find((entry) => entry.startsWith(prefix)); + const row = JSON.parse(line?.slice(prefix.length) ?? "null") as Record< + string, + unknown + > | null; + TestValidator.equals( + "topology phase traces are opt-in structured diagnostics", + [traced.status, row?.schemaVersion, row?.provider, row?.phase, row?.nodes], + [0, 1, "fixture", "join", 2], + ); + TestValidator.predicate( + "topology phase traces report a nonnegative duration", + typeof row?.durationMs === "number" && row.durationMs >= 0, + ); +} + function write(file: string, content: string): void { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, content); diff --git a/tests/test-graph/src/features/test_resident_bulk_provider_polls_generations_without_reprocessing_strict_facts.ts b/tests/test-graph/src/features/test_resident_bulk_provider_polls_generations_without_reprocessing_strict_facts.ts index 3b4d83d2..fe99fc43 100644 --- a/tests/test-graph/src/features/test_resident_bulk_provider_polls_generations_without_reprocessing_strict_facts.ts +++ b/tests/test-graph/src/features/test_resident_bulk_provider_polls_generations_without_reprocessing_strict_facts.ts @@ -7,6 +7,8 @@ import path from "node:path"; import { createResidentGraphSource } from "../../../../packages/graph/src/indexer/createResidentGraphSource"; import { IIndexerResult } from "../../../../packages/graph/src/indexer/IIndexerResult"; import { IBulkGraphSession } from "../../../../packages/graph/src/provider/IBulkGraphSession"; +import { dumpProvenanceOf } from "../../../../packages/graph/src/provider/dumpProvenanceOf"; +import { graphSnapshotDigests } from "../../../../packages/graph/src/provider/graphSnapshotDigests"; export const test_resident_bulk_provider_polls_generations_without_reprocessing_strict_facts = async () => { @@ -108,7 +110,39 @@ export const test_resident_bulk_provider_polls_generations_without_reprocessing_ [...resident.modes()], [["ttscgraph", "initial"]], ); - const unchanged = await resident.load(); + const originalReadFileSync = fs.readFileSync; + const originalReaddirSync = fs.readdirSync; + let sourceReads = 0; + let directoryReads = 0; + fs.readFileSync = ((target: fs.PathOrFileDescriptor, ...args: unknown[]) => { + if (typeof target === "string" && path.resolve(target) === file) { + sourceReads += 1; + } + return Reflect.apply(originalReadFileSync, fs, [ + target, + ...args, + ]) as ReturnType; + }) as typeof fs.readFileSync; + fs.readdirSync = ((...args: unknown[]) => { + directoryReads += 1; + return Reflect.apply( + originalReaddirSync as (...values: unknown[]) => unknown, + fs, + args, + ); + }) as typeof fs.readdirSync; + let unchanged; + try { + unchanged = await resident.load(); + } finally { + fs.readFileSync = originalReadFileSync; + fs.readdirSync = originalReaddirSync; + } + TestValidator.equals( + "an unchanged compiler-owned generation performs no coordinator corpus walk or source read", + [directoryReads, sourceReads], + [0, 0], + ); TestValidator.predicate( "an unchanged bulk generation reuses the resident dump object", unchanged === loaded, @@ -150,7 +184,645 @@ export const test_resident_bulk_provider_polls_generations_without_reprocessing_ ); await resident.close(); TestValidator.equals("resident shutdown closes its owned bulk session once", closes, 1); + await testFactEquivalentBulkGeneration(file, root); + await testFactEquivalentFallbacksAndRaces(); + }; + +async function testFactEquivalentBulkGeneration( + originalFile: string, + originalRoot: string, +): Promise { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), "samchon-graph-resident-equivalent-"), + ); + const file = path.join(root, "a.ts"); + const companion = path.join(root, "b.ts"); + const external = path.join(root, "reference.dll"); + const firstText = "export function answer() { return 1; }\n"; + const secondText = "export function answer() { return 2; }\n"; + const companionText = "export const stable = 1;\n"; + const externalText = "fixture reference bytes\n"; + fs.writeFileSync(file, firstText); + fs.writeFileSync(companion, companionText); + fs.writeFileSync(external, externalText); + const first = factEquivalentSnapshot( + file, + firstText, + companion, + companionText, + external, + externalText, + 1, + ); + const second = factEquivalentSnapshot( + file, + secondText, + companion, + companionText, + external, + externalText, + 2, + ); + const externalStale = factEquivalentSnapshot( + file, + secondText, + companion, + companionText, + external, + externalText, + 3, + ); + const stale = factEquivalentSnapshot( + file, + secondText, + companion, + companionText, + external, + externalText, + 4, + ); + const missing = factEquivalentSnapshot( + file, + secondText, + companion, + companionText, + external, + externalText, + 5, + ); + let current = first; + let generation = 1; + let refreshes = 0; + const session: IBulkGraphSession = { + kind: "bulk", + languages: ["typescript"], + root, + get generation() { + return generation; + }, + get current() { + return current; + }, + async refresh() { + refreshes += 1; + current = + refreshes === 1 + ? second + : refreshes === 2 + ? externalStale + : refreshes === 3 + ? stale + : missing; + generation += 1; + return { + changed: true, + generation, + mode: "incremental", + snapshot: current, + }; + }, + async close() {}, + }; + const initialDump = { + project: root, + languages: ["typescript"] as const, + indexer: "lsp" as const, + nodes: first.nodes, + edges: first.edges, + diagnostics: first.diagnostics, + coverage: first.coverage, + unresolved: first.unresolved, + warnings: first.warnings, + provenance: [dumpProvenanceOf(first)], + }; + const resident = createResidentGraphSource( + { cwd: root, languages: ["typescript"] }, + { + providers: [], + buildLspGraph: async () => + ({ + dump: initialDump, + warnings: [], + sessions: new Map([["typescript", session]]), + sources: new Map(), + inputManifest: new Map([ + [file, hash(firstText)], + [companion, hash(companionText)], + ]), + modes: new Map([ + ["fixture-compiler", "initial"], + ]), + }) as IIndexerResult, + }, + ); + const loaded = await resident.load(); + const initialGeneration = loaded.generation?.input; + fs.writeFileSync(file, secondText); + const originalReadFileSync = fs.readFileSync; + const originalReaddirSync = fs.readdirSync; + let sourceReads = 0; + let directoryReads = 0; + fs.readFileSync = ((target: fs.PathOrFileDescriptor, ...args: unknown[]) => { + if ( + typeof target === "string" && + [file, companion].includes(path.resolve(target)) + ) { + sourceReads += 1; + } + return Reflect.apply(originalReadFileSync, fs, [ + target, + ...args, + ]) as ReturnType; + }) as typeof fs.readFileSync; + fs.readdirSync = ((...args: unknown[]) => { + directoryReads += 1; + return Reflect.apply( + originalReaddirSync as (...values: unknown[]) => unknown, + fs, + args, + ); + }) as typeof fs.readdirSync; + let changed; + try { + changed = await withRoslynTrace(() => resident.load()); + } finally { + fs.readFileSync = originalReadFileSync; + fs.readdirSync = originalReaddirSync; + } + TestValidator.predicate( + "a fact-equivalent compiler generation advances only its manifest envelope", + changed !== loaded && + changed.nodes === loaded.nodes && + changed.edges === loaded.edges && + changed.provenance?.[0]?.manifest === + graphSnapshotDigests.manifestOf(second) && + changed.generation?.input !== initialGeneration, + ); + TestValidator.equals( + "a fact-equivalent generation fences every tracked provider source without a directory walk", + [directoryReads, sourceReads], + [0, 2], + ); + + fs.writeFileSync(external, "moved reference bytes\n"); + await rejects( + resident.load(), + "an untracked provider source moved after fact-equivalent preparation is rejected", + ); + TestValidator.predicate( + "an untracked stale source leaves the published dump intact", + changed.nodes === loaded.nodes && changed.provenance?.[0]?.manifest === + graphSnapshotDigests.manifestOf(second), + ); + fs.writeFileSync(external, externalText); + fs.writeFileSync(companion, "export const stable = 2;\n"); + await rejects( + resident.load(), + "a source moved after fact-equivalent preparation is rejected even when its provider digest is unchanged", + ); + TestValidator.predicate( + "an unrelated stale source leaves the published dump intact", + changed.nodes === loaded.nodes && changed.provenance?.[0]?.manifest === + graphSnapshotDigests.manifestOf(second), + ); + fs.writeFileSync(companion, companionText); + fs.rmSync(file); + await rejects( + resident.load(), + "a source deleted after fact-equivalent preparation is rejected", + ); + TestValidator.predicate( + "a rejected fact-equivalent generation leaves the published dump intact", + changed.nodes === loaded.nodes && changed.provenance?.[0]?.manifest === + graphSnapshotDigests.manifestOf(second), + ); + await resident.close(); + + // Keep both temporary roots live until their resident sessions have closed; + // the parent test's fixture remains an independent ordinary-refresh oracle. + TestValidator.predicate( + "fact-equivalent and ordinary refresh fixtures use isolated roots", + originalFile.startsWith(originalRoot) && root !== originalRoot, + ); +} + +async function testFactEquivalentFallbacksAndRaces(): Promise { + await testMetadataFallbacks(); + await testReplacedCandidateFence(); + await testCloseFence(); + await testMultipleBulkOwnerFallback(); +} + +async function testMetadataFallbacks(): Promise { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), "samchon-graph-resident-equivalent-metadata-"), + ); + const file = path.join(root, "a.ts"); + const companion = path.join(root, "b.ts"); + const external = path.join(root, "reference.dll"); + const added = path.join(root, "added-reference.dll"); + const firstText = "export function answer() { return 1; }\n"; + const secondText = "export function answer() { return 2; }\n"; + const companionText = "export const stable = 1;\n"; + const externalText = "fixture reference bytes\n"; + const addedText = "added reference bytes\n"; + fs.writeFileSync(file, firstText); + fs.writeFileSync(companion, companionText); + fs.writeFileSync(external, externalText); + fs.writeFileSync(added, addedText); + const initial = factEquivalentSnapshot( + file, + firstText, + companion, + companionText, + external, + externalText, + 1, + ); + const warningMoved = { + ...factEquivalentSnapshot( + file, + secondText, + companion, + companionText, + external, + externalText, + 2, + ), + warnings: ["moved warning"], + }; + const membershipMovedBase = factEquivalentSnapshot( + file, + secondText, + companion, + companionText, + external, + externalText, + 3, + ); + const membershipMoved = { + ...membershipMovedBase, + sources: new Map([ + ...membershipMovedBase.sources, + [ + added, + { + checkerDigest: hash(addedText), + diskDigest: hash(addedText), + }, + ] as const, + ]), + warnings: ["moved warning"], + }; + let current: IBulkGraphSession.ISnapshot = initial; + let generation = 1; + let refreshes = 0; + const session: IBulkGraphSession = { + kind: "bulk", + languages: ["typescript"], + root, + get generation() { + return generation; + }, + get current() { + return current; + }, + async refresh() { + refreshes += 1; + generation += 1; + current = refreshes === 1 ? warningMoved : membershipMoved; + return { + changed: refreshes < 3, + generation, + mode: refreshes < 3 ? "incremental" : "unchanged", + snapshot: current, + }; + }, + async close() {}, }; + const resident = residentFor(root, [file, companion], session, initial); + await resident.load(); + fs.writeFileSync(file, secondText); + const warningDump = await resident.load(); + const membershipDump = await resident.load(); + const generationOnlyDump = await resident.load(); + TestValidator.predicate( + "warning and source-membership movement take the full transaction before a generation-only poll", + warningDump.warnings.includes("moved warning") && + membershipDump.provenance?.[0]?.manifest === + graphSnapshotDigests.manifestOf(membershipMoved) && + generationOnlyDump.nodes.length === membershipDump.nodes.length, + ); + await resident.close(); +} + +async function testReplacedCandidateFence(): Promise { + const fixture = singleSessionFixture("replaced"); + const returned = fixture.next; + const replaced = { + ...fixture.next, + protocol: { ...fixture.next.protocol!, generation: "generation-3", sequence: 3 }, + }; + fixture.session.refresh = async () => { + fixture.advance(replaced); + return { + changed: true, + generation: fixture.session.generation, + mode: "incremental", + snapshot: returned, + }; + }; + const resident = residentFor( + fixture.root, + [fixture.file, fixture.companion], + fixture.session, + fixture.initial, + ); + await resident.load(); + fs.writeFileSync(fixture.file, fixture.secondText); + await rejects( + resident.load(), + "a provider replacing its candidate during preparation is rejected", + ); + await resident.close(); +} + +async function testCloseFence(): Promise { + const fixture = singleSessionFixture("close"); + fixture.session.refresh = async () => { + fixture.advance(fixture.next); + return { + changed: true, + generation: fixture.session.generation, + mode: "incremental", + snapshot: fixture.next, + }; + }; + const resident = residentFor( + fixture.root, + [fixture.file, fixture.companion], + fixture.session, + fixture.initial, + ); + await resident.load(); + fs.writeFileSync(fixture.file, fixture.secondText); + const originalReadFileSync = fs.readFileSync; + let closing: Promise | undefined; + fs.readFileSync = ((target: fs.PathOrFileDescriptor, ...args: unknown[]) => { + const value = Reflect.apply(originalReadFileSync, fs, [ + target, + ...args, + ]) as ReturnType; + if ( + closing === undefined && + typeof target === "string" && + path.resolve(target) === fixture.file + ) { + closing = resident.close(); + } + return value; + }) as typeof fs.readFileSync; + try { + await rejects( + resident.load(), + "closing during the final source fence aborts publication", + ); + } finally { + fs.readFileSync = originalReadFileSync; + } + await closing; +} + +async function testMultipleBulkOwnerFallback(): Promise { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), "samchon-graph-resident-multiple-bulk-"), + ); + const typescript = path.join(root, "a.ts"); + const go = path.join(root, "b.go"); + const typescriptText = "export const answer = 1;\n"; + const goText = "package fixture\nvar Answer = 1\n"; + fs.writeFileSync(typescript, typescriptText); + fs.writeFileSync(go, goText); + const firstTypescript = languageSnapshot( + typescript, + typescriptText, + "typescript", + "typescript-first", + ); + const secondTypescript = languageSnapshot( + typescript, + typescriptText, + "typescript", + "typescript-second", + ); + const firstGo = languageSnapshot(go, goText, "go", "go-first"); + const secondGo = languageSnapshot(go, goText, "go", "go-second"); + const typescriptSession = changingSession( + root, + "typescript", + firstTypescript, + secondTypescript, + ); + const goSession = changingSession(root, "go", firstGo, secondGo); + const initialDump = { + project: root, + languages: ["typescript", "go"] as const, + indexer: "lsp" as const, + nodes: [...firstTypescript.nodes, ...firstGo.nodes], + edges: [], + diagnostics: [], + warnings: [], + provenance: [ + dumpProvenanceOf(firstTypescript), + dumpProvenanceOf(firstGo), + ], + }; + const resident = createResidentGraphSource( + { cwd: root, languages: ["typescript", "go"] }, + { + providers: [], + buildLspGraph: async () => + ({ + dump: initialDump, + warnings: [], + sessions: new Map([ + ["typescript", typescriptSession], + ["go", goSession], + ]), + sources: new Map(), + inputManifest: new Map([ + [typescript, hash(typescriptText)], + [go, hash(goText)], + ]), + modes: new Map(), + }) as IIndexerResult, + }, + ); + await resident.load(); + const changed = await resident.load(); + TestValidator.equals( + "distinct bulk owners bypass the single-owner fact reuse optimization", + changed.nodes.map((node) => node.name).sort(), + ["go-second", "typescript-second"], + ); + await resident.close(); +} + +function residentFor( + root: string, + files: readonly string[], + session: IBulkGraphSession, + initial: IBulkGraphSession.ISnapshot, +) { + const initialDump = { + project: root, + languages: ["typescript"] as const, + indexer: "lsp" as const, + nodes: initial.nodes, + edges: initial.edges, + diagnostics: initial.diagnostics, + coverage: initial.coverage, + unresolved: initial.unresolved, + warnings: initial.warnings, + provenance: [dumpProvenanceOf(initial)], + }; + return createResidentGraphSource( + { cwd: root, languages: ["typescript"] }, + { + providers: [], + buildLspGraph: async () => + ({ + dump: initialDump, + warnings: [], + sessions: new Map([["typescript", session]]), + sources: new Map(), + inputManifest: new Map( + files.map((file) => [file, hash(fs.readFileSync(file))]), + ), + modes: new Map(), + }) as IIndexerResult, + }, + ); +} + +function singleSessionFixture(label: string) { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), `samchon-graph-resident-${label}-`), + ); + const file = path.join(root, "a.ts"); + const companion = path.join(root, "b.ts"); + const external = path.join(root, "reference.dll"); + const firstText = "export function answer() { return 1; }\n"; + const secondText = "export function answer() { return 2; }\n"; + const companionText = "export const stable = 1;\n"; + const externalText = "fixture reference bytes\n"; + fs.writeFileSync(file, firstText); + fs.writeFileSync(companion, companionText); + fs.writeFileSync(external, externalText); + const initial = factEquivalentSnapshot( + file, + firstText, + companion, + companionText, + external, + externalText, + 1, + ); + const next = factEquivalentSnapshot( + file, + secondText, + companion, + companionText, + external, + externalText, + 2, + ); + let current: IBulkGraphSession.ISnapshot = initial; + let generation = 1; + const session: IBulkGraphSession = { + kind: "bulk", + languages: ["typescript"], + root, + get generation() { + return generation; + }, + get current() { + return current; + }, + async refresh() { + throw new Error("fixture refresh was not installed"); + }, + async close() {}, + }; + return { + root, + file, + companion, + firstText, + secondText, + initial, + next, + session, + advance(snapshot: IBulkGraphSession.ISnapshot) { + generation += 1; + current = snapshot; + }, + }; +} + +function changingSession( + root: string, + language: "typescript" | "go", + initial: IBulkGraphSession.ISnapshot, + next: IBulkGraphSession.ISnapshot, +): IBulkGraphSession { + let current = initial; + let generation = 1; + return { + kind: "bulk", + languages: [language], + root, + get generation() { + return generation; + }, + get current() { + return current; + }, + async refresh() { + generation += 1; + current = next; + return { changed: true, generation, mode: "incremental", snapshot: next }; + }, + async close() {}, + }; +} + +function languageSnapshot( + file: string, + text: string, + language: "typescript" | "go", + name: string, +): IBulkGraphSession.ISnapshot { + const base = snapshot(file, text, name); + const relative = path.basename(file); + return { + ...base, + languages: [language], + nodes: base.nodes.map((node) => ({ + ...node, + id: `${relative}#${name}:variable`, + kind: "variable", + language, + file: relative, + evidence: { file: relative, startLine: 1, endLine: 1 }, + })), + provenance: { + ...base.provenance, + provider: `fixture-${language}`, + tool: `fixture-${language}`, + }, + }; +} function snapshot( file: string, @@ -194,6 +866,65 @@ function snapshot( }; } +function factEquivalentSnapshot( + file: string, + text: string, + companion: string, + companionText: string, + external: string, + externalText: string, + sequence: number, +): IBulkGraphSession.ISnapshot { + const base = snapshot(file, text, "stable"); + const companionDigest = hash(companionText); + const externalDigest = hash(externalText); + const sourceEntries = [ + ...base.sources, + [ + companion, + { + checkerDigest: companionDigest, + diskDigest: companionDigest, + }, + ] as const, + [ + external, + { + checkerDigest: externalDigest, + diskDigest: externalDigest, + }, + ] as const, + ]; + if (sequence % 2 === 0) sourceEntries.reverse(); + return { + ...base, + sources: new Map(sourceEntries), + provenance: { + ...base.provenance, + provider: "fixture-compiler", + }, + protocol: { + version: 1, + sequence, + generation: `generation-${sequence}`, + ...(sequence === 1 + ? {} + : { + baseSequence: sequence - 1, + baseGeneration: `generation-${sequence - 1}`, + }), + manifest: hash(text), + targets: ["app"], + shards: [], + factDigest: hash("stable graph facts"), + }, + }; +} + +function hash(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + async function rejects(task: Promise, label: string): Promise { let error: unknown; try { @@ -203,3 +934,14 @@ async function rejects(task: Promise, label: string): Promise { } TestValidator.predicate(label, error instanceof Error); } + +async function withRoslynTrace(task: () => Promise): Promise { + const prior = process.env["SAMCHON_GRAPH_ROSLYN_TRACE"]; + process.env["SAMCHON_GRAPH_ROSLYN_TRACE"] = "1"; + try { + return await task(); + } finally { + if (prior === undefined) delete process.env["SAMCHON_GRAPH_ROSLYN_TRACE"]; + else process.env["SAMCHON_GRAPH_ROSLYN_TRACE"] = prior; + } +} diff --git a/tests/test-graph/src/features/test_resident_close_interrupts_stalled_generic_lsp_build.ts b/tests/test-graph/src/features/test_resident_close_interrupts_stalled_generic_lsp_build.ts index 8caa206a..038ed371 100644 --- a/tests/test-graph/src/features/test_resident_close_interrupts_stalled_generic_lsp_build.ts +++ b/tests/test-graph/src/features/test_resident_close_interrupts_stalled_generic_lsp_build.ts @@ -4,6 +4,7 @@ import fs from "node:fs"; import path from "node:path"; import { GraphPaths } from "../internal/GraphPaths"; +import { waitForProcessId } from "../internal/waitForProcessId"; /** A resident close reaches generic LSP work that has not published a session. */ export const test_resident_close_interrupts_stalled_generic_lsp_build = @@ -44,8 +45,7 @@ const exercise = async (phase: string, serverArgs: string[]): Promise => { lspReadyQuietMs: 10, }); const loading = resident.load(); - await waitForFile(pidFile); - pid = Number(fs.readFileSync(pidFile, "utf8")); + pid = await waitForProcessId(pidFile); await waitForFile(phase === "readiness" ? progressFile : hangFile); const settled = await settleWithin( diff --git a/tests/test-graph/src/features/test_resident_close_interrupts_stalled_generic_lsp_refresh.ts b/tests/test-graph/src/features/test_resident_close_interrupts_stalled_generic_lsp_refresh.ts index dd25c852..ac34cbb7 100644 --- a/tests/test-graph/src/features/test_resident_close_interrupts_stalled_generic_lsp_refresh.ts +++ b/tests/test-graph/src/features/test_resident_close_interrupts_stalled_generic_lsp_refresh.ts @@ -4,6 +4,7 @@ import fs from "node:fs"; import path from "node:path"; import { GraphPaths } from "../internal/GraphPaths"; +import { waitForProcessId } from "../internal/waitForProcessId"; /** A resident close cancels every phase of an established LSP refresh. */ export const test_resident_close_interrupts_stalled_generic_lsp_refresh = @@ -40,8 +41,7 @@ const exercise = async (phase: string, serverArgs: string[]): Promise => { }); try { await resident.load(); - await waitForFile(pidFile); - pid = Number(fs.readFileSync(pidFile, "utf8")); + pid = await waitForProcessId(pidFile); fs.writeFileSync(source, "answer = 2\n"); const refreshing = resident.load(); diff --git a/tests/test-graph/src/features/test_resident_static_mode_never_starts_an_lsp_build.ts b/tests/test-graph/src/features/test_resident_static_mode_never_starts_an_lsp_build.ts index 7379d316..e26c6cdd 100644 --- a/tests/test-graph/src/features/test_resident_static_mode_never_starts_an_lsp_build.ts +++ b/tests/test-graph/src/features/test_resident_static_mode_never_starts_an_lsp_build.ts @@ -44,6 +44,11 @@ export const test_resident_static_mode_never_starts_an_lsp_build = async () => { staticBuilds, 2, ); + TestValidator.predicate( + "the initial static dump publishes the input generation its joins consume", + typeof dump.generation?.input === "string" && + dump.generation.input.length === 64, + ); let fallbackLspBuilds = 0; const fallback = createResidentGraphSource( @@ -59,7 +64,11 @@ export const test_resident_static_mode_never_starts_an_lsp_build = async () => { await fallback.close(); TestValidator.equals( "an omitted static dependency uses the canonical static builder", - [fallbackDump.indexer, fallbackLspBuilds], - ["static", 0], + [ + fallbackDump.indexer, + fallbackLspBuilds, + fallbackDump.generation?.input.length, + ], + ["static", 0, 64], ); }; diff --git a/tests/test-graph/src/features/test_rust_hir_client_restores_retries_and_fails_closed.ts b/tests/test-graph/src/features/test_rust_hir_client_restores_retries_and_fails_closed.ts index 525a8304..5928bae1 100644 --- a/tests/test-graph/src/features/test_rust_hir_client_restores_retries_and_fails_closed.ts +++ b/tests/test-graph/src/features/test_rust_hir_client_restores_retries_and_fails_closed.ts @@ -14,9 +14,10 @@ import { GraphPaths } from "../internal/GraphPaths.js"; * A resident producer answers before it is ready and restarts underneath a live * session, and neither condition is an error the caller may see as a fallback. * This pins the client's side of that: a cancelled or content-modified response - * is retried until the ready deadline rather than published, a no-op returns - * the exact resident object rather than an equal copy, a rejected restart - * checkpoint discards the persisted generation instead of reusing it, and a + * is retried until the caller's ready deadline when one exists rather than + * published, a no-op returns the exact resident object rather than an equal + * copy, a rejected restart checkpoint discards the persisted generation + * instead of reusing it, and a * checkpoint that cannot be written surfaces as a warning on the returned * snapshot rather than as a failed refresh. * @@ -143,12 +144,13 @@ async function assertCheckpointRejectionRecovers( } async function assertRetryBoundaries(root: string): Promise { - const retrying = rustClient(root, isolatedCache(), ["--retry=1", "--content-modified=1"], undefined, { - readyTimeoutMs: 1_000, - }); + const retrying = rustClient(root, isolatedCache(), [ + "--retry=1", + "--content-modified=1", + ]); TestValidator.equals( - "ServerCancelled and ContentModified are retried until the producer is ready", - (await retrying.refresh()).changed, + "an undefined deadline keeps retrying after the former private ceiling", + (await beyondLegacyReadyDeadline(() => retrying.refresh())).changed, true, ); await retrying.close(); @@ -510,6 +512,27 @@ function readRequests(file: string): Array<{ .map((line) => JSON.parse(line)); } +async function beyondLegacyReadyDeadline(operation: () => Promise) { + const original = Object.getOwnPropertyDescriptor(performance, "now"); + let first = true; + Object.defineProperty(performance, "now", { + configurable: true, + value: () => { + if (first) { + first = false; + return 0; + } + return 300_001; + }, + }); + try { + return await operation(); + } finally { + if (original === undefined) delete (performance as { now?: unknown }).now; + else Object.defineProperty(performance, "now", original); + } +} + function nodeShim( root: string, name: string, diff --git a/tests/test-graph/src/features/test_scalac_graph_publishes_bsp_semanticdb_generations.ts b/tests/test-graph/src/features/test_scalac_graph_publishes_bsp_semanticdb_generations.ts new file mode 100644 index 00000000..5d2f48c5 --- /dev/null +++ b/tests/test-graph/src/features/test_scalac_graph_publishes_bsp_semanticdb_generations.ts @@ -0,0 +1,359 @@ +import { TestValidator } from "@nestia/e2e"; +import { + IScalaGraphSnapshot, + SCALA_GRAPH_PROVIDER, + ScalaGraphSnapshotAdapter, + scalaGraphProvider, + selectGraphProviders, +} from "@samchon/graph"; +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { GraphPaths } from "../internal/GraphPaths"; + +const SOURCES = { + "src/scala-2/demo/Api.scala": + "package demo\n\nfinal class Api[A](value: A) {\n def run(): A = value\n}\n", + "src/scala-3/demo/Api.scala": + "package demo\n\nfinal class Api[A](value: A):\n def run(): A = value\n", +}; + +/** + * The Scala route keeps BSP targets separate and cross-checks typed-plugin + * facts against the SemanticDB document emitted by that same compile. + */ +export const test_scalac_graph_publishes_bsp_semanticdb_generations = + async (): Promise => { + const packagedScala = path.join( + GraphPaths.graphPackageRoot, + "sidecars", + "scala", + ); + TestValidator.predicate( + "the package carries the complete buildable Scala producer source", + [ + "README.md", + "pom.xml", + "scala2-plugin/src/main/resources/scalac-plugin.xml", + "scala3-plugin/src/main/resources/plugin.properties", + "server/src/main/scala/org/samchon/graph/scala/server/Main.scala", + ].every((file) => fs.existsSync(path.join(packagedScala, file))) && + !fs.existsSync(path.join(packagedScala, "server", "target")), + ); + const root = GraphPaths.createTempDirectory("samchon-graph-scalac-"); + for (const [file, text] of Object.entries(SOURCES)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + } + fs.mkdirSync(path.join(root, ".bsp"), { recursive: true }); + fs.writeFileSync( + path.join(root, ".bsp", "fixture.json"), + '{"name":"fixture","argv":["fixture-bsp"]}\n', + ); + fs.writeFileSync( + path.join(root, ".bsp", "other.json"), + '{"name":"other","argv":["other-bsp"]}\n', + ); + fs.writeFileSync(path.join(root, "build.sbt"), 'scalaVersion := "3.9.0"\n'); + + const windows = process.platform === "win32"; + const script = (name: string, body: string): string => { + const file = path.join(root, windows ? `${name}.cmd` : name); + fs.writeFileSync( + file, + windows ? `@echo off\r\n${body}\r\n` : `#!/bin/sh\n${body}\n`, + ); + if (!windows) fs.chmodSync(file, 0o755); + return file; + }; + const producer = (name: string, flags: readonly string[] = []): string => + script( + name, + `"${process.execPath}" "${GraphPaths.fakeScalaGraph}" ${flags.join(" ")} ${windows ? "%*" : '"$@"'}`, + ); + const java = script("java", "echo openjdk 21.0.12 2026-10-21"); + const current = producer("samchon-scala-graph"); + const environment = { + ...process.env, + SAMCHON_GRAPH_SCALA_GRAPH: current, + SAMCHON_GRAPH_JAVA_TOOLCHAIN: java, + }; + + const noBsp = GraphPaths.createTempDirectory("samchon-graph-scala-no-bsp-"); + fs.writeFileSync(path.join(noBsp, "Api.scala"), "object Api\n"); + TestValidator.predicate( + "a repository without a BSP connection declines the strict route", + selectGraphProviders(noBsp, ["scala"], {}, environment).candidates.every( + (candidate) => candidate.provider.name !== SCALA_GRAPH_PROVIDER, + ), + ); + TestValidator.predicate( + "a command without the resident capability declines", + selectGraphProviders(root, ["scala"], {}, { + ...environment, + SAMCHON_GRAPH_SCALA_GRAPH: producer("legacy", [ + "--fake-legacy-server", + ]), + }).candidates.every( + (candidate) => candidate.provider.name !== SCALA_GRAPH_PROVIDER, + ), + ); + TestValidator.predicate( + "a resident producer that rejects the BSP project declines", + selectGraphProviders(root, ["scala"], {}, { + ...environment, + SAMCHON_GRAPH_SCALA_GRAPH: producer("unsupported", [ + "--fake-unsupported", + ]), + }).candidates.every( + (candidate) => candidate.provider.name !== SCALA_GRAPH_PROVIDER, + ), + ); + + const selected = selectGraphProviders(root, ["scala"], {}, environment); + TestValidator.predicate( + "the BSP-capable producer owns Scala with compiler authority", + selected.candidates.some( + (candidate) => + candidate.provider.name === SCALA_GRAPH_PROVIDER && + candidate.provider.authority === "compiler" && + candidate.languages.join() === "scala", + ), + ); + TestValidator.predicate( + "whole-target options are refused explicitly", + selectGraphProviders( + root, + ["scala"], + { server: "metals", maxFiles: 3, lspReferenceLimit: 4 }, + environment, + ).warnings.some( + (warning) => + warning.includes(SCALA_GRAPH_PROVIDER) && + warning.includes("server, maxFiles, lspReferenceLimit"), + ), + ); + const configuration = scalaGraphProvider.configuration?.(root, environment); + const buildInputs = scalaGraphProvider.buildInputs?.(root) ?? []; + TestValidator.predicate( + "the target universe observes Java and the producer", + configuration?.length === 2 && + configuration[0]!.startsWith("java=") && + configuration[1]!.startsWith("samchon-scala-graph=") && + buildInputs.includes("build.sbt") && + buildInputs.includes(".bsp/fixture.json") && + buildInputs.includes(".bsp/other.json") && + scalaGraphProvider + .configurationDerivation?.(root, environment) + .inconclusive.length === 0, + ); + + const command = scalaGraphProvider.resolve(root, environment); + if (command === undefined) { + throw new Error("scalac-graph: the fixture producer did not resolve"); + } + const previous = new Map(); + for (const [key, value] of Object.entries({ + SAMCHON_GRAPH_SCALA_GRAPH: current, + SAMCHON_GRAPH_JAVA_TOOLCHAIN: java, + })) { + previous.set(key, process.env[key]); + process.env[key] = value; + } + try { + const session = scalaGraphProvider.open({ + root, + command, + languages: ["scala"], + options: { cwd: root }, + }); + try { + const cold = await session.refresh(); + TestValidator.predicate( + "one generation publishes separate Scala 2 and Scala 3 targets", + cold.mode === "initial" && + session.generation === 1 && + session.current === cold.snapshot && + cold.snapshot.provenance.provider === SCALA_GRAPH_PROVIDER && + cold.snapshot.provenance.authority === "compiler" && + cold.snapshot.provenance.tool === "samchon-scala-graph" && + cold.snapshot.provenance.compilerVersion === "2.13.18; 3.9.0" && + cold.snapshot.protocol?.targets.length === 2 && + cold.snapshot.coverage?.length === 30, + ); + const apis = cold.snapshot.nodes.filter( + (node) => node.name === "Api" && !node.external, + ); + TestValidator.predicate( + "cross-built twins keep target-scoped stable identities", + apis.length === 2 && + apis[0]!.id !== apis[1]!.id && + apis.every((node) => node.id.startsWith("@v2/scala/")), + ); + TestValidator.predicate( + "typed calls and SemanticDB diagnostics survive normalization", + cold.snapshot.edges.some((edge) => edge.kind === "calls") && + cold.snapshot.diagnostics.length === 2 && + cold.snapshot.diagnostics.every( + (diagnostic) => diagnostic.code === "scalac", + ) && + cold.snapshot.coverage?.filter( + (row) => + ["renders", "tests"].includes(row.family) && + row.state === "unsupported", + ).length === 4, + ); + + const unchanged = await session.refresh(); + TestValidator.predicate( + "an unchanged BSP universe reuses the exact snapshot", + unchanged.changed === false && unchanged.snapshot === cold.snapshot, + ); + fs.appendFileSync( + path.join(root, "src/scala-3/demo/Api.scala"), + "// body edit\n", + ); + const edited = await session.refresh(); + TestValidator.predicate( + "a source edit commits an incremental target generation", + edited.changed === true && + edited.mode === "incremental" && + edited.snapshot !== cold.snapshot, + ); + } finally { + await session.close(); + } + } finally { + for (const [key, value] of previous) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + + const artifactFile = path.join(root, "scala-artifact.json"); + const produced = spawnSync( + process.execPath, + [GraphPaths.fakeScalaGraph, "snapshot", "--output", artifactFile], + { cwd: root, encoding: "utf8" }, + ); + TestValidator.equals("the producer writes an exact fixture", produced.status, 0); + const valid = JSON.parse( + fs.readFileSync(artifactFile, "utf8"), + ) as IScalaGraphSnapshot; + const accepting = new ScalaGraphSnapshotAdapter(root); + const published = accepting.apply(structuredClone(valid)); + const reordered = structuredClone(valid); + reordered.targets.reverse(); + const ordering = new ScalaGraphSnapshotAdapter(root); + const forward = ordering.apply(structuredClone(valid)); + const reverse = ordering.apply(reordered); + TestValidator.predicate( + "BSP response order does not move the committed generation", + reverse.protocol?.generation === forward.protocol?.generation && + reverse.provenance.universe === forward.provenance.universe && + JSON.stringify(reverse.coverage) === JSON.stringify(forward.coverage), + ); + const rejects = ( + label: string, + mutate: (value: IScalaGraphSnapshot) => void, + ): void => { + const candidate = structuredClone(valid); + mutate(candidate); + TestValidator.error(label, () => accepting.apply(candidate)); + TestValidator.predicate( + `${label} keeps the prior generation`, + accepting.current === published, + ); + }; + + for (const capability of ["bsp", "semanticdb", "typedPlugins", "zinc"] as const) { + rejects(`a producer without ${capability}`, (value) => { + value.producer.capabilities[capability] = false; + }); + } + rejects("a target whose identity is not its BSP URI", (value) => { + value.targets[0]!.name = "file:///different"; + }); + rejects("a target with no absolute BSP URI", (value) => { + value.targets[0]!.bspUri = "not a URI"; + value.targets[0]!.name = "not a URI"; + }); + rejects("a target on an unsupported Scala line", (value) => { + value.targets[0]!.scalaVersion = "2.11.12"; + }); + rejects("a Scala 2 target with the wrong binary line", (value) => { + value.targets[0]!.scalaBinaryVersion = "2.12"; + }); + rejects("a target with a non-string binary line", (value) => { + value.targets[0]!.scalaBinaryVersion = 213 as unknown as string; + }); + rejects("a Scala 3 target with the wrong binary line", (value) => { + value.targets[1]!.scalaBinaryVersion = "3.7"; + }); + rejects("a target without a platform", (value) => { + value.targets[0]!.platform = ""; + }); + rejects("a target without a valid source encoding", (value) => { + value.targets[0]!.sourceEncoding = ""; + }); + for (const coordinate of [ + "scalacOptionsDigest", + "classpathDigest", + "sourceRootsDigest", + "semanticdbOptionsDigest", + "compilerPluginsDigest", + "zincAnalysisDigest", + "generatedSourcesDigest", + ] as const) { + rejects(`a target with a malformed ${coordinate}`, (value) => { + value.targets[0]![coordinate] = "not-a-digest"; + }); + } + rejects("a shard from another compiler version", (value) => { + value.targets[0]!.shards[0]!.compilerVersion = "2.13.15"; + }); + rejects("a Scala 2 shard from the Scala 3 plugin", (value) => { + value.targets[0]!.shards[0]!.compilerPlugin = "scala3"; + }); + rejects("a shard without a plugin version", (value) => { + value.targets[0]!.shards[0]!.compilerPluginVersion = ""; + }); + rejects("a shard from another SemanticDB schema", (value) => { + value.targets[0]!.shards[0]!.semanticdbSchema = 5; + }); + rejects("a SemanticDB document for another URI", (value) => { + value.targets[0]!.shards[0]!.semanticdbUri = "Other.scala"; + }); + rejects("a SemanticDB document from another build target", (value) => { + value.targets[0]!.shards[0]!.semanticdbBuildTarget = "file:///other"; + }); + rejects("a malformed SemanticDB md5", (value) => { + value.targets[0]!.shards[0]!.semanticdbMd5 = "not-md5"; + }); + rejects("a stale SemanticDB md5", (value) => { + value.targets[0]!.shards[0]!.semanticdbMd5 = createHash("md5") + .update("different") + .digest("hex"); + }); + rejects("a SemanticDB source that cannot be read", (value) => { + const shard = value.targets[0]!.shards[0]!; + shard.source = "src/scala-2/demo/Missing.scala"; + shard.semanticdbUri = shard.source; + shard.nodes.forEach((node) => { + node.file = shard.source; + node.evidence.file = shard.source; + }); + shard.edges.forEach((edge) => { + edge.evidence.file = shard.source; + if (edge.from === "src/scala-2/demo/Api.scala") edge.from = shard.source; + }); + shard.unresolved.forEach((site) => { + site.evidence.file = shard.source; + }); + shard.diagnostics.forEach((diagnostic) => { + diagnostic.evidence.file = shard.source; + }); + }); + }; diff --git a/tests/test-graph/src/features/test_scip_session_publishes_only_a_whole_validated_index.ts b/tests/test-graph/src/features/test_scip_session_publishes_only_a_whole_validated_index.ts index 12aaa8a7..99adf750 100644 --- a/tests/test-graph/src/features/test_scip_session_publishes_only_a_whole_validated_index.ts +++ b/tests/test-graph/src/features/test_scip_session_publishes_only_a_whole_validated_index.ts @@ -3,6 +3,7 @@ import { ScipSession, scipProvider } from "@samchon/graph"; import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { pathToFileURL } from "node:url"; import { GraphPaths } from "../internal/GraphPaths"; @@ -373,6 +374,17 @@ async function assertGenerations(): Promise { ); await bare.close(); + const singleSlash = sessionOf(root, { + indexRoot: singleSlashFileUri(root).replace(/^file:/u, "FiLe:"), + plainRoot: true, + }); + TestValidator.equals( + "a SCIP session accepts a mixed-case single-slash file URI for its exact project root", + (await singleSlash.refresh()).snapshot.nodes.map((node) => node.name), + ["first"], + ); + await singleSlash.close(); + // A failure that is not an Error still has to arrive as one: a caller cannot // read `.message` off a string. const rethrown = new ScipSession({ @@ -468,6 +480,20 @@ async function assertFailuresRetainTheGeneration(): Promise { !loudFailureMessage.includes("OPENING LINE") && loudFailureMessage.includes("…"), ); + const stderrFailure = sessionOf(root, { mode: "stderr-fail" }); + let stderrFailureMessage = ""; + try { + await stderrFailure.refresh(); + } catch (error) { + stderrFailureMessage = (error as Error).message; + } + TestValidator.predicate( + "a stderr-only failure is likewise bounded to its actionable tail", + stderrFailureMessage.includes("FAILURE: compiler rejected the project") && + !stderrFailureMessage.includes("OPENING ERROR") && + stderrFailureMessage.includes("…") && + stderrFailureMessage.length < 2_200, + ); // The ordinary shape: one line, nothing to cut. An ellipsis here would claim // the tool said more than it did, which is the same kind of untruth as // dropping what it said. @@ -483,6 +509,23 @@ async function assertFailuresRetainTheGeneration(): Promise { shortFailureMessage.endsWith("cannot open project") && !shortFailureMessage.includes("…"), ); + const splitFailure = sessionOf(root, { mode: "both-streams-fail" }); + let splitFailureMessage = ""; + try { + await splitFailure.refresh(); + } catch (error) { + splitFailureMessage = (error as Error).message; + } + TestValidator.predicate( + "a benign stderr notice cannot hide the stdout build failure", + splitFailureMessage.includes("stderr tail: Picked up JAVA_TOOL_OPTIONS") && + splitFailureMessage.includes("stdout tail: …") && + splitFailureMessage.includes( + "FAILURE: Maven could not compile the project", + ) && + !splitFailureMessage.includes("OPENING LINE") && + splitFailureMessage.length < 2_200, + ); TestValidator.predicate( "a silent non-zero exit has no invented stderr suffix", silentFailureMessage.endsWith("exited with code 3"), @@ -985,6 +1028,10 @@ function settle(): Promise { return new Promise((resolve) => setTimeout(resolve, 50)); } +function singleSlashFileUri(file: string): string { + return pathToFileURL(file).href.replace(/^file:\/\/\//u, "file:/"); +} + /** A deterministic signal for the two synchronous cancellation handoffs. */ function abortsOnRead(abortedAt: number): AbortSignal { let reads = 0; diff --git a/tests/test-graph/src/features/test_sidecar_session_enforces_the_common_semantic_contract.ts b/tests/test-graph/src/features/test_sidecar_session_enforces_the_common_semantic_contract.ts index 6fbc160d..01afa221 100644 --- a/tests/test-graph/src/features/test_sidecar_session_enforces_the_common_semantic_contract.ts +++ b/tests/test-graph/src/features/test_sidecar_session_enforces_the_common_semantic_contract.ts @@ -2,6 +2,7 @@ import { TestValidator } from "@nestia/e2e"; import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { pathToFileURL } from "node:url"; import { ISidecarSnapshot, @@ -68,6 +69,18 @@ export const test_sidecar_session_enforces_the_common_semantic_contract = await session.close(); await session.close(); + write(payload, { + ...snapshotOf(root, source), + projectRoot: singleSlashFileUri(root), + }); + const singleSlash = sessionOf(root, payload); + TestValidator.equals( + "a sidecar accepts the single-slash file URI for its exact project root", + (await singleSlash.refresh()).snapshot.nodes.map((node) => node.name), + ["main"], + ); + await singleSlash.close(); + await rejected(root, payload, "an oversized artifact is refused", { maxArtifactBytes: 1, }); @@ -375,6 +388,10 @@ function sessionOf( let configuration = "GOOS=linux"; +function singleSlashFileUri(file: string): string { + return pathToFileURL(file).href.replace(/^file:\/\/\//u, "file:/"); +} + function snapshotOf(root: string, source: string): ISidecarSnapshot { return { schemaVersion: 1, diff --git a/tests/test-graph/src/features/test_standard_providers_execute_their_exact_contracts.ts b/tests/test-graph/src/features/test_standard_providers_execute_their_exact_contracts.ts index 8f163cdf..e2f98a39 100644 --- a/tests/test-graph/src/features/test_standard_providers_execute_their_exact_contracts.ts +++ b/tests/test-graph/src/features/test_standard_providers_execute_their_exact_contracts.ts @@ -7,13 +7,17 @@ import { type IGraphProvider, RUST_GRAPH_PRODUCER_COMMIT, CPP_CLANG_PRODUCER_COMMIT, + csharpGraphProvider, cppGraphProvider, javaGraphProvider, goGraphProvider, + kotlinGraphProvider, luaGraphProvider, rustGraphProvider, + scalaGraphProvider, standardScipProviders, standardSidecarProviders, + swiftGraphProvider, } from "@samchon/graph"; import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; @@ -931,12 +935,22 @@ function assertFixtureRegistryCoverage(): void { rustGraphProvider, cppGraphProvider, javaGraphProvider, - // The two SCIP entries a strict route owns as its fallback tier. They are - // exercised by the loop above, but the registry does not list them as + kotlinGraphProvider, + scalaGraphProvider, + csharpGraphProvider, + // Swift's explicit-output-unit corpus is exercised by the dedicated + // IndexStoreDB contract suite because its snapshot cannot be represented + // by the generic sidecar fixture protocol used below. + swiftGraphProvider, + // SCIP entries a strict route owns as its fallback tier are exercised by + // the loop above, but the registry does not list them as // owners, so the ledger must not either. ...standardScipProviders.filter( (provider) => - provider.name !== "scip-clang" && provider.name !== "scip-java", + provider.name !== "scip-clang" && + provider.name !== "scip-java" && + provider.name !== "scip-kotlinc" && + provider.name !== "scip-dotnet", ), ...standardSidecarProviders, ] @@ -1249,6 +1263,12 @@ async function assertRemainingRegisteredFixtures(root: string): Promise { }; await assertRegisteredFixture(cppGraphProvider, cppCommand, root, "calls"); + const csharpCommand: IGraphProvider.ICommand = { + command: process.execPath, + args: [GraphPaths.fakeCsharpGraphServer, "--conformance"], + }; + await assertRegisteredFixture(csharpGraphProvider, csharpCommand, root); + // Lua's producer is the language server itself, driven through its `--doc` // export with our exporter injected, so the fixture stands in for the server // rather than for a binary of ours. `prepare` writes the config that carries diff --git a/tests/test-graph/src/features/test_strict_lifecycle_performance_sampling_is_atomic.ts b/tests/test-graph/src/features/test_strict_lifecycle_performance_sampling_is_atomic.ts new file mode 100644 index 00000000..d234dcb1 --- /dev/null +++ b/tests/test-graph/src/features/test_strict_lifecycle_performance_sampling_is_atomic.ts @@ -0,0 +1,203 @@ +import { TestValidator } from "@nestia/e2e"; + +import { measureLifecyclePerformance } from "../../../experiment/src/lifecycle-performance.mjs"; + +/** Sampling must preserve resident identity on no-op and restore source state. */ +export const test_strict_lifecycle_performance_sampling_is_atomic = async () => { + const original = "fn run() { channel(1); }"; + let text = original; + let current = { generation: "initial" }; + let identity = "initial"; + let edited = false; + let evidence = 0; + const writes: string[] = []; + const load = async () => { + if (text === original) { + if (edited) { + current = { generation: "restored" }; + identity = "initial"; + return { + dump: current, + identity, + mode: "incremental", + elapsedMs: 20, + }; + } + return { + dump: current, + identity, + mode: "unchanged", + elapsedMs: 5, + }; + } + edited = true; + current = { generation: text }; + identity = text; + return { + dump: current, + identity, + mode: "incremental", + elapsedMs: text.includes("channel(2)") ? 10 : 20, + }; + }; + const measured = await measureLifecyclePerformance({ + language: "fixture", + sourceText: original, + editFind: "channel(1)", + editReplacements: ["channel(2)", "channel(3)"], + noopSamples: 2, + editSamples: 4, + noopP95MaxMs: 250, + editP95MaxMs: 2_000, + changedModes: ["incremental"], + currentDump: current, + currentIdentity: identity, + writeSource: (next: string) => { + text = next; + writes.push(next); + }, + captureEditEvidence: () => ({ sample: ++evidence }), + load, + }); + TestValidator.equals( + "performance sampling alternates body edits and restores the original", + writes, + [ + "fn run() { channel(2); }", + "fn run() { channel(3); }", + "fn run() { channel(2); }", + "fn run() { channel(3); }", + original, + ], + ); + TestValidator.equals( + "performance sampling reports every observation and nearest-rank p95", + measured.row, + { + name: "performance", + status: "passed", + noopSamples: [5, 5], + editSamples: [10, 20, 10, 20], + noopP95Ms: 5, + editP95Ms: 20, + noopP95MaxMs: 250, + editP95MaxMs: 2_000, + editEvidence: [ + { sample: 1 }, + { sample: 2 }, + { sample: 3 }, + { sample: 4 }, + ], + }, + ); + TestValidator.equals( + "performance sampling returns the restored resident generation", + [measured.dump, measured.identity, text], + [current, "initial", original], + ); + + let staleText = original; + let staleEdited = false; + const staleInitial = { generation: "stale-initial" }; + await TestValidator.error( + "restoring source text refuses a stale edited provenance identity", + () => + measureLifecyclePerformance({ + language: "fixture", + sourceText: original, + editFind: "channel(1)", + editReplacements: ["channel(2)", "channel(3)"], + noopSamples: 1, + editSamples: 1, + noopP95MaxMs: 250, + editP95MaxMs: 2_000, + changedModes: ["incremental"], + currentDump: staleInitial, + currentIdentity: "stale-initial", + writeSource: (next: string) => { + staleText = next; + }, + load: async () => { + if (!staleEdited) { + staleEdited = staleText !== original; + if (!staleEdited) { + return { + dump: staleInitial, + identity: "stale-initial", + mode: "unchanged", + elapsedMs: 5, + }; + } + } + return { + dump: { generation: staleText }, + identity: "edited-stale", + mode: "incremental", + elapsedMs: 10, + }; + }, + }), + ); + + text = original; + current = { generation: "threshold" }; + identity = "initial"; + edited = false; + let thresholdError: unknown; + try { + await measureLifecyclePerformance({ + language: "fixture", + sourceText: original, + editFind: "channel(1)", + editReplacements: ["channel(2)", "channel(3)"], + noopSamples: 1, + editSamples: 1, + noopP95MaxMs: 5, + editP95MaxMs: 2_000, + changedModes: ["incremental"], + currentDump: current, + currentIdentity: identity, + writeSource: (next: string) => { + text = next; + }, + load, + }); + } catch (error) { + thresholdError = error; + } + TestValidator.predicate( + "a sample at the strict less-than ceiling rejects the row", + thresholdError instanceof Error && + thresholdError.message.includes("lifecycle performance missed its target"), + ); + let invalidConfigurationError: unknown; + try { + await measureLifecyclePerformance({ + language: "fixture", + sourceText: original, + editFind: "absent()", + editReplacements: ["channel(2)", "channel(3)"], + noopSamples: 1, + editSamples: 1, + noopP95MaxMs: 250, + editP95MaxMs: 2_000, + changedModes: ["incremental"], + currentDump: { generation: "invalid-config" }, + currentIdentity: "invalid-config", + writeSource: () => { + throw new Error("invalid configuration reached writeSource"); + }, + load: async () => { + throw new Error("invalid configuration reached load"); + }, + }); + } catch (error) { + invalidConfigurationError = error; + } + TestValidator.predicate( + "sampling rejects a replacement that cannot edit the source", + invalidConfigurationError instanceof Error && + invalidConfigurationError.message === + "fixture: lifecycle performance requires two real body-edit replacements", + ); +}; diff --git a/tests/test-graph/src/features/test_swift_indexstore_freezes_explicit_output_units.ts b/tests/test-graph/src/features/test_swift_indexstore_freezes_explicit_output_units.ts new file mode 100644 index 00000000..097b39c7 --- /dev/null +++ b/tests/test-graph/src/features/test_swift_indexstore_freezes_explicit_output_units.ts @@ -0,0 +1,477 @@ +import { TestValidator } from "@nestia/e2e"; +import { + ISwiftGraphSnapshot, + SWIFT_GRAPH_PROVIDER, + SwiftGraphSnapshotAdapter, + resolveSwiftGraphCommand, + selectGraphProviders, + swiftGraphProvider, +} from "@samchon/graph"; +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { GraphPaths } from "../internal/GraphPaths"; + +const SOURCE = `import Foundation +@MainActor +public protocol Service { + associatedtype Output + func fetch(_ value: T) async -> Output +} +open class Base { + public init() {} + open var value: Int { get { 0 } set {} } + open func run() {} +} +public final class Api: Base, Service { + public typealias Output = String + public override var value: Int { + get { 1 } + set {} + } + public override final func run() { + var local = value + local += 1 + print(local) + } + public func fetch(_ value: T) async -> String { String(describing: value) } +} +extension Api { + public convenience init(seed: Int) { + self.init() + } +} +#if FEATURE_FLAG +@attached(member, names: named(generated)) +public macro FixtureMacro() = #externalMacro(module: "FixtureMacros", type: "FixtureMacro") +@FixtureMacro +public struct MacroHost {} +#endif +public func testApi() async { + let subject = Api(seed: 1) + subject.value = 2 + _ = subject.value + subject.run() + _ = await subject.fetch("x") +} +// fixture warning +`; + +/** + * The Swift route admits an exact output-unit set and keeps USRs separate for + * build triples while one source enrichment pass supplies syntax-only facts. + */ +export const test_swift_indexstore_freezes_explicit_output_units = + async (): Promise => { + const packaged = path.join(GraphPaths.graphPackageRoot, "sidecars", "swift"); + TestValidator.predicate( + "the package carries the complete buildable Swift producer source", + [ + "README.md", + "Package.resolved", + "Package.swift", + "Sources/SamchonSwiftGraph/main.swift", + "Sources/SamchonSwiftGraph/SwiftGraphProducer.swift", + ].every((file) => fs.existsSync(path.join(packaged, file))) && + !fs.existsSync(path.join(packaged, ".build")), + ); + const producerSource = fs.readFileSync( + path.join( + packaged, + "Sources", + "SamchonSwiftGraph", + "SwiftGraphProducer.swift", + ), + "utf8", + ); + TestValidator.predicate( + "the shipped producer selects current SwiftPM objects without a store poll", + producerSource.includes('"-index-include-locals"') && + producerSource.includes('document["swiftCommands"]') && + producerSource.includes('command["objects"]') && + producerSource.includes('command["sources"]') && + producerSource.includes('"--build-tests"') && + producerSource.includes("useExplicitOutputUnits: true") && + producerSource.includes("addUnitOutFilePaths") && + !producerSource.includes("pollForUnitChangesAndWait"), + ); + + const root = GraphPaths.createTempDirectory("samchon-graph-swift-"); + fs.mkdirSync(path.join(root, "Sources", "Demo"), { recursive: true }); + fs.writeFileSync(path.join(root, "Sources", "Demo", "Api.swift"), SOURCE); + fs.writeFileSync( + path.join(root, "Package.swift"), + '// swift-tools-version: 6.0\nimport PackageDescription\nlet package = Package(name: "Demo")\n', + ); + fs.writeFileSync(path.join(root, "Package.resolved"), '{"pins":[]}\n'); + + const windows = process.platform === "win32"; + const script = (name: string, body: string): string => { + const file = path.join(root, windows ? `${name}.cmd` : name); + fs.writeFileSync( + file, + windows ? `@echo off\r\n${body}\r\n` : `#!/bin/sh\n${body}\n`, + ); + if (!windows) fs.chmodSync(file, 0o755); + return file; + }; + const producer = (name: string, flags: readonly string[] = []): string => + script( + name, + `"${process.execPath}" "${GraphPaths.fakeSwiftGraph}" ${flags.join(" ")} ${windows ? "%*" : '\"$@\"'}`, + ); + const swift = script("swift", "echo Swift version 6.0"); + const current = producer("samchon-swift-graph"); + const environment = { + ...process.env, + SAMCHON_GRAPH_SWIFT_GRAPH: current, + SAMCHON_GRAPH_SWIFT_TOOLCHAIN: swift, + }; + + const missing = GraphPaths.createTempDirectory("samchon-graph-swift-no-package-"); + fs.writeFileSync(path.join(missing, "Api.swift"), "struct Api {}\n"); + TestValidator.predicate( + "a repository without Package.swift declines the strict route", + resolveSwiftGraphCommand(missing, environment, "linux") === undefined, + ); + const selected = selectGraphProviders(root, ["swift"], {}, environment); + TestValidator.predicate( + "unsupported Windows declines while macOS and Linux accept the sidecar", + selected.candidates.some( + (candidate) => candidate.provider.name === SWIFT_GRAPH_PROVIDER, + ) === !windows, + ); + TestValidator.predicate( + "a producer without the explicit resident capability declines", + resolveSwiftGraphCommand( + root, + { + ...environment, + SAMCHON_GRAPH_SWIFT_GRAPH: producer("legacy", ["--fake-legacy-server"]), + }, + "linux", + ) === undefined, + ); + TestValidator.predicate( + "a sidecar that rejects the package declines", + resolveSwiftGraphCommand( + root, + { + ...environment, + SAMCHON_GRAPH_SWIFT_GRAPH: producer("unsupported", ["--fake-unsupported"]), + }, + "linux", + ) === undefined, + ); + TestValidator.predicate( + "the resolver accepts its exact macOS and Linux contract only", + resolveSwiftGraphCommand(root, environment, "linux") !== undefined && + resolveSwiftGraphCommand(root, environment, "darwin") !== undefined && + resolveSwiftGraphCommand(root, environment, "win32") === undefined, + ); + TestValidator.predicate( + "whole-module options are refused explicitly", + selectGraphProviders( + root, + ["swift"], + { server: "sourcekit-lsp", maxFiles: 3, lspReferenceLimit: 4 }, + environment, + ).warnings.some( + (warning) => + warning.includes(SWIFT_GRAPH_PROVIDER) && + warning.includes("server, maxFiles, lspReferenceLimit"), + ), + ); + const configuration = swiftGraphProvider.configuration?.(root, environment); + const inputs = swiftGraphProvider.buildInputs?.(root) ?? []; + TestValidator.predicate( + "the universe observes Swift, the producer, the pin and package inputs", + configuration?.length === 3 && + configuration[0]!.startsWith("swift=") && + configuration[1]!.startsWith("samchon-swift-graph=") && + configuration[2] === + `indexstore-db=${ISwiftGraphSnapshot.INDEX_STORE_DB_COMMIT}` && + inputs.includes("Package.swift") && + inputs.includes("Package.resolved") && + swiftGraphProvider + .configurationDerivation?.(root, environment) + .inconclusive.length === 0, + ); + + const artifactFile = path.join(root, "swift-artifact.json"); + const produced = spawnSync( + process.execPath, + [GraphPaths.fakeSwiftGraph, "snapshot", "--output", artifactFile], + { cwd: root, encoding: "utf8" }, + ); + TestValidator.equals("the producer writes an exact fixture", produced.status, 0); + const valid = JSON.parse( + fs.readFileSync(artifactFile, "utf8"), + ) as ISwiftGraphSnapshot; + TestValidator.predicate( + "the surrounding stale unit is excluded from every frozen generation", + fs.existsSync( + path.join( + root, + ".build/x86_64-unknown-linux-gnu/debug/Stale.build/Old.swift.o", + ), + ) && + valid.targets.every( + (target) => + target.outputUnits.length === 1 && + !target.outputUnits[0]!.path.includes("Stale"), + ), + ); + const accepting = new SwiftGraphSnapshotAdapter(root); + const published = accepting.apply(structuredClone(valid)); + const apis = published.nodes.filter( + (node) => node.name === "Api" && !node.external, + ); + TestValidator.predicate( + "one USR compiled for two triples receives two stable identities", + apis.length === 2 && + apis[0]!.id !== apis[1]!.id && + apis.every((node) => node.id.startsWith("@v2/swift/")) && + published.protocol?.targets.length === 2, + ); + TestValidator.predicate( + "all fact families and syntax enrichment are explicit", + published.coverage?.length === 30 && + published.coverage.filter( + (row) => row.family === "renders" && row.state === "unsupported", + ).length === 2 && + published.edges.some((edge) => edge.kind === "imports") && + published.edges.some((edge) => edge.kind === "decorates") && + ["explicitOutputUnits", "indexStoreDB", "sourceEnrichment", "swiftpm"].every( + (capability) => published.provenance.capabilities.includes(capability), + ) && + published.diagnostics.every((diagnostic) => diagnostic.code === "swiftc"), + ); + const declarations = new Set( + published.nodes.filter((node) => !node.external).map((node) => node.name), + ); + const edgeKinds = new Set(published.edges.map((edge) => edge.kind)); + TestValidator.predicate( + "the exact fixture covers Swift declarations and every supported fact", + [ + "Service", + "Base", + "Api", + "Api extension", + "init(seed:)", + "value", + "get value", + "set value", + "fetch", + "local", + "FixtureMacro", + "MacroHost", + "testApi", + ].every((name) => declarations.has(name)) && + [ + "contains", + "exports", + "imports", + "calls", + "accesses", + "instantiates", + "type_ref", + "extends", + "implements", + "overrides", + "dispatches", + "decorates", + "tests", + "references", + ].every((kind) => edgeKinds.has(kind)) && + valid.targets.every( + (target) => + target.shards.some((shard) => + shard.edges.some( + (edge) => edge.kind === "accesses" && edge.access === "read", + ), + ) && + target.shards.some((shard) => + shard.edges.some( + (edge) => edge.kind === "accesses" && edge.access === "write", + ), + ), + ) && + published.nodes.some( + (node) => + node.name === "fetch" && + node.signature?.includes("") === true && + node.modifiers?.includes("async") === true, + ) && + published.unresolved.some( + (site) => + site.reason === "conditional-build" && + site.family === "references", + ) && + published.unresolved.some( + (site) => + site.reason === "macro-or-generated" && + site.family === "references", + ) && + published.unresolved.some( + (site) => site.reason === "dynamic" && site.family === "dispatches", + ), + ); + + const command = { command: process.execPath, args: [GraphPaths.fakeSwiftGraph] }; + const previous = new Map(); + for (const [key, value] of Object.entries({ + SAMCHON_GRAPH_SWIFT_GRAPH: current, + SAMCHON_GRAPH_SWIFT_TOOLCHAIN: swift, + })) { + previous.set(key, process.env[key]); + process.env[key] = value; + } + try { + const session = swiftGraphProvider.open({ + root, + command, + languages: ["swift"], + options: { cwd: root }, + }); + try { + const cold = await session.refresh(); + TestValidator.predicate( + "the Swift session exposes its committed resident state", + session.generation === 1 && session.current === cold.snapshot, + ); + const unchanged = await session.refresh(); + fs.appendFileSync(path.join(root, "Sources", "Demo", "Api.swift"), "// edit\n"); + const edited = await session.refresh(); + TestValidator.predicate( + "the resident process reuses no-op and commits edited output units", + cold.mode === "initial" && + unchanged.changed === false && + unchanged.snapshot === cold.snapshot && + edited.changed === true && + edited.mode === "incremental" && + edited.snapshot !== cold.snapshot, + ); + } finally { + await session.close(); + } + } finally { + for (const [key, value] of previous) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + + TestValidator.equals( + "the producer refreshes the rejection oracle after the lifecycle edit", + spawnSync( + process.execPath, + [GraphPaths.fakeSwiftGraph, "snapshot", "--output", artifactFile], + { cwd: root, encoding: "utf8" }, + ).status, + 0, + ); + const guardedArtifact = JSON.parse( + fs.readFileSync(artifactFile, "utf8"), + ) as ISwiftGraphSnapshot; + const guarded = new SwiftGraphSnapshotAdapter(root); + const guardedPublished = guarded.apply(structuredClone(guardedArtifact)); + const rejects = ( + label: string, + mutate: (value: ISwiftGraphSnapshot) => void, + ): void => { + const candidate = structuredClone(guardedArtifact); + mutate(candidate); + TestValidator.error(label, () => guarded.apply(candidate)); + TestValidator.predicate( + `${label} keeps the prior generation`, + guarded.current === guardedPublished, + ); + }; + for (const capability of [ + "explicitOutputUnits", + "indexStoreDB", + "sourceEnrichment", + "swiftpm", + ] as const) { + rejects(`a producer without ${capability}`, (value) => { + value.producer.capabilities[capability] = false; + }); + } + rejects("a standalone producer claiming SourceKit residency", (value) => { + value.producer.capabilities.sourceKitResident = true; + }); + rejects("a target with a different identity", (value) => { + value.targets[0]!.name = "Other@arm64-apple-macosx13.0/debug"; + }); + for (const field of ["moduleName", "targetTriple", "configuration", "swiftLanguageVersion"] as const) { + rejects(`a target without ${field}`, (value) => { + value.targets[0]![field] = ""; + }); + } + rejects("a target from another IndexStoreDB commit", (value) => { + value.targets[0]!.indexStoreDBCommit = "0".repeat(40); + }); + for (const field of [ + "compilerFlagsDigest", + "moduleDependenciesDigest", + "packageResolutionDigest", + "pluginsDigest", + "generatedSourcesDigest", + ] as const) { + rejects(`a malformed ${field}`, (value) => { + value.targets[0]![field] = "not-a-digest"; + }); + } + rejects("an empty explicit output-unit set", (value) => { + value.targets[0]!.outputUnits = []; + }); + rejects("an empty output-unit path", (value) => { + value.targets[0]!.outputUnits[0]!.path = ""; + }); + rejects("an absolute output-unit path", (value) => { + value.targets[0]!.outputUnits[0]!.path = path.resolve(root, "unit.o"); + }); + rejects("an escaping output-unit path", (value) => { + value.targets[0]!.outputUnits[0]!.path = "../unit.o"; + }); + rejects("an output-unit path naming the project root", (value) => { + value.targets[0]!.outputUnits[0]!.path = "."; + }); + rejects("an output-unit path naming the parent exactly", (value) => { + value.targets[0]!.outputUnits[0]!.path = ".."; + }); + rejects("a malformed output-unit digest", (value) => { + value.targets[0]!.outputUnits[0]!.digest = "not-a-digest"; + }); + rejects("a stale output-unit digest", (value) => { + value.targets[0]!.outputUnits[0]!.digest = "0".repeat(64); + }); + rejects("a missing output unit", (value) => { + value.targets[0]!.outputUnits[0]!.path = ".build/missing.o"; + }); + rejects("an unsorted output-unit set", (value) => { + const first = value.targets[0]!.outputUnits[0]!; + fs.writeFileSync(path.join(root, "z.o"), "second unit"); + value.targets[0]!.outputUnits = [ + { + path: "z.o", + digest: createHash("sha256") + .update(fs.readFileSync(path.join(root, "z.o"))) + .digest("hex"), + }, + first, + ]; + }); + rejects("a source enriched more than once", (value) => { + value.targets[0]!.shards[0]!.sourceEnrichmentPasses = 2; + }); + rejects("a source attributed to another module", (value) => { + value.targets[0]!.shards[0]!.moduleName = "Other"; + }); + rejects("a source attributed to another triple", (value) => { + value.targets[0]!.shards[0]!.targetTriple = "other-triple"; + }); + }; diff --git a/tests/test-graph/src/features/test_ttscgraph_native_requests_recover_from_stalls.ts b/tests/test-graph/src/features/test_ttscgraph_native_requests_recover_from_stalls.ts index e407dafa..06a22412 100644 --- a/tests/test-graph/src/features/test_ttscgraph_native_requests_recover_from_stalls.ts +++ b/tests/test-graph/src/features/test_ttscgraph_native_requests_recover_from_stalls.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { TtscGraphClient } from "../../../../packages/graph/src/provider/ttscgraph/TtscGraphClient"; import { GraphPaths } from "../internal/GraphPaths"; +import { waitForProcessId } from "../internal/waitForProcessId"; /** A stalled request owns one child generation, never the resident queue. */ export const test_ttscgraph_native_requests_recover_from_stalls = async () => { @@ -304,7 +305,7 @@ const assertRetiredChildIsClosed = async (): Promise => { const childSource = [ 'const fs = require("node:fs");', 'const readline = require("node:readline");', - `fs.writeFileSync(${JSON.stringify(started)}, String(process.pid));`, + `fs.writeFileSync(${JSON.stringify(started)}, process.pid + "\\n");`, "readline.createInterface({ input: process.stdin }).once(\"line\", () =>", ` fs.writeFileSync(${JSON.stringify(requested)}, "request\\n"));`, "process.on(\"SIGTERM\", () =>", @@ -323,7 +324,7 @@ const assertRetiredChildIsClosed = async (): Promise => { "-e", [ 'const fs = require("node:fs");', - `fs.writeFileSync(${JSON.stringify(unrelatedStarted)}, String(process.pid));`, + `fs.writeFileSync(${JSON.stringify(unrelatedStarted)}, process.pid + "\\n");`, "setInterval(() => undefined, 1_000);", ].join("\n"), ], @@ -334,7 +335,7 @@ const assertRetiredChildIsClosed = async (): Promise => { const controller = new AbortController(); const stalled = client.refresh({ signal: controller.signal }); await waitForFile(requested); - const pid = Number(fs.readFileSync(started, "utf8")); + const pid = await waitForProcessId(started); controller.abort("retire stubborn generation"); await rejectionOf(stalled); await waitForFile(terminated); diff --git a/tests/test-graph/src/features/test_workflows_use_current_core_action_runtimes.ts b/tests/test-graph/src/features/test_workflows_use_current_core_action_runtimes.ts index 39c1b8f3..3d531d90 100644 --- a/tests/test-graph/src/features/test_workflows_use_current_core_action_runtimes.ts +++ b/tests/test-graph/src/features/test_workflows_use_current_core_action_runtimes.ts @@ -32,6 +32,7 @@ const MAINTAINED: Record = { cache: 6, checkout: 7, "setup-go": 7, + "setup-java": 6, "setup-node": 7, "upload-artifact": 7, "download-artifact": 8, @@ -144,99 +145,110 @@ export const test_workflows_use_current_core_action_runtimes = () => { path.join(directory, "experiment.yml"), "utf8", ); - // One boundary declaration for the whole matrix, and the exception set - // written into it rather than left to a reader. - // - // This originally refused any per-language exception, and the reason it gave - // was a cause: the lane that wanted more than ninety minutes wanted it - // because its provider had been serialized, so raising the budget preserved - // that cause instead of bounding it. The serialization was real and was - // removed, and ninety still does not fit — the same build completed in 56 - // minutes on one runner and 107 on another in one workflow, with setup and a - // real-corpus lifecycle run around it. C and C++ build a compiler from - // source and the other fourteen rows install a released producer, so the - // difference is a property of those two rows and not a defect inside them. - // - // Asserted as the exact expression. That is the same shape of assertion as - // the single number it replaces, not a tighter one — what changed is the - // policy, not the grip. The grip is what matters here: the ninety-minute - // bound still governs every other row, and a third language cannot reach the - // wider one, nor a fourth number appear, without editing this line and - // answering for it. - // - // Scoped to the matrix job, not to the file. Counting `timeout-minutes:` - // lines across the whole workflow passes just as well when the only one has - // been moved up into the classification job — leaving all sixteen real-tool - // lanes on GitHub's six-hour default, which is the unbounded state this - // assertion exists to prevent. - const experimentJob = experiment.slice(experiment.indexOf("\n experiment:")); + // Compiler construction and corpus execution now have separate owners and + // separate clocks. The predecessor retains the measured 150-minute cold-build + // budget; every matrix row returns to the original 90-minute lifecycle bound. + const producerStart = experiment.indexOf("\n clang_producer:"); + const experimentStart = experiment.indexOf("\n experiment:"); + const producerJob = experiment.slice(producerStart, experimentStart); + const experimentJob = experiment.slice(experimentStart); + const producerTimeouts = producerJob + .split("\n") + .filter((line) => line.trim().startsWith("timeout-minutes:")) + .map((line) => line.trim()); const experimentTimeouts = experimentJob .split("\n") .filter((line) => line.trim().startsWith("timeout-minutes:")) .map((line) => line.trim()); TestValidator.equals( - "one bound governs the matrix and only the compiler-building rows widen it", + "the Clang owner alone receives the cold-build budget", + producerTimeouts, + ["timeout-minutes: 150"], + ); + TestValidator.equals( + "one lifecycle bound governs every experiment consumer", experimentTimeouts, - [ - "timeout-minutes: ${{ (matrix.language == 'c' || matrix.language == 'cpp') && 150 || 90 }}", - ], + ["timeout-minutes: 90"], ); - // The wider bound is only defensible while an ordinary push does not reach - // it, and that depends entirely on the restore below. Pinned per step rather - // than as loose substrings over the job: four independent `includes` calls - // are satisfied by four unrelated steps, which would let the key, the path - // and the condition drift apart while the assertion stayed green. - // - // The key is pinned by its exact file list because that list is the whole - // claim. `catalog.mjs` is where the commit is actually read from, so leaving - // it out would bind the cache to the producer only by convention; the key - // would then survive a bump, the restored binary would fail its version - // check, the build would run in full, and — having hit an exact key — never - // re-save. Permanent silent full-cost rebuilding, with the widened bound as - // the normal path. - const steps = experimentSteps(experimentJob); - const restore = steps.find((step) => + const producerSteps = experimentSteps(producerJob); + const consumerSteps = experimentSteps(experimentJob); + const producerRestore = producerSteps.find((step) => step.body.includes("uses: actions/cache/restore@v6"), ); - const save = steps.find((step) => + const producerSave = producerSteps.find((step) => step.body.includes("uses: actions/cache/save@v6"), ); + const producerUpload = producerSteps.find((step) => + step.body.includes("uses: actions/upload-artifact@v7"), + ); + const producerPack = producerSteps.find( + (step) => step.name === "Pack the verified Clang producer", + ); + const consumerDownload = consumerSteps.find((step) => + step.body.includes("uses: actions/download-artifact@v8"), + ); + const consumerUnpack = consumerSteps.find( + (step) => step.name === "Unpack the verified Clang producer", + ); + const install = consumerSteps.find( + (step) => step.name === "Install language server", + ); + const narrowKey = + "key: clang-producer-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts', 'tests/experiment/src/clang-producer.mjs') }}"; TestValidator.equals( - "the producer is restored and saved around the build, on the same key", + "one predecessor builds and saves the exact Clang generation before consumers", [ - restore?.body.includes("id: clang_producer"), - restore?.body.includes("path: tests/experiment/.work/tools"), - restore?.body.includes( - "key: clang-producer-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('packages/graph/src/provider/cpp/CPP_CLANG_PRODUCER_COMMIT.ts', 'tests/experiment/src/catalog.mjs', 'tests/experiment/src/setup-language.mjs') }}", - ), - save?.body.includes("path: tests/experiment/.work/tools"), - save?.body.includes( + producerRestore?.body.includes("id: clang_producer"), + producerRestore?.body.includes(narrowKey), + producerSave?.body.includes( "key: ${{ steps.clang_producer.outputs.cache-primary-key }}", ), - save?.body.includes("steps.clang_producer.outputs.cache-hit != 'true'"), - [restore, save].every((step) => - step?.body.includes("(matrix.language == 'c' || matrix.language == 'cpp')"), + producerSave?.body.includes("continue-on-error: true"), + producerUpload?.body.includes("name: pinned-clang-producer"), + producerUpload?.body.includes("path: pinned-clang-producer.tar"), + producerPack?.body.includes( + "tar -C tests/experiment/.work/tools -cf pinned-clang-producer.tar .", ), - // Order is the whole safety argument. Saving before the build writes an - // empty tree under the exact primary key, which then restores as a hit - // forever, fails `setup`'s version check, rebuilds, and never re-saves — - // the same terminal state as saving under a key the restore cannot hit. - // Saving after the corpus run instead loses a correct build to an - // unrelated assertion. - // - // Relative, not absolute. The argument is about what comes before what, - // so pinning positions would make an unrelated step inserted anywhere - // above fail an assertion that has nothing to say about it. isStrictlyOrdered( - [restore, "Install language server", save, "Run LSP experiment"].map( - (entry) => - typeof entry === "string" - ? steps.findIndex((step) => step.name === entry) - : (entry?.index ?? -1), + [ + "Restore the pinned Clang producer", + "Provision the pinned Clang producer", + "Save the pinned Clang producer", + "Pack the verified Clang producer", + "Upload the verified Clang producer", + ].map((name) => + producerSteps.findIndex((step) => step.name === name), ), ), + experimentJob.includes("needs: [latest_update, clang_producer]"), + consumerDownload?.body.includes("name: pinned-clang-producer"), + consumerDownload?.body.includes( + "path: tests/experiment/.work/clang-producer-artifact", + ), + consumerUnpack?.body.includes( + "tar -C tests/experiment/.work/tools -xf tests/experiment/.work/clang-producer-artifact/pinned-clang-producer.tar", + ), + install?.body.includes( + 'SAMCHON_GRAPH_CLANG_PRODUCER_ALLOW_BUILD: "0"', + ), + occurrences(experiment, "Save the pinned Clang producer") === 1, + ], + [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, ], - [true, true, true, true, true, true, true, true], ); TestValidator.predicate( "the Rust experiment launches the exact binary provisioned by setup", @@ -244,6 +256,22 @@ export const test_workflows_use_current_core_action_runtimes = () => { "SAMCHON_GRAPH_RUST_ANALYZER_HIR: ${{ github.workspace }}/tests/experiment/.work/tools/bin/samchon-rust-analyzer", ), ); + const testWorkflow = fs.readFileSync( + path.join(directory, "test.yml"), + "utf8", + ); + const experimentSetup = fs.readFileSync( + path.join(GraphPaths.repositoryRoot, "tests", "experiment", "src", "setup-language.mjs"), + "utf8", + ); + TestValidator.predicate( + "Roslyn builds enforce the lock file through the MSBuild property", + [testWorkflow, experimentSetup].every( + (source) => + source.includes("-p:RestoreLockedMode=true") && + !source.includes("--locked-mode"), + ), + ); const indexTime = fs.readFileSync( path.join(directory, "index-time.yml"), "utf8", @@ -322,12 +350,12 @@ function isStrictlyOrdered(positions: readonly number[]): boolean { } /** - * The matrix job's steps, in order, with their comments removed. + * One workflow job's steps, in order, with their comments removed. * * Two properties this file needs and cannot get from a substring search over - * the whole job. Order, because the cache save has to happen after the step - * that builds and before the step that can fail for unrelated reasons, and a - * job-wide `indexOf` cannot tell those apart from a save at the top. And + * the whole job. Order, because the producer cache save has to happen after + * the step that builds, and a job-wide `indexOf` cannot tell those apart from + * a save at the top. And * comment removal, because this workflow explains itself at length: every * string these assertions look for also appears in prose a few lines above the * step that implements it, so an `includes` over raw text is satisfied by the diff --git a/tests/test-graph/src/internal/GraphPaths.ts b/tests/test-graph/src/internal/GraphPaths.ts index e5ce156d..4004aa55 100644 --- a/tests/test-graph/src/internal/GraphPaths.ts +++ b/tests/test-graph/src/internal/GraphPaths.ts @@ -64,11 +64,16 @@ export const GraphPaths = { fakeCmake: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-cmake.cjs"), fakeLspServer: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-lsp-server.cjs"), fakeCppGraphServer: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-cpp-graph-server.cjs"), + fakeCsharpGraphServer: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-csharp-graph-server.cjs"), + fakeJdtGraphServer: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-jdt-graph-server.cjs"), fakeRustGraphServer: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-rust-graph-server.cjs"), fakeTtscGraphServer: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-ttscgraph-server.cjs"), fakePub: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-pub.cjs"), fakeScipIndexer: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-scip-indexer.cjs"), fakeScipJava: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-scip-java.cjs"), + fakeKotlinGraph: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-kotlin-graph.cjs"), + fakeScalaGraph: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-scala-graph.cjs"), + fakeSwiftGraph: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-swift-graph.cjs"), fakeScipDecoder: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-scip-decoder.cjs"), fakeStandardProvider: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-standard-provider.cjs"), fakeToolchain: path.join(repositoryRoot, "tests", "test-graph", "src", "internal", "fake-toolchain.cjs"), diff --git a/tests/test-graph/src/internal/fake-cpp-graph-server.cjs b/tests/test-graph/src/internal/fake-cpp-graph-server.cjs index 9dab7bae..274fb4d1 100644 --- a/tests/test-graph/src/internal/fake-cpp-graph-server.cjs +++ b/tests/test-graph/src/internal/fake-cpp-graph-server.cjs @@ -68,6 +68,13 @@ if (args.includes("--version")) { process.stdout.write(`clangd version 22.1.8 (${commit})\n`); process.exit(0); } +if ( + args.includes("--require-info-log") && + (!args.includes("--log=info") || args.includes("--log=verbose")) +) { + process.stderr.write("fixture requires bounded info logging\n"); + process.exit(2); +} if (args.includes("--snapshot")) { process.stdout.write(JSON.stringify(snapshot(null, undefined, 32))); process.exit(0); diff --git a/tests/test-graph/src/internal/fake-csharp-graph-server.cjs b/tests/test-graph/src/internal/fake-csharp-graph-server.cjs new file mode 100644 index 00000000..f08e1847 --- /dev/null +++ b/tests/test-graph/src/internal/fake-csharp-graph-server.cjs @@ -0,0 +1,372 @@ +#!/usr/bin/env node +"use strict"; + +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); + +const { GraphSnapshotProtocol } = require(path.resolve( + __dirname, + "../../../../packages/graph/lib/provider/GraphSnapshotProtocol.js", +)); + +const args = process.argv.slice(2); +const valueOf = (prefix) => + args.find((argument) => argument.startsWith(prefix))?.slice(prefix.length); +const marker = valueOf("--marker="); +const requestLog = valueOf("--request-log="); +const expectInitializationOptions = args.includes("--expect-initialization-options"); +const conformance = args.includes("--conformance"); +const change = args.includes("--change"); +const malformed = valueOf("--malformed="); +const internalError = args.includes("--internal-error"); +const hang = args.includes("--hang"); +const hangInitialize = args.includes("--hang-initialize"); +const initializeError = args.includes("--initialize-error"); +const transition = valueOf("--transition="); +let contentModified = Number(valueOf("--content-modified=") ?? 0); +let sequence = 1; +let buffer = Buffer.alloc(0); + +process.stdin.on("data", (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + for (;;) { + const headerEnd = buffer.indexOf("\r\n\r\n"); + if (headerEnd < 0) return; + const header = buffer.slice(0, headerEnd).toString("ascii"); + const length = Number(/Content-Length:\s*(\d+)/iu.exec(header)?.[1]); + const bodyStart = headerEnd + 4; + const bodyEnd = bodyStart + length; + if (!Number.isSafeInteger(length) || buffer.length < bodyEnd) return; + const message = JSON.parse(buffer.slice(bodyStart, bodyEnd).toString("utf8")); + buffer = buffer.slice(bodyEnd); + handle(message); + } +}); + +function handle(message) { + if (requestLog !== undefined) { + fs.appendFileSync(requestLog, `${JSON.stringify(message)}\n`); + } + if (message.method === "initialize") { + if (hangInitialize) return; + if (initializeError) { + sendError(message.id, -32603, "fixture initialize failure"); + return; + } + if ( + expectInitializationOptions && + JSON.stringify(message.params?.initializationOptions) !== '{"fixture":true}' + ) { + process.exitCode = 31; + } + send({ jsonrpc: "2.0", id: message.id, result: { capabilities: {} } }); + return; + } + if (message.method === "workspace/executeCommand") { + if (hang) return; + if (internalError) { + sendError(message.id, -32603, "fixture internal failure"); + return; + } + if (contentModified > 0) { + contentModified -= 1; + sendError(message.id, -32801, "fixture solution moved"); + return; + } + const known = message.params?.arguments?.[0]?.knownGeneration ?? null; + const result = snapshot(known); + corrupt(result); + send({ jsonrpc: "2.0", id: message.id, result }); + return; + } + if (message.method === "shutdown") { + send({ jsonrpc: "2.0", id: message.id, result: null }); + return; + } + if (message.method === "exit") finish(); +} + +function snapshot(known) { + const initial = transaction("initial", 1, null, "fixture"); + const initialGeneration = initial.generation; + const edited = change + ? transaction("incremental", 2, initial, "edited") + : undefined; + const transitioned = + transition === "reload" + ? transaction("reload", 2, null, "reloaded", "reloaded-universe") + : transition === "rebuild" + ? transaction("rebuild", 2, null, "rebuilt") + : transition === "reload-nonfull" + ? transaction( + "reload", + 2, + initial, + "invalid-reload", + "reloaded-universe", + ) + : transition === "stale-base" + ? staleBase(initial) + : undefined; + if (known === transitioned?.generation) { + return { + protocolVersion: 1, + mode: "unchanged", + sequence: 2, + generation: transitioned.generation, + universe: transitioned.universe, + frames: [], + }; + } + if (known === edited?.generation) { + return { + protocolVersion: 1, + mode: "unchanged", + sequence: 2, + generation: edited.generation, + universe: edited.universe, + frames: [], + }; + } + if (known === initialGeneration && transitioned !== undefined) { + sequence = 2; + return transitioned; + } + if (known === initialGeneration && !change) { + return { + protocolVersion: 1, + mode: "unchanged", + sequence: 1, + generation: initialGeneration, + universe: initial.universe, + frames: [], + }; + } + if (known === initialGeneration && change) { + sequence = 2; + return edited; + } + return initial; +} + +function transaction( + mode, + nextSequence, + prior, + name, + universeName = "fixture-universe", +) { + const graphFile = conformance ? "src/Main.cs" : "Program.cs"; + const sourceFile = path.join(process.cwd(), ...graphFile.split("/")); + const bytes = fs.readFileSync(sourceFile); + const target = `roslyn:${sha256("fixture-target")}`; + const universe = sha256(universeName); + const evidence = { + file: graphFile, + startLine: 1, + startCol: 1, + endLine: 1, + endCol: 6, + }; + const nodes = conformance + ? ["caller", "callee"] + .map((symbol) => ({ + id: `@v2/csharp/${sha256(`fixture-${symbol}`)}#${symbol}:function`, + kind: "function", + language: "csharp", + name: symbol, + file: graphFile, + external: false, + evidence: wordSpans(bytes.toString("utf8"), graphFile, symbol).at(-1), + })) + .sort((left, right) => left.id.localeCompare(right.id)) + : [{ + id: `@v2/csharp/${sha256(`fixture-${name}`)}#${name}:class`, + kind: "class", + language: "csharp", + name, + file: graphFile, + external: false, + evidence, + }]; + const caller = nodes.find((node) => node.name === "caller"); + const callee = nodes.find((node) => node.name === "callee"); + const edges = conformance + ? [{ + from: caller.id, + to: callee.id, + kind: "references", + evidence: wordSpans(bytes.toString("utf8"), graphFile, "callee").at(-2), + }] + : []; + const families = [ + "contains", "exports", "imports", "calls", "accesses", + "instantiates", "type_ref", "extends", "implements", "overrides", + "dispatches", "decorates", "renders", "tests", "references", + ]; + const coverage = families.map((family) => ({ + provider: "roslyn-workspace", + language: "csharp", + target, + family, + state: family === "renders" ? "unsupported" : "partial", + })); + const unresolved = families + .filter((family) => family !== "renders") + .map((family) => ({ + provider: "roslyn-workspace", + language: "csharp", + target, + universe, + family, + evidence, + reason: "provider-gap", + candidates: [], + })); + const source = { + file: sourceFile, + checkerDigest: sha256(bytes), + diskDigest: sha256(bytes), + }; + const shard = { + key: "csharp-fixture|Program.cs", + target, + languages: ["csharp"], + nodes, + edges, + diagnostics: [], + coverage, + unresolved, + sources: [source], + }; + const shardDigest = GraphSnapshotProtocol.shardDigest(shard); + const manifest = [{ key: shard.key, digest: shardDigest }]; + const generation = sha256(`${universe}:${name}`); + const hello = { + type: "hello", + protocolVersion: 1, + schemaVersion: 1, + producerSchemaVersion: 1, + provider: "roslyn-workspace", + producer: "samchon-roslyn", + producerVersion: "1.0.0-fixture", + compilerVersion: "5.9.0-fixture", + languages: ["csharp"], + authority: "compiler", + supportedFacts: families.filter((family) => family !== "renders"), + capabilities: [ + "coverage", "diagnostics", "diskDigests", "incremental", + "sourceDigests", "universe", "unresolved", "immutableSolution", + "sourceGeneratedDocuments", + ], + }; + const begin = { + type: "begin", + sequence: nextSequence, + generation, + universe, + manifest: GraphSnapshotProtocol.manifestDigest([source]), + targets: [target], + ...(prior === null + ? {} + : { baseSequence: prior.sequence, baseGeneration: prior.generation }), + }; + const provenance = { + provider: hello.provider, + authority: hello.authority, + facts: hello.supportedFacts, + schemaVersion: hello.producerSchemaVersion, + tool: hello.producer, + toolVersion: hello.producerVersion, + compilerVersion: hello.compilerVersion, + protocolVersion: hello.protocolVersion, + universe, + capabilities: hello.capabilities, + }; + const factDigest = GraphSnapshotProtocol.factDigest({ + languages: hello.languages, + nodes: shard.nodes, + edges: shard.edges, + diagnostics: shard.diagnostics, + coverage: shard.coverage, + unresolved: shard.unresolved, + provenance, + }); + return { + protocolVersion: 1, + mode, + sequence: nextSequence, + generation, + universe, + frames: [ + hello, + begin, + { type: "upsertShard", digest: shardDigest, shard }, + { type: "commit", sequence: nextSequence, generation, shards: manifest, factDigest }, + ], + }; +} + +function corrupt(result) { + if (malformed === undefined) return; + if (malformed === "envelope") result.protocolVersion = 2; + if (malformed === "mode") result.mode = "incremental"; + if (malformed === "initial-base") { + const begin = result.frames.find((frame) => frame.type === "begin"); + begin.baseSequence = 7; + begin.baseGeneration = sha256("unexpected-base"); + } + if (malformed === "sequence") result.sequence += 1; + if (malformed === "generation") result.generation = sha256("wrong-generation"); + if (malformed === "universe") result.universe = sha256("wrong-universe"); + if (malformed === "unchanged-frames") { + result.mode = "unchanged"; + } +} + +function staleBase(initial) { + const result = transaction("incremental", 2, initial, "stale-base"); + const begin = result.frames.find((frame) => frame.type === "begin"); + begin.baseSequence = 7; + begin.baseGeneration = sha256("stale-base"); + return result; +} + +function send(message) { + const body = Buffer.from(JSON.stringify(message), "utf8"); + process.stdout.write(`Content-Length: ${body.length}\r\n\r\n`); + process.stdout.write(body); +} + +function sendError(id, code, message) { + send({ jsonrpc: "2.0", id, error: { code, message } }); +} + +function sha256(value) { + return crypto.createHash("sha256").update(value).digest("hex"); +} + +function wordSpans(text, file, word) { + const output = []; + for (let offset = 0; ; ) { + const found = text.indexOf(word, offset); + if (found < 0) return output; + const prefix = text.slice(0, found); + const line = prefix.split("\n").length; + const column = found - prefix.lastIndexOf("\n"); + output.push({ + file, + startLine: line, + startCol: column, + endLine: line, + endCol: column + word.length, + }); + offset = found + word.length; + } +} + +function finish() { + if (marker !== undefined) fs.writeFileSync(marker, "closed"); + process.exit(process.exitCode ?? 0); +} diff --git a/tests/test-graph/src/internal/fake-jdt-graph-server.cjs b/tests/test-graph/src/internal/fake-jdt-graph-server.cjs new file mode 100644 index 00000000..05d8d048 --- /dev/null +++ b/tests/test-graph/src/internal/fake-jdt-graph-server.cjs @@ -0,0 +1,132 @@ +#!/usr/bin/env node +"use strict"; + +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const { pathToFileURL } = require("node:url"); + +const args = process.argv.slice(2); +const valueOf = (prefix) => + args.find((argument) => argument.startsWith(prefix))?.slice(prefix.length); +const requestLog = valueOf("--request-log="); +const marker = valueOf("--marker="); +const delayCommand = Number(valueOf("--delay-command=") ?? 0); +const delayInitialize = Number(valueOf("--delay-initialize=") ?? 0); +const failInitialize = args.includes("--fail-initialize"); +const reuseAfterChange = args.includes("--reuse-after-change"); +const source = path.join(process.cwd(), "src", "Example.java"); +let buffer = Buffer.alloc(0); +let lastGeneration; +let sequence = 0; + +process.stdin.on("data", (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + for (;;) { + const end = buffer.indexOf("\r\n\r\n"); + if (end < 0) return; + const header = buffer.subarray(0, end).toString("ascii"); + const length = Number(/Content-Length:\s*(\d+)/i.exec(header)?.[1]); + if (!Number.isSafeInteger(length) || buffer.length < end + 4 + length) return; + const body = buffer.subarray(end + 4, end + 4 + length).toString("utf8"); + buffer = buffer.subarray(end + 4 + length); + handle(JSON.parse(body)); + } +}); + +function handle(message) { + if (requestLog) fs.appendFileSync(requestLog, `${JSON.stringify(message)}\n`); + if (message.method === "initialize") { + if (failInitialize) { + return error(message.id, -32000, "fixture initialize failure"); + } + const result = { capabilities: { executeCommandProvider: { commands: ["java.graph.snapshot"] } } }; + return delayInitialize > 0 + ? setTimeout(() => respond(message.id, result), delayInitialize) + : respond(message.id, result); + } + if (message.method === "workspace/executeCommand") { + if (message.params?.command !== "java.graph.snapshot") { + return error(message.id, -32601, "unsupported command"); + } + const result = snapshot(); + return delayCommand > 0 + ? setTimeout(() => respond(message.id, result), delayCommand) + : respond(message.id, result); + } + if (message.method === "shutdown") return respond(message.id, null); + if (message.method === "exit") { + if (marker) fs.writeFileSync(marker, "closed"); + process.exit(0); + } +} + +function snapshot() { + const exists = fs.existsSync(source); + const text = exists ? fs.readFileSync(source) : Buffer.alloc(0); + const observed = sha256(Buffer.concat([Buffer.from("generation:"), text])); + const generation = reuseAfterChange && lastGeneration !== undefined ? lastGeneration : observed; + const mode = lastGeneration === undefined ? "initial" : lastGeneration === generation ? "unchanged" : "incremental"; + if (lastGeneration !== generation) sequence += 1; + lastGeneration = generation; + const uri = pathToFileURL(source).href; + const evidence = { uri, startLine: 1, startColumn: 1, endLine: 1, endColumn: 20 }; + const file = "java/fixture/file/example"; + const type = "java/fixture/type/example.Example"; + const method = `${type}/method/run()`; + return { + schemaVersion: 1, + protocolVersion: 1, + producer: { name: "eclipse-jdtls-graph-snapshot", version: "1.50.0.fixture", compilerVersion: "21" }, + capabilities: { + atomicGenerations: true, + resident: true, + sourceDigests: true, + diskDigests: true, + unsavedBuffers: true, + diagnostics: true, + facts: ["contains"], + }, + universe: sha256("fixture-universe"), + generation, + complete: true, + mode, + sequence, + projects: [{ name: "fixture", location: pathToFileURL(process.cwd()).href, output: "/fixture/bin", compilerVersion: "21", options: {}, classpath: [] }], + sources: exists ? [{ project: "fixture", uri, checkerDigest: sha256(Buffer.concat([Buffer.from("checker:"), text])), checkerEncoding: "jdt-utf16-code-units-v1", diskDigest: sha256(text) }] : [], + nodes: exists ? [ + node(file, file, "persistent", uri, "Example.java", uri, "file", "", "file", [], evidence), + node(type, "Lexample/Example;", "persistent", uri, "Example", "example.Example", "class", "", "type", ["public", "final"], evidence), + node(method, "Lexample/Example;.run()V", "structural", uri, "run", "example.Example.run", "method", "():void", "method", ["public"], evidence), + ] : [], + edges: exists ? [ + { from: file, to: type, kind: "contains", evidence }, + { from: type, to: method, kind: "contains", evidence }, + ] : [], + diagnostics: exists ? [{ uri, severity: "information", code: "fixture", message: "fixture note", evidence }] : [], + coverage: { contains: "complete" }, + unresolved: [], + }; +} + +function node(symbol, nativeKey, stability, uri, name, qualifiedName, kind, signature, declarationKind, modifiers, evidence) { + return { project: "fixture", symbol, nativeKey, stability, uri, name, qualifiedName, kind, signature, declarationKind, exported: modifiers.includes("public"), modifiers, evidence }; +} + +function respond(id, result) { + write({ jsonrpc: "2.0", id, result }); +} + +function error(id, code, message) { + write({ jsonrpc: "2.0", id, error: { code, message } }); +} + +function write(message) { + const body = Buffer.from(JSON.stringify(message), "utf8"); + process.stdout.write(`Content-Length: ${body.length}\r\n\r\n`); + process.stdout.write(body); +} + +function sha256(value) { + return crypto.createHash("sha256").update(value).digest("hex"); +} diff --git a/tests/test-graph/src/internal/fake-kotlin-graph.cjs b/tests/test-graph/src/internal/fake-kotlin-graph.cjs new file mode 100644 index 00000000..5b55473b --- /dev/null +++ b/tests/test-graph/src/internal/fake-kotlin-graph.cjs @@ -0,0 +1,652 @@ +// A stand-in for `scip-java` with the kotlinc graph plugin attached. +// +// It answers the three questions the strict Kotlin route asks a launcher: what +// version it is, whether its `index` command publishes `--kotlin-graph-output`, and +// what that option writes. The artifact it writes is modelled on the producer's +// own Gradle lifecycle contract, including per-compilation targets and +// project-relative source paths. +// +// Every fault flag below stands for a producer state a real build can reach and +// a fixture cannot produce by asking politely: a launcher that predates the +// option, a generation with a hole in its coverage matrix, an edge whose target +// this compilation never saw, a build that recompiled one file. +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); + +const args = process.argv.slice(2); +const flag = (name) => args.includes(`--fake-${name}`); +const valueOf = (name) => { + const prefix = `--fake-${name}=`; + const found = args.find((argument) => argument.startsWith(prefix)); + return found === undefined ? undefined : found.slice(prefix.length); +}; +const optionAfter = (name) => { + const index = args.indexOf(name); + return index === -1 ? undefined : args[index + 1]; +}; + +if (args.includes("--version")) { + process.stdout.write("scip-java 0.13.1-fake\n"); + process.exit(0); +} + +const serving = args.includes("kotlin-graph-server"); +if (serving) void serve(); +else indexOnce(); + +async function serve() { + if (args.includes("--help")) { + if (!flag("legacy-server")) { + process.stdout.write( + "Serve compiler-owned Kotlin graph generations over NDJSON.\n", + ); + } + return; + } + if (flag("server-close-stdin")) { + process.stdin.destroy(); + setInterval(() => undefined, 1_000); + return; + } + const readline = require("node:readline"); + const childProcess = require("node:child_process"); + const prefix = args.slice(0, args.indexOf("kotlin-graph-server")); + const lines = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); + for await (const line of lines) { + if (line.trim() === "") continue; + const request = JSON.parse(line); + const serverLog = valueOf("server-log"); + if (serverLog !== undefined) { + fs.appendFileSync(serverLog, `${process.pid}\n`); + } + const once = (name) => { + const marker = valueOf(name); + if (marker === undefined || fs.existsSync(marker)) return false; + fs.writeFileSync(marker, String(process.pid)); + return true; + }; + if (flag("server-stall") || once("server-stall-once")) continue; + if (once("server-crash-once")) process.exit(17); + if (flag("server-stderr")) { + process.stderr.write("resident fixture stderr\n"); + } + if (once("server-malformed-once")) { + process.stdout.write("not-json\n"); + continue; + } + if (once("server-string-once")) { + process.stdout.write('"wrong"\n'); + continue; + } + if (once("server-null-once")) { + process.stdout.write("null\n"); + continue; + } + if (once("server-non-object-once")) { + process.stdout.write("[]\n"); + continue; + } + if (once("server-bad-identity-once")) { + process.stdout.write( + `${JSON.stringify({ id: "wrong", protocolVersion: 1, ok: true })}\n`, + ); + continue; + } + if (once("server-bad-protocol-once")) { + process.stdout.write( + `${JSON.stringify({ id: request.id, protocolVersion: 2, ok: true })}\n`, + ); + continue; + } + if (once("server-bad-result-once")) { + process.stdout.write( + `${JSON.stringify({ id: request.id, protocolVersion: 1, ok: "yes" })}\n`, + ); + continue; + } + if (once("server-non-string-error-once")) { + process.stdout.write( + `${JSON.stringify({ id: request.id, protocolVersion: 1, ok: false, error: 1 })}\n`, + ); + continue; + } + if (once("server-empty-error-once")) { + process.stdout.write( + `${JSON.stringify({ id: request.id, protocolVersion: 1, ok: false, error: "" })}\n`, + ); + continue; + } + if (once("server-unexpected-id-once")) { + process.stdout.write( + `${JSON.stringify({ id: request.id + 1, protocolVersion: 1, ok: true })}\n`, + ); + continue; + } + if (once("server-oversized-once")) { + process.stdout.write(`${"x".repeat(4096)}\n`); + continue; + } + if (once("server-error-once")) { + process.stdout.write( + `${JSON.stringify({ + id: request.id, + protocolVersion: 1, + ok: false, + error: "deliberate resident failure", + })}\n`, + ); + continue; + } + const result = childProcess.spawnSync( + process.execPath, + [ + __filename, + ...prefix, + "index", + "--output", + path.join(path.dirname(request.output), "fake-index.scip"), + "--kotlin-graph-output", + request.output, + ], + { cwd: process.cwd(), encoding: "utf8" }, + ); + const response = `${JSON.stringify({ + id: request.id, + protocolVersion: 1, + ok: result.status === 0, + ...(result.status === 0 + ? {} + : { error: (result.stderr || `fake producer exited ${result.status}`).trim() }), + })}\n`; + if (flag("server-blank-prefix")) process.stdout.write("\n"); + if (flag("server-split-response")) { + process.stdout.write(response.slice(0, 5)); + await new Promise((resolve) => setTimeout(resolve, 10)); + process.stdout.write(response.slice(5)); + } else { + process.stdout.write(response); + } + } +} + +function indexOnce() { +const indexing = args.includes("index"); + +if (indexing && args.includes("--help")) { + process.stdout.write( + [ + "Usage: scip-java index [OPTIONS]", + "", + "Options:", + " --output PATH Where to generate the SCIP index.", + ...(flag("legacy-launcher") + ? [] + : [ + " --kotlin-graph-output PATH Write the committed kotlinc graph generations here.", + ]), + "", + ].join("\n"), + ); + process.exit(0); +} + +if (!indexing) { + process.stderr.write( + `fake scip-java: unexpected command ${args.join(" ")}\n`, + ); + process.exit(2); +} + +if (flag("build-failure")) { + process.stderr.write("[ERROR] COMPILATION ERROR :\n"); + process.exit(1); +} + +const scipOutput = optionAfter("--output"); +if (scipOutput !== undefined) { + fs.mkdirSync(path.dirname(scipOutput), { recursive: true }); + fs.writeFileSync(scipOutput, ""); +} + +const graphOutput = optionAfter("--kotlin-graph-output"); +if (graphOutput === undefined) { + process.stderr.write("fake scip-java: no --kotlin-graph-output was requested\n"); + process.exit(2); +} + +if (flag("empty-artifact")) { + fs.writeFileSync(graphOutput, ""); + process.exit(0); +} + +if (flag("not-json")) { + fs.writeFileSync(graphOutput, "> Task :compileKotlin\nBUILD SUCCESSFUL\n"); + process.exit(0); +} + +const project = process.cwd(); +const marker = valueOf("marker"); +// A real build recompiles what changed. The marker counts invocations so a +// second one can rewrite exactly one source shard, which is the condition the +// consumer's delta path exists for. +const invocation = (() => { + if (marker === undefined) return 1; + const previous = fs.existsSync(marker) + ? Number(fs.readFileSync(marker, "utf8")) + : 0; + const next = previous + 1; + fs.mkdirSync(path.dirname(marker), { recursive: true }); + fs.writeFileSync(marker, String(next)); + return next; +})(); + +const digest = (value) => + crypto.createHash("sha256").update(value).digest("hex"); + +const COVERAGE = { + contains: "complete", + exports: "partial", + imports: "complete", + calls: "complete", + accesses: "complete", + instantiates: "complete", + type_ref: "partial", + extends: "complete", + implements: "complete", + overrides: "complete", + dispatches: "partial", + decorates: "complete", + renders: "unsupported", + tests: "partial", + references: "partial", +}; + +const evidence = (file, startLine, startColumn, endLine, endColumn) => ({ + file, + startLine, + startColumn, + endLine, + endColumn, +}); + +const sourceDigest = (source) => { + const file = path.join(project, source); + return fs.existsSync(file) + ? digest(fs.readFileSync(file)) + : digest(`fake:${source}`); +}; + +/** One compilation unit, in the producer's own shard schema. */ +const shard = (target, source, body) => ({ + schemaVersion: 1, + language: "kotlin", + source, + checkerDigest: sourceDigest(source), + diskDigest: fs.existsSync(path.join(project, source)) + ? sourceDigest(source) + : "", + target, + compilerVersion: "2.3.20", + ...body, + nodes: body.nodes.map((node) => ({ origin: "Source", ...node })), + diagnostics: + body.diagnostics ?? + (source.endsWith("Caller.kt") + ? [ + { + severity: "warning", + message: "fixture warning", + evidence: evidence(source, 5, 1, 5, 8), + }, + ] + : []), +}); + +const exampleShard = (target) => + shard(target, "src/main/kotlin/com/Example.kt", { + nodes: [ + { + symbol: "semanticdb maven . . com/Example#", + kind: "class", + name: "Example", + qualifiedName: "com.Example", + file: "src/main/kotlin/com/Example.kt", + exported: true, + modifiers: ["public"], + signature: "public class Example", + evidence: evidence("src/main/kotlin/com/Example.kt", 2, 1, 2, 27), + }, + ], + edges: [ + { + from: "src/main/kotlin/com/Example.kt", + to: "semanticdb maven . . com/Example#", + kind: "contains", + access: null, + provenance: null, + targetKind: "class", + targetName: "Example", + targetQualifiedName: "com.Example", + evidence: evidence("src/main/kotlin/com/Example.kt", 2, 1, 2, 27), + }, + { + from: "src/main/kotlin/com/Example.kt", + to: "semanticdb maven . . com/Example#", + kind: "exports", + access: null, + provenance: null, + targetKind: "class", + targetName: "Example", + targetQualifiedName: "com.Example", + evidence: evidence("src/main/kotlin/com/Example.kt", 2, 1, 2, 27), + }, + ], + unresolved: [], + }); + +const callerShard = (target) => + shard(target, "src/main/kotlin/com/Caller.kt", { + nodes: [ + { + symbol: "semanticdb maven . . com/Caller#", + kind: "class", + name: "Caller", + qualifiedName: "com.Caller", + file: "src/main/kotlin/com/Caller.kt", + exported: true, + modifiers: ["public"], + signature: "public class Caller", + evidence: evidence("src/main/kotlin/com/Caller.kt", 2, 1, 6, 2), + }, + { + symbol: "semanticdb maven . . com/Caller#make().", + kind: "method", + // The producer displays an executable with its parameter list, which + // is what tells two overloads apart on sight. The graph's name is the + // simple declared name and its signature is where a list belongs. + name: "make", + qualifiedName: "com.Caller.make", + file: "src/main/kotlin/com/Caller.kt", + exported: true, + modifiers: ["public", "static"], + signature: "public static Example make()", + evidence: evidence("src/main/kotlin/com/Caller.kt", 3, 5, 5, 6), + }, + // A constructor the producer displays with its parameter list and + // formats no signature for. The list is the only statement of its shape + // there is, so cutting it out of the name has to put it somewhere. + { + symbol: "semanticdb maven . . com/Caller#``().", + kind: "constructor", + name: "", + qualifiedName: "com.Caller.", + file: "src/main/kotlin/com/Caller.kt", + exported: true, + modifiers: ["public"], + signature: "constructor()", + evidence: evidence("src/main/kotlin/com/Caller.kt", 2, 1, 2, 20), + }, + // A local: no owner to qualify it, nothing exported, no modifier kotlinc + // records in the shared vocabulary, and no signature. Every optional + // fact absent at once, which is the shape a declaration most often has. + { + symbol: "semanticdb maven . . com/Caller#make().(made)", + kind: "variable", + name: "made", + qualifiedName: "", + file: "src/main/kotlin/com/Caller.kt", + exported: false, + modifiers: [], + signature: "", + evidence: evidence("src/main/kotlin/com/Caller.kt", 4, 9, 4, 13), + }, + ], + edges: [ + { + from: "src/main/kotlin/com/Caller.kt", + to: "semanticdb maven . . com/Caller#", + kind: "contains", + access: null, + provenance: null, + targetKind: "class", + targetName: "Caller", + targetQualifiedName: "com.Caller", + evidence: evidence("src/main/kotlin/com/Caller.kt", 2, 1, 6, 2), + }, + { + from: "semanticdb maven . . com/Caller#", + to: "semanticdb maven . . com/Caller#make().", + kind: "contains", + access: null, + provenance: null, + targetKind: "method", + targetName: "make", + targetQualifiedName: "com.Caller.make", + evidence: evidence("src/main/kotlin/com/Caller.kt", 3, 5, 5, 6), + }, + // The cross-file relationship: the method instantiates a class declared + // in another compilation unit of the same target. + { + from: "semanticdb maven . . com/Caller#make().", + to: "semanticdb maven . . com/Example#", + kind: "instantiates", + access: null, + provenance: null, + targetKind: "class", + targetName: "Example", + targetQualifiedName: "com.Example", + evidence: evidence("src/main/kotlin/com/Caller.kt", 4, 16, 4, 31), + }, + // The same relationship written twice, at two call sites. The producer + // keys its edges by evidence as well as endpoints; the graph's triple is + // unique, so the consumer has to fold these into one. + { + from: "semanticdb maven . . com/Caller#make().", + to: "semanticdb maven . . com/Example#", + kind: "instantiates", + access: null, + provenance: null, + targetKind: "class", + targetName: "Example", + targetQualifiedName: "com.Example", + evidence: evidence("src/main/kotlin/com/Caller.kt", 4, 40, 4, 55), + }, + // An endpoint outside this compilation: the JDK's Object, which no shard + // in the target declares. + { + from: "semanticdb maven . . com/Caller#make().", + to: "semanticdb maven . . kotlin/lang/Object#toString().", + kind: "calls", + access: null, + provenance: null, + targetKind: "method", + targetName: "toString", + targetQualifiedName: "kotlin.lang.Object.toString", + evidence: evidence("src/main/kotlin/com/Caller.kt", 4, 60, 4, 70), + }, + // The same external symbol reached a second time. One endpoint outside + // the compilation is one node however many sources name it. + { + from: "semanticdb maven . . com/Caller#", + to: "semanticdb maven . . kotlin/lang/Object#toString().", + kind: "references", + access: "read", + provenance: null, + targetKind: "method", + targetName: "toString", + targetQualifiedName: "kotlin.lang.Object.toString", + evidence: evidence("src/main/kotlin/com/Caller.kt", 5, 9, 5, 19), + }, + // The same endpoint reached from two sites, one of which could name it + // and one of which could not. The description that says something wins + // whichever order they arrive in—which is what kotlinc does with a + // reference it attributes at one site and not another. + { + from: "semanticdb maven . . com/Caller#", + to: "semanticdb maven . . kotlin/lang/Deprecated#", + kind: "type_ref", + access: null, + provenance: null, + targetKind: "interface", + targetName: "Deprecated", + targetQualifiedName: "kotlin.lang.Deprecated", + evidence: evidence("src/main/kotlin/com/Caller.kt", 2, 1, 2, 12), + }, + { + from: "semanticdb maven . . com/Caller#make().", + to: "semanticdb maven . . kotlin/lang/Deprecated#", + kind: "decorates", + access: null, + provenance: null, + targetKind: null, + targetName: null, + targetQualifiedName: null, + evidence: evidence("src/main/kotlin/com/Caller.kt", 3, 1, 3, 12), + }, + // An edge whose *origin* the target does not declare. kotlinc takes the + // enclosing symbol of a reference site, and inside an anonymous class + // body that owner is a symbol no compilation unit here declares; the + // producer names what an edge points at, never where it came from, so + // this endpoint has only its own symbol to be displayed by. + { + from: "semanticdb maven . . com/Caller#make().$anon1#run().", + to: "semanticdb maven . . com/Example#", + kind: "calls", + access: null, + provenance: null, + targetKind: "class", + targetName: "Example", + targetQualifiedName: "com.Example", + evidence: evidence("src/main/kotlin/com/Caller.kt", 4, 20, 4, 27), + }, + ], + unresolved: + invocation === 1 || !flag("incremental") + ? [ + { + family: "dispatches", + reason: "dynamic", + evidence: evidence("src/main/kotlin/com/Caller.kt", 4, 60, 4, 70), + // One candidate the target declares and one it does not. The + // first becomes a node identity; the second stays the producer's + // own symbol, because inventing a declaration for it would be + // the opposite of publishing an unresolved site. + candidates: [ + "semanticdb maven . . com/Example#", + "semanticdb maven . . kotlin/lang/Object#", + ], + }, + // A site with nothing to offer. kotlinc could not attribute the + // expression at all, so there is no possibility to name. + { + family: "references", + reason: "analysis-error", + evidence: evidence("src/main/kotlin/com/Caller.kt", 5, 1, 5, 8), + candidates: [], + }, + ] + : [], + }); + +const target = (name, shards) => { + const coverage = { ...COVERAGE }; + if (flag("hole-in-coverage")) delete coverage.tests; + if (flag("claims-unsupported")) coverage.renders = "complete"; + return { + name, + generation: digest(`${name}:${invocation}:${JSON.stringify(shards)}`), + universe: digest( + `${name}:${valueOf("universe") ?? "default"}:${ + flag("moving-universe") ? invocation : "" + }`, + ), + coverage, + shards, + }; +}; + +const shards = [exampleShard(":|jvm|main")]; +if (!flag("single-source")) shards.push(callerShard(":|jvm|main")); +// One build, two JDKs. A Gradle toolchain per source set can do this, and no +// single version is then the build's—which is a different fact from a build +// that never said. +if (flag("two-compilers")) shards[shards.length - 1].compilerVersion = "2.3.10"; +if (flag("incremental") && invocation > 1) { + // Exactly one source recompiled: `Caller.kt` keeps its identity while its + // body moves, and `Example.kt` is byte-identical to the last generation. + const caller = shards[shards.length - 1]; + caller.checkerDigest = digest(`recompiled:${invocation}`); + caller.diskDigest = caller.checkerDigest; +} +if (flag("deleted-source") && invocation > 1) shards.pop(); + +const artifact = { + schemaVersion: flag("future-schema") ? 2 : 1, + projectRoot: flag("foreign-root") + ? path.join(project, "elsewhere") + : project, + producer: { + name: flag("foreign-producer") ? "some-other-graph" : "scip-kotlinc-k2-graph", + version: "0.13.1-fake", + protocolVersion: flag("future-protocol") ? 2 : 1, + capabilities: { + atomicGenerations: !flag("no-atomic-generations"), + incremental: !flag("no-incremental"), + diagnostics: !flag("no-diagnostics"), + }, + }, + targets: flag("no-target") + ? [] + : flag("two-targets") + ? [ + target(":module-a|jvm|main", [exampleShard(":module-a|jvm|main")]), + target(":module-b|jvm|main", [exampleShard(":module-b|jvm|main")]), + ] + : [target(":|jvm|main", shards)], +}; + +// An edge with nothing on one end. A symbol the target does not declare is an +// ordinary external endpoint and becomes a node; an empty string is not an +// endpoint at all, and no reader can be told which declaration it meant. +// One symbol described two ways by two sites that both claim to know. Not the +// same as one site knowing and another not: this is a contradiction, and +// picking either would publish a name the compiler never gave it. +if (flag("two-named-externals")) { + const shard = artifact.targets[0].shards.find((entry) => entry.edges.length > 3); + const named = shard.edges.filter( + (edge) => edge.targetQualifiedName === "kotlin.lang.Object.toString()", + ); + named[1].targetName = "hashCode()"; + named[1].targetQualifiedName = "kotlin.lang.Object.hashCode()"; +} + +if (flag("empty-endpoint")) { + artifact.targets[0].shards[0].edges[0].to = ""; +} +if (flag("unclaimed-family")) { + artifact.targets[0].shards[0].edges[0].kind = "renders"; +} +// One symbol declared by two compilation units of one target. Both sources +// exist, so this reaches the identity rule rather than tripping the disk-digest +// one on the way: kotlinc attributes one declaration per symbol, and two shards +// carrying it would put the same node in one generation twice. +if (flag("duplicate-symbol")) { + const owner = artifact.targets[0]; + const destination = owner.shards[owner.shards.length - 1]; + const duplicate = structuredClone(exampleShard(owner.name).nodes[0]); + duplicate.file = destination.source; + duplicate.evidence.file = destination.source; + destination.nodes.push(duplicate); +} +if (flag("foreign-shard-target")) { + artifact.targets[0].shards[0].target = ":elsewhere|jvm|main"; +} +if (flag("bad-evidence")) { + artifact.targets[0].shards[0].nodes[0].evidence.startLine = 0; +} + +fs.mkdirSync(path.dirname(graphOutput), { recursive: true }); +fs.writeFileSync(graphOutput, `${JSON.stringify(artifact)}\n`); +process.exit(0); +} diff --git a/tests/test-graph/src/internal/fake-lsp-server.cjs b/tests/test-graph/src/internal/fake-lsp-server.cjs index b364ebc3..081200fd 100644 --- a/tests/test-graph/src/internal/fake-lsp-server.cjs +++ b/tests/test-graph/src/internal/fake-lsp-server.cjs @@ -65,7 +65,10 @@ if (process.env.SAMCHON_GRAPH_FAKE_LSP_CWD_FILE) { fs.writeFileSync(process.env.SAMCHON_GRAPH_FAKE_LSP_CWD_FILE, process.cwd()); } if (process.env.SAMCHON_GRAPH_FAKE_LSP_PID_FILE) { - fs.writeFileSync(process.env.SAMCHON_GRAPH_FAKE_LSP_PID_FILE, String(process.pid)); + fs.writeFileSync( + process.env.SAMCHON_GRAPH_FAKE_LSP_PID_FILE, + `${process.pid}\n`, + ); } // Delay only the FIRST textDocument/references response by this many ms, then // answer the rest immediately — models a server that builds its reference index @@ -221,7 +224,7 @@ if (stubbornDescendantPidFile !== undefined) { stdio: "ignore", }, ); - fs.writeFileSync(stubbornDescendantPidFile, String(descendant.pid)); + fs.writeFileSync(stubbornDescendantPidFile, `${descendant.pid}\n`); descendant.unref(); } if (options.ignoreTermination && process.platform !== "win32") { diff --git a/tests/test-graph/src/internal/fake-scala-graph.cjs b/tests/test-graph/src/internal/fake-scala-graph.cjs new file mode 100644 index 00000000..84480893 --- /dev/null +++ b/tests/test-graph/src/internal/fake-scala-graph.cjs @@ -0,0 +1,241 @@ +// Deterministic stand-in for the resident BSP/Scala compiler producer. +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const readline = require("node:readline"); + +const args = process.argv.slice(2); +const sha256 = (value) => + crypto.createHash("sha256").update(value).digest("hex"); +const md5 = (value) => crypto.createHash("md5").update(value).digest("hex"); + +if (args.includes("--version")) { + process.stdout.write("samchon-scala-graph 0.1.0-fake\n"); + process.exit(0); +} + +if (args.includes("supports")) { + process.exit(args.includes("--fake-unsupported") ? 1 : 0); +} + +if (args.includes("snapshot")) { + const index = args.indexOf("--output"); + const output = index === -1 ? undefined : args[index + 1]; + if (output === undefined) { + process.stderr.write("fake Scala graph: snapshot requires --output\n"); + process.exit(2); + } + writeArtifact(output); + process.exit(0); +} + +if (!args.includes("graph-server")) { + process.stderr.write("fake Scala graph: expected graph-server\n"); + process.exit(2); +} +if (args.includes("--help")) { + if (!args.includes("--fake-legacy-server")) { + process.stdout.write( + "Serve BSP-driven Scala compiler graph generations over NDJSON.\n", + ); + } + process.exit(0); +} + +void serve(); + +async function serve() { + const lines = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); + for await (const line of lines) { + if (line.trim() === "") continue; + const request = JSON.parse(line); + try { + writeArtifact(request.output); + process.stdout.write( + `${JSON.stringify({ id: request.id, protocolVersion: 1, ok: true })}\n`, + ); + } catch (error) { + process.stdout.write( + `${JSON.stringify({ + id: request.id, + protocolVersion: 1, + ok: false, + error: error instanceof Error ? error.message : String(error), + })}\n`, + ); + } + } +} + +function writeArtifact(output) { + const root = process.cwd(); + const targets = [ + target(root, "2.13.18", "2.13", "scala2", "src/scala-2/demo/Api.scala"), + target(root, "3.9.0", "3", "scala3", "src/scala-3/demo/Api.scala"), + ]; + const artifact = { + schemaVersion: 1, + projectRoot: root, + producer: { + name: "samchon-scala-graph", + version: "0.1.0-fake", + protocolVersion: 1, + capabilities: { + atomicGenerations: true, + incremental: true, + diagnostics: true, + bsp: true, + semanticdb: true, + typedPlugins: true, + zinc: true, + }, + }, + targets, + }; + fs.mkdirSync(path.dirname(output), { recursive: true }); + fs.writeFileSync(output, `${JSON.stringify(artifact)}\n`); +} + +function target(root, scalaVersion, scalaBinaryVersion, plugin, source) { + const bspUri = `file:///samchon-graph/${plugin}`; + const bytes = fs.readFileSync(path.join(root, source)); + const coordinate = (name) => sha256(`${bspUri}:${name}`); + const owner = `scala ${bspUri} demo Api#`; + const method = `scala ${bspUri} demo Api#run().`; + const nodes = [ + { + symbol: owner, + kind: "class", + name: "Api", + qualifiedName: "demo.Api", + file: source, + exported: true, + modifiers: ["public"], + signature: "class Api[A](value: A)", + origin: "Source", + evidence: evidence(source, 3, 1, 3, 28), + }, + { + symbol: method, + kind: "method", + name: "run", + qualifiedName: "demo.Api.run", + file: source, + exported: true, + modifiers: ["public"], + signature: "def run(): A", + origin: "Source", + evidence: evidence(source, 4, 3, 4, 23), + }, + ]; + const edges = [ + edge(source, owner, "contains", source, 3, 1, 3, 28, "class", "Api", "demo.Api"), + edge(source, owner, "exports", source, 3, 1, 3, 28, "class", "Api", "demo.Api"), + edge(owner, method, "contains", source, 4, 3, 4, 23, "method", "run", "demo.Api.run"), + edge( + method, + "scala-library scala Predef.println().", + "calls", + source, + 4, + 16, + 4, + 23, + "method", + "println", + "scala.Predef.println", + ), + ]; + const coverage = {}; + for (const family of [ + "contains", "exports", "imports", "calls", "accesses", "instantiates", + "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", + "renders", "tests", "references", + ]) coverage[family] = ["renders", "tests"].includes(family) + ? "unsupported" + : "partial"; + coverage.contains = "complete"; + coverage.calls = "partial"; + return { + name: bspUri, + generation: sha256(JSON.stringify({ bspUri, bytes: bytes.toString("hex") })), + universe: coordinate(`universe:${scalaVersion}`), + bspUri, + scalaVersion, + scalaBinaryVersion, + platform: "jvm", + sourceEncoding: "UTF-8", + scalacOptionsDigest: coordinate("scalacOptions"), + classpathDigest: coordinate("classpath"), + sourceRootsDigest: coordinate("sourceRoots"), + semanticdbOptionsDigest: coordinate("semanticdbOptions"), + compilerPluginsDigest: coordinate("compilerPlugins"), + zincAnalysisDigest: coordinate("zincAnalysis"), + generatedSourcesDigest: coordinate("generatedSources"), + coverage, + shards: [ + { + schemaVersion: 1, + language: "scala", + source, + checkerDigest: sha256(bytes), + diskDigest: sha256(bytes), + target: bspUri, + compilerVersion: scalaVersion, + compilerPlugin: plugin, + compilerPluginVersion: "0.1.0-fake", + semanticdbSchema: 4, + semanticdbUri: source, + semanticdbMd5: md5(bytes), + semanticdbBuildTarget: bspUri, + nodes, + edges, + unresolved: [ + { + family: "dispatches", + reason: "dynamic", + evidence: evidence(source, 4, 16, 4, 23), + candidates: ["scala-library scala Predef.println()."], + }, + ], + diagnostics: [ + { + severity: "warning", + message: `${plugin} fixture warning`, + evidence: evidence(source, 2, 1, 2, 12), + }, + ], + }, + ], + }; +} + +function evidence(file, startLine, startColumn, endLine, endColumn) { + return { file, startLine, startColumn, endLine, endColumn }; +} + +function edge( + from, + to, + kind, + file, + startLine, + startColumn, + endLine, + endColumn, + targetKind, + targetName, + targetQualifiedName, +) { + return { + from, + to, + kind, + access: null, + provenance: kind === "calls" ? "typed-plugin" : "semanticdb", + targetKind, + targetName, + targetQualifiedName, + evidence: evidence(file, startLine, startColumn, endLine, endColumn), + }; +} diff --git a/tests/test-graph/src/internal/fake-scip-indexer.cjs b/tests/test-graph/src/internal/fake-scip-indexer.cjs index 4989cc5e..33022609 100644 --- a/tests/test-graph/src/internal/fake-scip-indexer.cjs +++ b/tests/test-graph/src/internal/fake-scip-indexer.cjs @@ -67,6 +67,23 @@ if (mode === "stdout-fail") { process.exit(3); } +if (mode === "stderr-fail") { + // The conventional tool shape, made long enough to prove that stderr is + // subject to the same diagnostic bound as stdout and a split pair. + process.stderr.write(`OPENING ERROR\n${"diagnostic noise\n".repeat(300)}`); + process.stderr.write("FAILURE: compiler rejected the project\n"); + process.exit(3); +} + +if (mode === "both-streams-fail") { + // The Maven witness: the JVM writes an informational environment notice to + // stderr while the build tool writes the actionable failure to stdout. + process.stderr.write("Picked up JAVA_TOOL_OPTIONS: -Dfixture=true\n"); + process.stdout.write(`OPENING LINE\n${"build progress\n".repeat(300)}`); + process.stdout.write("FAILURE: Maven could not compile the project\n"); + process.exit(3); +} + if (mode === "silent") { // Exits cleanly having written nothing. The session must notice the missing // artifact rather than decoding whatever was there before. diff --git a/tests/test-graph/src/internal/fake-swift-graph.cjs b/tests/test-graph/src/internal/fake-swift-graph.cjs new file mode 100644 index 00000000..85b9093e --- /dev/null +++ b/tests/test-graph/src/internal/fake-swift-graph.cjs @@ -0,0 +1,281 @@ +// Deterministic stand-in for the SwiftPM/IndexStoreDB sidecar. +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const readline = require("node:readline"); + +const args = process.argv.slice(2); +const sha256 = (value) => crypto.createHash("sha256").update(value).digest("hex"); + +if (args.includes("--version")) { + process.stdout.write("samchon-swift-graph 0.1.0-fake\n"); + process.exit(0); +} +if (args.includes("supports")) { + process.exit(args.includes("--fake-unsupported") ? 1 : 0); +} +if (args.includes("snapshot")) { + const index = args.indexOf("--output"); + const output = index === -1 ? undefined : args[index + 1]; + if (output === undefined) { + process.stderr.write("fake Swift graph: snapshot requires --output\n"); + process.exit(2); + } + writeArtifact(output); + process.exit(0); +} +if (!args.includes("graph-server")) { + process.stderr.write("fake Swift graph: expected graph-server\n"); + process.exit(2); +} +if (args.includes("--help")) { + if (!args.includes("--fake-legacy-server")) { + process.stdout.write( + "Serve explicit-output-unit SwiftPM IndexStoreDB generations over NDJSON.\n", + ); + } + process.exit(0); +} + +void serve(); + +async function serve() { + const lines = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); + for await (const line of lines) { + if (line.trim() === "") continue; + const request = JSON.parse(line); + try { + writeArtifact(request.output); + process.stdout.write( + `${JSON.stringify({ id: request.id, protocolVersion: 1, ok: true })}\n`, + ); + } catch (error) { + process.stdout.write( + `${JSON.stringify({ + id: request.id, + protocolVersion: 1, + ok: false, + error: error instanceof Error ? error.message : String(error), + })}\n`, + ); + } + } +} + +function writeArtifact(output) { + const root = process.cwd(); + const source = "Sources/Demo/Api.swift"; + const bytes = fs.readFileSync(path.join(root, source)); + const stale = path.join( + root, + ".build/x86_64-unknown-linux-gnu/debug/Stale.build/Old.swift.o", + ); + fs.mkdirSync(path.dirname(stale), { recursive: true }); + fs.writeFileSync(stale, "stale unit that must remain outside the generation"); + const triples = ["arm64-apple-macosx13.0", "x86_64-unknown-linux-gnu"]; + const targets = triples.map((triple) => { + const unit = `.build/${triple}/debug/Demo.build/Api.swift.o`; + const unitFile = path.join(root, unit); + fs.mkdirSync(path.dirname(unitFile), { recursive: true }); + fs.writeFileSync(unitFile, sha256(bytes)); + return target(root, source, bytes, unit, triple); + }); + const artifact = { + schemaVersion: 1, + projectRoot: root, + producer: { + name: "samchon-swift-graph", + version: "0.1.0-fake", + protocolVersion: 1, + capabilities: { + atomicGenerations: true, + incremental: true, + diagnostics: true, + explicitOutputUnits: true, + indexStoreDB: true, + sourceEnrichment: true, + swiftpm: true, + sourceKitResident: false, + }, + }, + targets, + }; + fs.mkdirSync(path.dirname(output), { recursive: true }); + fs.writeFileSync(output, `${JSON.stringify(artifact)}\n`); +} + +function target(root, source, bytes, unit, triple) { + const name = `Demo@${triple}/debug`; + const service = "s:4Demo7ServiceP"; + const base = "s:4Demo4BaseC"; + const baseRun = "s:4Demo4BaseC3runyyF"; + const api = "s:4Demo3ApiC"; + const extension = "s:e:4Demo3ApiC"; + const constructor = "s:4Demo3ApiC4seedACSi_tcfc"; + const property = "s:4Demo3ApiC5valueSivp"; + const getter = "s:4Demo3ApiC5valueSivg"; + const setter = "s:4Demo3ApiC5valueSivs"; + const run = "s:4Demo3ApiC3runyyF"; + const fetch = "s:4Demo3ApiC5fetchySSqd__Ya_lF"; + const local = "s:L_4Demo3ApiC3runyyF5localL_Sivp"; + const macro = "s:4Demo12FixtureMacrofMp"; + const macroHost = "s:4Demo9MacroHostV"; + const test = "s:4Demo7testApiyyYaF"; + const coverage = {}; + for (const family of [ + "contains", "exports", "imports", "calls", "accesses", "instantiates", + "type_ref", "extends", "implements", "overrides", "dispatches", "decorates", + "renders", "tests", "references", + ]) coverage[family] = family === "renders" ? "unsupported" : "partial"; + const coordinate = (value) => sha256(`${name}:${value}`); + return { + name, + generation: coordinate(`generation:${sha256(bytes)}`), + universe: coordinate("universe"), + moduleName: "Demo", + targetTriple: triple, + sdk: triple.startsWith("arm64") ? "/SDK/MacOSX.sdk" : "", + configuration: "debug", + swiftLanguageVersion: "Swift 6.0", + compilerFlagsDigest: coordinate("flags"), + moduleDependenciesDigest: coordinate("modules"), + packageResolutionDigest: coordinate("resolution"), + pluginsDigest: coordinate("plugins"), + generatedSourcesDigest: coordinate("generated"), + indexStoreDBCommit: "54212fce1aecb199070808bdb265e7f17e396015", + outputUnits: [{ path: unit, digest: sha256(fs.readFileSync(path.join(root, unit))) }], + coverage, + shards: [{ + schemaVersion: 1, + language: "swift", + source, + checkerDigest: sha256(bytes), + diskDigest: sha256(bytes), + target: name, + compilerVersion: "Swift 6.0", + moduleName: "Demo", + targetTriple: triple, + sourceEnrichmentPasses: 1, + nodes: [ + node(service, "interface", "Service", "Demo.Service", source, 3, "public protocol Service"), + node(base, "class", "Base", "Demo.Base", source, 7, "open class Base"), + node(baseRun, "method", "run", "Demo.Base.run", source, 10, "open func run()"), + node(api, "class", "Api", "Demo.Api", source, 12, "public final class Api: Base, Service"), + node(extension, "type", "Api extension", "Demo.Api.extension", source, 25, "extension Api", [], false), + node(constructor, "constructor", "init(seed:)", "Demo.Api.init(seed:)", source, 26, "public convenience init(seed: Int)"), + node(property, "property", "value", "Demo.Api.value", source, 14, "public override var value: Int"), + node(getter, "method", "get value", "Demo.Api.value.get", source, 15, "get", [], false), + node(setter, "method", "set value", "Demo.Api.value.set", source, 16, "set", [], false), + node(run, "method", "run", "Demo.Api.run", source, 18, "public override final func run()"), + node(fetch, "method", "fetch", "Demo.Api.fetch", source, 23, "public func fetch(_ value: T) async -> String", ["public", "async"]), + node(local, "variable", "local", "Demo.Api.run.local", source, 19, "var local", [], false, "IndexStoreDB:index-include-locals"), + node(macro, "type", "FixtureMacro", "Demo.FixtureMacro", source, 32, "public macro FixtureMacro()"), + node(macroHost, "type", "MacroHost", "Demo.MacroHost", source, 34, "public struct MacroHost"), + node(test, "function", "testApi", "Demo.testApi", source, 36, "public func testApi() async", ["public", "async"]), + ], + edges: [ + edge(source, service, "contains", source, 3, "interface", "Service", "Demo.Service"), + edge(source, api, "exports", source, 12, "class", "Api", "Demo.Api"), + edge(source, "swift-module:Foundation", "imports", source, 1, "module", "Foundation", "Foundation"), + edge(api, "swift-attribute:MainActor", "decorates", source, 2, "type", "MainActor", "MainActor"), + edge(api, base, "extends", source, 12, "class", "Base", "Demo.Base"), + edge(api, service, "implements", source, 12, "interface", "Service", "Demo.Service"), + edge(extension, api, "type_ref", source, 25, "class", "Api", "Demo.Api"), + edge(extension, constructor, "contains", source, 26, "constructor", "init(seed:)", "Demo.Api.init(seed:)"), + edge(property, getter, "contains", source, 15, "method", "get value", "Demo.Api.value.get"), + edge(property, setter, "contains", source, 16, "method", "set value", "Demo.Api.value.set"), + edge(run, baseRun, "overrides", source, 18, "method", "run", "Demo.Base.run"), + edge(test, constructor, "calls", source, 37, "constructor", "init(seed:)", "Demo.Api.init(seed:)"), + edge(test, constructor, "instantiates", source, 37, "constructor", "init(seed:)", "Demo.Api.init(seed:)"), + edge(test, property, "accesses", source, 38, "property", "value", "Demo.Api.value", "write"), + edge(test, property, "accesses", source, 39, "property", "value", "Demo.Api.value", "read"), + edge(test, run, "calls", source, 40, "method", "run", "Demo.Api.run"), + edge(test, run, "dispatches", source, 40, "method", "run", "Demo.Api.run"), + edge(test, run, "tests", source, 40, "method", "run", "Demo.Api.run"), + edge(fetch, service, "references", source, 23, "interface", "Service", "Demo.Service"), + edge(run, local, "references", source, 20, "variable", "local", "Demo.Api.run.local"), + ], + unresolved: [ + { + family: "dispatches", + reason: "dynamic", + evidence: evidence(source, 40, 3, 40, 14), + candidates: [baseRun, run], + }, + { + family: "references", + reason: "conditional-build", + evidence: evidence(source, 30, 1, 30, 17), + candidates: [], + }, + { + family: "references", + reason: "macro-or-generated", + evidence: evidence(source, 32, 32, 32, 46), + candidates: [macroHost], + }, + ], + diagnostics: [{ + severity: "warning", + message: "fixture warning", + evidence: evidence(source, 43, 4, 43, 11), + }], + }], + }; +} + +function node( + symbol, + kind, + name, + qualifiedName, + file, + line, + signature, + modifiers = ["public"], + exported = true, + origin = "IndexStoreDB+source-enrichment", +) { + return { + symbol, + kind, + name, + qualifiedName, + file, + exported, + modifiers, + signature, + origin, + evidence: evidence(file, line, 1, line, name.length + 1), + }; +} + +function edge( + from, + to, + kind, + file, + line, + targetKind, + targetName, + targetQualifiedName, + access = null, +) { + return { + from, + to, + kind, + access, + provenance: kind === "imports" || kind === "decorates" + ? "source-enrichment" + : "IndexStoreDB", + targetKind, + targetName, + targetQualifiedName, + evidence: evidence(file, line, 1, line, targetName.length + 1), + }; +} + +function evidence(file, startLine, startColumn, endLine, endColumn) { + return { file, startLine, startColumn, endLine, endColumn }; +} diff --git a/tests/test-graph/src/internal/waitForProcessId.ts b/tests/test-graph/src/internal/waitForProcessId.ts new file mode 100644 index 00000000..fab7d90a --- /dev/null +++ b/tests/test-graph/src/internal/waitForProcessId.ts @@ -0,0 +1,31 @@ +import fs from "node:fs"; + +/** Wait until a fixture has published one complete, positive process id. */ +export const waitForProcessId = async ( + file: string, + timeoutMs = 5_000, +): Promise => { + const deadline = Date.now() + timeoutMs; + for (;;) { + const candidate = readProcessId(file); + if (candidate !== undefined) return candidate; + if (Date.now() >= deadline) { + throw new Error(`timed out waiting for a complete process id in ${file}`); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } +}; + +const readProcessId = (file: string): number | undefined => { + let value: string; + try { + value = fs.readFileSync(file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + const match = /^([1-9]\d*)\r?\n$/.exec(value); + if (match === null) return undefined; + const pid = Number(match[1]); + return Number.isSafeInteger(pid) ? pid : undefined; +};