perf: add latency benchmarks, and fix the latency and correctness issues of a code review

.github/workflows/ci.yml

@@ -66,6 +66,30 @@ jobs:
       - name: Test
         run: nix develop ".#${{ matrix.shell }}" -c make test
 
+  # Shared runners are noisy, so a failed comparison is a warning to read in the job summary, not a failed check
+  bench:
+    name: Benchmarks
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          # The pull request merge commit and its parents. The first parent is the base branch.
+          fetch-depth: 2
+      - name: Install Nix
+        uses: cachix/install-nix-action@448cde739abec1eaf0646249e3d25bb8d9bc7702 # v31.11.2
+        with:
+          extra_nix_config: |
+            extra-substituters = https://nix-community.cachix.org
+            extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=
+      - name: Compare with the base branch
+        env:
+          BENCH_RUNS: 3
+        run: nix develop -c scripts/bench-compare.sh HEAD^1
+
   # Dependabot and `nix flake update` can lock an overlay commit that is not in the cache yet. Then
   # `nix develop .#nightly` builds Neovim from source.
   nightly-cache:

CONTRIBUTING.md

@@ -39,16 +39,18 @@ extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq
 
 ## Make targets
 
-| Target                       | Action                                                                                              |
-| ---------------------------- | --------------------------------------------------------------------------------------------------- |
-| `make test`                  | Runs all tests.                                                                                     |
-| `make test-file FILE=<path>` | Runs one test file.                                                                                 |
-| `make format`                | Formats `lua/` and `tests/` with StyLua.                                                            |
-| `make format-check`          | Checks the formatting.                                                                              |
-| `make lint`                  | Lints `lua/` and `tests/` with Selene. Warnings fail the target.                                    |
-| `make typecheck`             | Checks the types with lua-language-server. Warnings fail the target.                                |
-| `make ci`                    | Runs `format-check`, `lint`, `typecheck`, and `test`.                                               |
-| `make test-deps`             | Clones mini.nvim at `MINI_VERSION` into `deps/`. `test`, `test-file`, and `typecheck` run it first. |
+| Target                         | Action                                                                                              |
+| ------------------------------ | --------------------------------------------------------------------------------------------------- |
+| `make test`                    | Runs all tests.                                                                                     |
+| `make test-file FILE=<path>`   | Runs one test file.                                                                                 |
+| `make format`                  | Formats `lua/`, `tests/`, and `benchmarks/` with StyLua.                                            |
+| `make format-check`            | Checks the formatting.                                                                              |
+| `make lint`                    | Lints `lua/`, `tests/`, and `benchmarks/` with Selene. Warnings fail the target.                    |
+| `make typecheck`               | Checks the types with lua-language-server. Warnings fail the target.                                |
+| `make ci`                      | Runs `format-check`, `lint`, `typecheck`, and `test`.                                               |
+| `make test-deps`               | Clones mini.nvim at `MINI_VERSION` into `deps/`. `test`, `test-file`, and `typecheck` run it first. |
+| `make bench`                   | Runs the benchmarks. Options go in `BENCH_ARGS`.                                                    |
+| `make bench-compare REF=<rev>` | Compares the benchmarks of `<rev>` (default `main`) with the working tree.                          |
 
 To use a different tool, set its variable: `NVIM`, `STYLUA`, `SELENE`, or `LUA_LANGUAGE_SERVER`. For example:
 
@@ -101,18 +103,72 @@ nix develop .#nightly --no-write-lock-file \
   --override-input neovim-nightly-overlay "github:nix-community/neovim-nightly-overlay/$rev" -c make test
 ```
 
+## Benchmarks
+
+The benchmarks in `benchmarks/` measure the two costs that users feel: the work that runs while they type, and the time
+until the completion items show. Nothing in them uses the network.
+
+| Group        | Cases                            | Measures                                                                              |
+| ------------ | -------------------------------- | ------------------------------------------------------------------------------------- |
+| `keystroke/` | `benchmarks/cases/keystroke.lua` | The checks that nvim-cmp and blink.cmp make on each keystroke, and declined requests. |
+| `request/`   | `benchmarks/cases/request.lua`   | Routing, parsing, and item formatting for one request. Also the startup cost.         |
+| `e2e/`       | `benchmarks/cases/e2e.lua`       | One request from start to the final callback, in a git repository with 1000 commits.  |
+
+The `e2e/` cases run the real git commands. A fake `gh` in a temporary directory answers the GitHub requests at once,
+so the times include the process starts, but not the network or the run time of the real `gh`. `cold` cases use a new
+source for each request. `warm` cases repeat a request that the cache can answer.
+
+The `startup/` cases measure the cost to load cmp-git. `startup/setup` is the cost of `setup`. Core loads the sources
+and the repository module when a request first needs them, so `startup/require_all` loads all the modules. The test
+`setup_and_a_declined_request_load_only_the_core_modules` in `tests/test_core.lua` lists the modules that `setup` and a
+declined request load. A new `require` at the top of one of these modules adds to the cost of each session. Put the
+`require` in the code that needs the module.
+
+Each line of the report has these values:
+
+| Value      | Meaning                                                                                                  |
+| ---------- | -------------------------------------------------------------------------------------------------------- |
+| `median`   | The median time of one operation.                                                                        |
+| `alloc`    | The Lua memory that one operation allocates. It does not change between runs, so it shows small changes. |
+| `main`     | `e2e/` only. The main-thread time of one request. Neovim cannot handle a keystroke during this time.     |
+| `block`    | `e2e/` only. The longest uninterrupted main-thread slice of one request.                                 |
+| `spawns`   | The processes that one operation starts.                                                                 |
+| `blocking` | The synchronous process calls (`vim.fn.system`) of one operation. These stop Neovim until they finish.   |
+
+To run the benchmarks:
+
+```sh
+make bench
+make bench BENCH_ARGS="--filter keystroke"
+```
+
+To compare your changes with `main`:
+
+```sh
+make bench-compare
+```
+
+`scripts/bench-compare.sh` runs the cases of the working tree on the `lua/` code of both revisions, two runs each,
+and keeps the best value of each side. It reports a regression when a time grows by more than 25%, an allocation
+grows by more than 10%, or a process count grows. Very small changes do not count. The options of
+`benchmarks/compare.lua` change the limits.
+
+To add a case, add it to the file of its group. A case that starts a process must be an async case that calls `done`.
+Use the fixtures in `benchmarks/fixtures.lua` so that the case does not use the network.
+
 ## CI
 
 The workflow in `.github/workflows/ci.yml` runs on pushes to `main`, on pull requests, and on manual dispatch. It
 has these jobs:
 
-| Job              | Steps                                                                                                     | Blocks |
-| ---------------- | --------------------------------------------------------------------------------------------------------- | ------ |
-| `Quality`        | `make format-check`, `make lint`, and `make typecheck` in the default shell                               | Yes    |
-| `Neovim stable`  | `make test` in the default shell                                                                          | Yes    |
-| `Neovim 0.12.0`  | `make test` in the `nvim-0_12_0` shell                                                                    | Yes    |
-| `Nightly cache`  | `scripts/nightly-cache.sh check` (the locked nightly Neovim is in the cache for each system)              | Yes    |
-| `Neovim nightly` | `make test` with the newest `neovim-nightly-overlay` commit that is in the cache, not the locked revision | No     |
+| Job              | Steps                                                                                                      | Blocks |
+| ---------------- | ---------------------------------------------------------------------------------------------------------- | ------ |
+| `Quality`        | `make format-check`, `make lint`, and `make typecheck` in the default shell                                | Yes    |
+| `Neovim stable`  | `make test` in the default shell                                                                           | Yes    |
+| `Neovim 0.12.0`  | `make test` in the `nvim-0_12_0` shell                                                                     | Yes    |
+| `Nightly cache`  | `scripts/nightly-cache.sh check` (the locked nightly Neovim is in the cache for each system)               | Yes    |
+| `Neovim nightly` | `make test` with the newest `neovim-nightly-overlay` commit that is in the cache, not the locked revision  | No     |
+| `Benchmarks`     | Pull requests only. `scripts/bench-compare.sh` against the base branch, with the report in the job summary | No     |
 
 A weekly schedule (Monday 06:00 UTC) runs only the nightly job. GitHub disables scheduled workflows after 60 days
 without repository activity. To enable the schedule again, use the Actions tab of the repository.

Makefile

@@ -4,9 +4,10 @@ SELENE ?= selene
 LUA_LANGUAGE_SERVER ?= lua-language-server
 MINI_VERSION := v0.18.0
 MINI_DIR := deps/mini.nvim
-LUA_PATHS := lua tests
+LUA_PATHS := lua tests benchmarks
+REF ?= main
 
-.PHONY: test-deps test test-file format format-check lint typecheck ci
+.PHONY: test-deps test test-file bench bench-compare format format-check lint typecheck ci
 
 test-deps:
 	@if [ ! -d "$(MINI_DIR)/.git" ]; then \
@@ -27,6 +28,14 @@ test-file: test-deps
 	TEST_FILE="$(FILE)" $(NVIM) --headless --noplugin -u tests/minimal_init.lua \
 		-c "lua MiniTest.run_file(vim.env.TEST_FILE)"
 
+# Prints one line for each case. Options go in BENCH_ARGS, for example BENCH_ARGS="--filter keystroke".
+bench:
+	$(NVIM) --headless --clean -l benchmarks/run.lua $(BENCH_ARGS)
+
+# Compares the benchmarks of REF with the working tree
+bench-compare:
+	NVIM="$(NVIM)" scripts/bench-compare.sh "$(REF)"
+
 format:
 	$(STYLUA) $(LUA_PATHS)
 

README.md

@@ -27,7 +27,7 @@ Git source for [hrsh7th/nvim-cmp](https://github.com/hrsh7th/nvim-cmp) and [Sagh
 
 - Neovim >= 0.12
 - git (>= 2.31 to find trailer keys in history)
-- curl
+- curl (>= 7.55)
 - [GitHub CLI](https://cli.github.com/) (optional, cmp-git uses curl if it is not available)
 - [GitLab CLI](https://gitlab.com/gitlab-org/cli) (optional, cmp-git uses curl if it is not available)
 
@@ -195,7 +195,7 @@ require("cmp_git").setup({
         },
     },
     gitlab = {
-        hosts = {},  -- list of private instances of gitlab
+        hosts = {},  -- list of private instances of gitlab, with the path prefix if any, such as "example.com/gitlab"
         issues = {
             limit = 100,
             state = "opened", -- opened, closed, all
@@ -220,7 +220,8 @@ require("cmp_git").setup({
             actions = { "git_trailer_keys" },
         },
         {
-            pattern = [[\%(^[[:alnum:]-]\+:\s*\)\@<=\%([^[:space:]<>:#@!][^<>:#@!]*\%(<[^>]*\)\?\)\?]],
+            pattern = [[^[[:alnum:]-]\+:\s*\zs\%([^[:space:]<>:#@!][^<>:#@!]*\%(<[^>]*\)\?\)\?]],
+            keyword_pattern = [[^[[:alnum:]-]\+\zs:\S*\|^[[:alnum:]-]\+:\s*\zs\%([^[:space:]<>:#@!][^<>:#@!]*\%(<[^>]*\)\?\)\?]],
             actions = { "git_trailer_identities", "git_trailer_values" },
         },
         {

benchmarks/cases/e2e.lua

@@ -0,0 +1,84 @@
+---Completion requests from start to the final callback, in a git repository with 1000 commits. Each request runs the
+---real git commands and a fake `gh` (see `fixtures.isolate_environment`), so the times include process starts but no
+---network. `cold` cases use a new source for each request, and `warm` cases repeat a request that the cache can
+---answer.
+local fixtures = require("benchmarks.fixtures")
+
+local COMMITS = 1000
+
+local LINES = {
+    "Fix the parser",
+    "",
+    "See the notes",
+    "",
+    "Co-authored-by: Ad",
+    "Refs: AB",
+    "Sig",
+}
+
+---@type table<string, { row: integer, line: string, trigger_character?: string }>
+local REQUESTS = {
+    commits = { row = 3, line = "See :", trigger_character = ":" },
+    trailer_keys = { row = 7, line = "Sig" },
+    trailer_identities = { row = 5, line = "Co-authored-by: Ad" },
+    trailer_values = { row = 6, line = "Refs: AB" },
+    github_issues_and_prs = { row = 3, line = "See #", trigger_character = "#" },
+    github_mentions = { row = 3, line = "See @", trigger_character = "@" },
+}
+
+---@param name string A key of `REQUESTS`.
+---@param cold boolean
+---@return bench.AsyncCase
+local function request_case(name, cold)
+    local request = REQUESTS[name]
+    return {
+        name = string.format("e2e/%s/%s", name, cold and "cold" or "warm"),
+        async = true,
+        setup = function()
+            local dir = fixtures.repo(COMMITS)
+            return {
+                bufnr = fixtures.commit_buffer(dir, LINES),
+                core = require("cmp_git.core").new({}),
+            }
+        end,
+        before_each = function(state)
+            vim.api.nvim_set_current_buf(state.bufnr)
+            if cold then
+                state.core = require("cmp_git.core").new({})
+            end
+        end,
+        run = function(state, done)
+            state.core:complete({
+                trigger_character = request.trigger_character,
+                cursor = { row = request.row, character = #request.line },
+                line = request.line,
+                bufnr = state.bufnr,
+            }, function(list)
+                if list.isIncomplete then
+                    return
+                end
+                if #list.items == 0 then
+                    done("the request gave no items")
+                    return
+                end
+                done()
+            end)
+        end,
+    }
+end
+
+---@type bench.Case[]
+local cases = {}
+for _, name in ipairs({
+    "commits",
+    "trailer_keys",
+    "trailer_identities",
+    "trailer_values",
+    "github_issues_and_prs",
+    "github_mentions",
+}) do
+    table.insert(cases, request_case(name, true))
+    table.insert(cases, request_case(name, false))
+end
+
+return cases

benchmarks/cases/keystroke.lua

@@ -0,0 +1,103 @@
+---Work that runs while the user types: the checks that nvim-cmp and blink.cmp make on each keystroke, and requests
+---that cmp-git declines without starting a process.
+local fixtures = require("benchmarks.fixtures")
+
+local LINES = {
+    "Fix the parser",
+    "",
+    "Body text",
+    "",
+    "Co-authored-by: Ada Lovelace <ada@example.com>",
+}
+
+local function noop() end
+
+---nvim-cmp finds the keyword start on each keystroke with the source's keyword pattern. It compiles the regex once
+---(`cmp.utils.pattern.offset`).
+---@param line string
+---@return bench.SyncCase
+local function keyword_offset(name, line)
+    return {
+        name = "keystroke/keyword_offset/" .. name,
+        setup = function()
+            local core = require("cmp_git.core").new({})
+            return { regex = vim.regex([[\%(]] .. core:get_keyword_pattern() .. [[\)\m$]]), line = line }
+        end,
+        run = function(state)
+            state.regex:match_str(state.line)
+        end,
+    }
+end
+
+---An nvim-cmp request from `cmp.Source.complete`.
+---@param line string
+---@param row integer
+---@return cmp_git.CmpCompletionParams
+local function cmp_params(line, row)
+    return {
+        context = {
+            cursor_before_line = line,
+            cursor = { row = row, col = #line + 1, character = #line },
+        },
+        completion_context = { triggerKind = 1 },
+    }
+end
+
+---@type bench.Case[]
+return {
+    {
+        name = "keystroke/is_available",
+        setup = function()
+            fixtures.scratch_buffer(LINES)
+            return require("cmp_git.core").new({})
+        end,
+        run = function(core)
+            core:is_available()
+        end,
+    },
+    keyword_offset("prose_80", fixtures.prose(80)),
+    keyword_offset("prose_300", fixtures.prose(300)),
+    keyword_offset("prose_1000", fixtures.prose(1000)),
+    keyword_offset("trailer_value", "Co-authored-by: Ada Lov"),
+    {
+        -- The first word of the subject matches the trailer key pattern, but trailers start below the subject
+        name = "keystroke/nvim_cmp/complete_declined",
+        setup = function()
+            fixtures.scratch_buffer(LINES)
+            return { source = require("cmp_git.source").new({}), params = cmp_params("Fix", 1) }
+        end,
+        run = function(state)
+            state.source:complete(state.params, noop)
+        end,
+    },
+    {
+        name = "keystroke/nvim_cmp/complete_declined_prose_300",
+        setup = function()
+            local line = fixtures.prose(300)
+            fixtures.scratch_buffer({ "Fix the parser", "", line })
+            return { source = require("cmp_git.source").new({}), params = cmp_params(line, 3) }
+        end,
+        run = function(state)
+            state.source:complete(state.params, noop)
+        end,
+    },
+    {
+        name = "keystroke/blink/get_completions_declined",
+        setup = function()
+            local bufnr = fixtures.scratch_buffer(LINES)
+            return {
+                source = require("cmp_git.blink").new({}),
+                ctx = {
+                    bufnr = bufnr,
+                    line = "Fix the parser",
+                    cursor = { 1, 3 },
+                    trigger = { kind = "keyword" },
+                },
+            }
+        end,
+        run = function(state)
+            local cancel = state.source:get_completions(state.ctx, noop)
+            cancel()
+        end,
+    },
+}