paperless.2: filtered-KNN performance PRs and AVX cosine dispatch

.github/workflows/test.yaml

@@ -39,7 +39,7 @@ jobs:
           name: sqlite-vec-macos-x86_64-extension
           path: dist/*
   build-macos-aarch64-extension:
-    runs-on: macos-14
+    runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v7
@@ -53,26 +53,6 @@ jobs:
         with:
           name: sqlite-vec-macos-aarch64-extension
           path: dist/*
-  build-windows-x86_64-extension:
-    runs-on: windows-2022
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ilammy/msvc-dev-cmd@v1
-      - uses: astral-sh/setup-uv@v7
-        with:
-          enable-cache: true
-      - run: ./scripts/vendor.sh
-        shell: bash
-      - run: make sqlite-vec.h
-      - run: mkdir dist
-      - run: cl.exe /fPIC -shared /W4 /Ivendor/ /O2 /LD sqlite-vec.c -o dist/vec0.dll
-      - run: uv sync --directory tests
-      - run: make test-loadable
-        shell: bash
-      - uses: actions/upload-artifact@v4
-        with:
-          name: sqlite-vec-windows-x86_64-extension
-          path: dist/*
   build-linux-aarch64-extension:
     runs-on: ubuntu-22.04-arm
     steps:
@@ -87,165 +67,6 @@ jobs:
         with:
           name: sqlite-vec-linux-aarch64-extension
           path: dist/*
-  build-android-extensions:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        platforms:
-          [
-            { name: android-aarch64, cc: aarch64-linux-android21-clang },
-            { name: android-i686, cc: i686-linux-android21-clang },
-            { name: android-x86_64, cc: x86_64-linux-android21-clang },
-            { name: android-armv7a, cc: armv7a-linux-androideabi21-clang },
-          ]
-    steps:
-      - uses: actions/checkout@v4
-      - run: ./scripts/vendor.sh
-      - uses: actions/setup-java@v4
-        with:
-          java-version: "17"
-          distribution: "temurin"
-      - uses: android-actions/setup-android@v3
-      - run: |
-          sdkmanager --install "ndk;27.0.12077973"
-          echo "ANDROID_NDK_HOME=$ANDROID_SDK_ROOT/ndk/27.0.12077973" >> $GITHUB_ENV
-      - run: |
-          ls $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/
-          ls $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin
-      - run: |
-          export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH
-          make CC=${{ matrix.platforms.cc }} \
-             AR=llvm-ar \
-             RANLIB=llvm-ranlib \
-             STRIP=llvm-strip loadable
-      - uses: actions/upload-artifact@v4
-        with:
-          name: sqlite-vec-android-${{ matrix.platforms.name }}-extension
-          path: dist/*.so
-  build-ios-extensions:
-    runs-on: ${{ matrix.platforms.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        platforms: [
-            {
-              name: ios-arm64,
-              target: arm64-apple-ios,
-              sdk: iphoneos,
-              runner: macos-14,
-            },
-            #{
-            #  name: ios-x86_64,
-            #  target: x86_64-apple-ios,
-            #  sdk: iphoneos,
-            #  runner: macos-12,
-            #},
-            {
-              name: ios-simulator-arm64,
-              target: arm64-apple-ios-simulator,
-              sdk: iphonesimulator,
-              runner: macos-14,
-            },
-            {
-              name: ios-simulator-x86_64,
-              target: x86_64-apple-ios-simulator,
-              sdk: iphonesimulator,
-              runner: macos-14,
-            },
-          ]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: maxim-lobanov/setup-xcode@v1
-        with:
-          xcode-version: latest-stable
-      - run: ./scripts/vendor.sh
-
-      - run: make CFLAGS="-target ${{ matrix.platforms.target }} -isysroot $(xcrun -sdk ${{ matrix.platforms.sdk }} --show-sdk-path) -fembed-bitcode -DNDEBUG=1" loadable static
-      - uses: actions/upload-artifact@v4
-        with:
-          name: sqlite-vec-${{ matrix.platforms.name }}-extension
-          path: dist/*
-  build-wasm32-emscripten:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: mymindstorm/setup-emsdk@v12
-        with:
-          version: "latest"
-      - run: ./scripts/vendor.sh
-      - run: make sqlite-vec.h
-      - run: make wasm
-      - run: ls; ls dist; ls dist/.wasm
-      - uses: actions/upload-artifact@v4
-        with:
-          name: sqlite-vec-wasm32-emscripten
-          path: dist/.wasm/*
-          include-hidden-files: true
-  build-pyodide:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: mymindstorm/setup-emsdk@v12
-        with:
-          version: "latest"
-      - run: ./scripts/vendor.sh
-      - run: make sqlite-vec.h
-      - run: |
-          emcc -c ./sqlite-vec.c -o ./sqlite-vec.o -I ./vendor -fPIC -g3 \
-          -DSTDC_HEADERS=1 -DHAVE_SYS_TYPES_H=1 -DHAVE_SYS_STAT_H=1 -DHAVE_STDLIB_H=1 -DHAVE_STRING_H=1 -DHAVE_MEMORY_H=1 -DHAVE_STRINGS_H=1 -DHAVE_INTTYPES_H=1 -DHAVE_STDINT_H=1 -DHAVE_UNISTD_H=1 -DHAVE_FDATASYNC=1 -DHAVE_USLEEP=1 -DHAVE_LOCALTIME_R=1 -DHAVE_GMTIME_R=1 -DHAVE_DECL_STRERROR_R=1 -DHAVE_STRERROR_R=1 -DHAVE_POSIX_FALLOCATE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_OMIT_POPEN=1 -DSQLITE_THREADSAFE=0
-          emcc ./sqlite-vec.o -o vec0.so -s SIDE_MODULE=1 -g3 -s WASM_BIGINT=1
-      - run: ls
-      - uses: actions/upload-artifact@v4
-        with:
-          name: sqlite-vec-pyodide
-          path: vec0.so
-  build-ncruces-go:
-    if: false
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: make sqlite-vec.h
-      - uses: actions/checkout@v4
-        with:
-          repository: ncruces/go-sqlite3
-          path: go-sqlite3
-      - run: git apply ../bindings/go/ncruces/go-sqlite3.patch
-        working-directory: go-sqlite3/
-      - run: |
-          mkdir -p tools/
-          [ -d "tools/wasi-sdk" ] || curl -#L "$WASI_SDK" | tar xzC tools &
-          [ -d "tools/binaryen" ] || curl -#L "$BINARYEN" | tar xzC tools &
-          wait
-
-          mv "tools/wasi-sdk"* "tools/wasi-sdk"
-          mv "tools/binaryen"* "tools/binaryen"
-
-          sqlite3/download.sh
-          embed/build.sh
-        env:
-          WASI_SDK: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-23/wasi-sdk-23.0-x86_64-linux.tar.gz"
-          BINARYEN: "https://github.com/WebAssembly/binaryen/releases/download/version_118/binaryen-version_118-x86_64-linux.tar.gz"
-        working-directory: go-sqlite3/
-      - uses: actions/upload-artifact@v4
-        with:
-          name: sqlite-vec-ncruces-go
-          path: go-sqlite3/embed/sqlite3.wasm
-  build-cosmopolitan:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: |
-          mkdir $HOME/cosmo
-          curl -L -o cosmocc-$COSMO_VERSION.zip https://github.com/jart/cosmopolitan/releases/download/$COSMO_VERSION/cosmocc-$COSMO_VERSION.zip
-          unzip cosmocc-$COSMO_VERSION.zip -d $HOME/cosmo
-        env:
-          COSMO_VERSION: "3.5.4"
-      - run: ./scripts/vendor.sh
-      - run: make cli CC=$HOME/cosmo/bin/cosmocc AR=$HOME/cosmo/bin/cosmoar OMIT_SIMD=1
-      - uses: actions/upload-artifact@v4
-        with:
-          name: sqlite-vec-cosmopolitan
-          path: dist/*
   test-minimum:
     runs-on: macos-latest
     steps:

CHANGELOG-paperless.md

@@ -0,0 +1,45 @@
+# paperless integration line changelog
+
+Changes carried by the `paperless` branch of `stumpylog/sqlite-vec` beyond upstream `asg017/sqlite-vec`. Each release is an annotated tag `v0.1.10-paperless.N`, built into the wheel `0.1.10+paperless.N` by `paperless-ngx/builder`. Drop an entry from the carried set when upstream merges it.
+
+## v0.1.10-paperless.2 (unreleased)
+
+Performance for filtered KNN (one-document chat queries no longer read the whole index), plus delete and scan improvements.
+
+Merged from upstream, as separate merge commits:
+
+- [#330](https://github.com/asg017/sqlite-vec/pull/330) Skip reading a chunk's vectors when the KNN filters keep none of its rows (the key fix for paperless-ngx#14600)
+- [#327](https://github.com/asg017/sqlite-vec/pull/327) Fix `rowid IN (...)` on KNN queries when rowids are far apart
+- [#328](https://github.com/asg017/sqlite-vec/pull/328) Binary-search an integer metadata column's `IN (...)` list
+- [#329](https://github.com/asg017/sqlite-vec/pull/329) Make KNN filters keep the rows SQLite keeps
+- [#364](https://github.com/asg017/sqlite-vec/pull/364) AVX cosine distance for float vectors
+- [#337](https://github.com/asg017/sqlite-vec/pull/337) Select each chunk's k nearest rows with the `min_idx()` heap
+- [#358](https://github.com/asg017/sqlite-vec/pull/358) Check only the deleted row's validity bit on DELETE
+- [#359](https://github.com/asg017/sqlite-vec/pull/359) Check a DELETE's chunk for emptiness through its validity handle
+- [#336](https://github.com/asg017/sqlite-vec/pull/336) Read a full scan's vectors through one blob handle per chunk (test-file conflict with the parser-guard test resolved by keeping both)
+
+Fork-only:
+
+- Runtime-dispatch #364's AVX cosine kernel: `target("avx,avx2")` on the kernel and `__builtin_cpu_supports("avx2")` at the call site. As merged it was unguarded, which is the pre-AVX SIGILL class fixed by #307.
+
+Considered and not carried: [#338](https://github.com/asg017/sqlite-vec/pull/338) (cosine sums in double, nobody has hit it), [#316](https://github.com/asg017/sqlite-vec/pull/316) (conflicts with #309 and #329), [#339](https://github.com/asg017/sqlite-vec/pull/339) (NEON, Makefile conflict), [#243](https://github.com/asg017/sqlite-vec/pull/243), [#269](https://github.com/asg017/sqlite-vec/pull/269).
+
+## v0.1.10-paperless.1 plus two commits on `paperless` (included in .2)
+
+- [#319](https://github.com/asg017/sqlite-vec/pull/319) vec0 parser token-type guards: `&&` to `||` (20 sites)
+
+## v0.1.10-paperless.1
+
+Base: upstream `v0.1.10-alpha.4`.
+
+- [#303](https://github.com/asg017/sqlite-vec/pull/303) Locale-dependent JSON float parsing (fixes #241)
+- [#304](https://github.com/asg017/sqlite-vec/pull/304) Link ordering, link libs scoped per target
+- [#305](https://github.com/asg017/sqlite-vec/pull/305) `test-unit.c` compiles and links, plus `test-unit` passing
+- [#306](https://github.com/asg017/sqlite-vec/pull/306) Vendored SQLite amalgamation 3.53.2
+- [#307](https://github.com/asg017/sqlite-vec/pull/307) AVX/AVX2 dispatch: target attributes and runtime CPU detection (upstream issue #302)
+- [#309](https://github.com/asg017/sqlite-vec/pull/309) Distance constraints in rescore KNN queries (fixes #308)
+- [#310](https://github.com/asg017/sqlite-vec/pull/310) Reject malformed vector column types
+- [#311](https://github.com/asg017/sqlite-vec/pull/311) Clamp DiskANN int8 quantization
+- [#318](https://github.com/asg017/sqlite-vec/pull/318) OOB read on the metadata `IN` path
+
+Fork-only: Makefile `EXTRA_CFLAGS` hook; CI gating for `paperless`, PRs and tags; ASan/UBSan; nightly valgrind and CodeQL.

sqlite-vec-rescore.c

@@ -393,12 +393,11 @@ static int rescore_knn(vec0_vtab *p, vec0_cursor *pCur,
   f32 *chunk_distances = sqlite3_malloc(p->chunk_size * sizeof(f32));
   i32 *chunk_topk_idxs = sqlite3_malloc(k_oversample * sizeof(i32));
   u8 *b = sqlite3_malloc(p->chunk_size / CHAR_BIT);
-  u8 *bTaken = sqlite3_malloc(p->chunk_size / CHAR_BIT);
   u8 *bmRowids = NULL;
   void *baseVectors = sqlite3_malloc((i64)p->chunk_size * (i64)qsize);
 
   if (!cand_rowids || !cand_distances || !tmp_rowids || !tmp_distances ||
-      !chunk_distances || !chunk_topk_idxs || !b || !bTaken || !baseVectors) {
+      !chunk_distances || !chunk_topk_idxs || !b || !baseVectors) {
     rc = SQLITE_NOMEM;
     goto cleanup;
   }
@@ -503,7 +502,7 @@ static int rescore_knn(vec0_vtab *p, vec0_cursor *pCur,
 
     int used1;
     min_idx(chunk_distances, p->chunk_size, b, chunk_topk_idxs,
-            min(k_oversample, p->chunk_size), bTaken, &used1);
+            min(k_oversample, p->chunk_size), &used1);
 
     i64 merged_used;
     merge_sorted_lists(cand_distances, cand_rowids, cand_used, chunk_distances,
@@ -685,7 +684,6 @@ static int rescore_knn(vec0_vtab *p, vec0_cursor *pCur,
   sqlite3_free(chunk_distances);
   sqlite3_free(chunk_topk_idxs);
   sqlite3_free(b);
-  sqlite3_free(bTaken);
   sqlite3_free(bmRowids);
   sqlite3_free(baseVectors);
   return rc;