Python bindings using scikit build core

.github/workflows/clang-tidy-review.yml

@@ -18,6 +18,7 @@ jobs:
       with:
         split_workflow: true
         build_dir: build
+        exclude: 'python-pybind/*'
         apt_packages: "libsnappy-dev, libzzip-dev, zlib1g-dev, libboost-all-dev, libzstd-dev, libboost-process1.88-dev"
         clang_tidy_checks: ''
         cmake_command: "cmake -Bbuild -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ."

.github/workflows/python-cibuildwheel-pybind.yml

@@ -0,0 +1,155 @@
+name: Python Pybind cibuildwheel
+
+on:
+  push:
+    branches: [ master, release-* ]
+    paths-ignore:
+      - '.github/workflows/docs.yml'
+      - '.github/workflows/keyvi.yml'
+      - '.github/workflows/python-dockerimages-**.yml'
+      - '.github/workflows/rust**.yml'
+      - 'docker/**'
+      - 'doc/**'
+      - 'rust/**'
+  pull_request:
+    branches: [ master ]
+    paths-ignore:
+      - '.github/workflows/docs.yml'
+      - '.github/workflows/keyvi.yml'
+      - '.github/workflows/python-dockerimages-**.yml'
+      - '.github/workflows/rust**.yml'
+      - 'docker/**'
+      - 'doc/**'
+      - 'rust/**'
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: pybind-cibuildwheel ${{ matrix.os }}/${{ matrix.flavor }}/${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # macos-15: Arm64
+        os: [ubuntu-22.04, ubuntu-24.04-arm, macos-15]
+        # skip pypy, https://github.com/pypa/distutils/issues/283
+        flavor: ['cpython']
+        # separate musl and many on linux, for mac we just skip one of those
+        target: [ 'many', 'musl' ]
+        exclude:
+          - os: macos-15
+            target: musl
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2.24
+        with:
+          key: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.flavor }}-python-pybind
+
+      - name: Setup musllinux build
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }}
+        # workaround: set CXX to g++, so it does not pick clang++ for python 3.14 builds
+        run: |
+          echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV
+          echo "CIBW_ENVIRONMENT_LINUX=CXX=g++" >> $GITHUB_ENV
+
+      - name: Setup manylinux build
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }}
+        run: |
+          echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV
+
+      - name: Skip pypy for cpython
+        if: ${{ matrix.flavor == 'cpython' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} pp*" >> $GITHUB_ENV
+
+      - name: Skip cpython for pypy
+        if: ${{ matrix.flavor == 'pypy' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} cp*" >> $GITHUB_ENV
+
+      - name: install mac dependencies
+        if: ${{ runner.os == 'macOS' }}
+        # 2nd command: workaround https://github.com/actions/setup-python/issues/577
+        run: |
+          brew update && \
+          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \
+          brew install ccache zlib snappy boost
+
+      - name: set mac deployment target arm64
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        run: |
+          echo "MACOSX_DEPLOYMENT_TARGET=15.0" >> $GITHUB_ENV
+
+      - name: Build python wheels for ${{ matrix.os }}
+        uses: pypa/cibuildwheel@v4.2.1
+        env:
+          # Skip CPython 3.15
+          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp315*
+
+          # only build native packages
+          CIBW_ARCHS: native
+
+          # skip tests on pypy, currently fails for indexer tests
+          CIBW_TEST_SKIP: "pp*"
+
+          # prefix ccache in path
+          CIBW_ENVIRONMENT_MACOS: ${{ env.CIBW_ENVIRONMENT_MACOS }} PATH=/usr/local/opt/ccache/libexec:/opt/homebrew/opt/ccache/libexec:$PATH
+          CIBW_ENVIRONMENT_LINUX: ${{ env.CIBW_ENVIRONMENT_LINUX }} PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf
+
+          # for debugging set this to 1,2 or 3
+          # CIBW_BUILD_VERBOSITY: 2
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: artifact-${{ matrix.os }}-${{ matrix.flavor }}-${{ matrix.target }}
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: sdist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v7
+    - name: install Linux deps
+      run: |
+        sudo apt-get update && \
+        sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev ccache
+    - name: ccache
+      uses: hendrikmuhs/ccache-action@v1.2.24
+      with:
+        key: ubuntu-sdist-python-pybind
+
+    - name: Build SDist
+      run: |
+        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+        pipx run build --sdist
+        python -m pip install dist/*.tar.gz -v && \
+        python -m pip install pytest python-snappy zstd && \
+        KEYVI_SKIP_TEST_DEPRECATIONS=1 python -m pytest python/tests && \
+        python -m pip uninstall -y keyvi
+
+    - uses: actions/upload-artifact@v7
+      with:
+        name: artifact-sdist
+        path: dist/*.tar.gz
+
+  upload_all:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release'
+    environment:
+      name: pypi
+      url: https://pypi.org/p/keyvi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: artifact-*
+          merge-multiple: true
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/python-cibuildwheel.yml

@@ -118,6 +118,7 @@ jobs:
           # for debugging set this to 1,2 or 3
           # CIBW_BUILD_VERBOSITY: 2
         with:
+          config-file: "{package}/pyproject.toml"
           package-dir: python
 
       - uses: actions/upload-artifact@v7

.gitignore

@@ -31,7 +31,7 @@
 *.orig
 
 # cmake build dir
-build/*
+/*build*
 */cmake-build-debug/*
 build_dir_debug/
 cmake-build-debug/
@@ -45,3 +45,12 @@ cmake-build-debug/
 
 # vim swap files
 *.swp
+
+# python
+*.egg-info
+.venv
+
+# pybind build folder
+python*/*build*
+python*/dist
+python*/.cache/

CMakeLists.txt

@@ -9,11 +9,13 @@ endif()
 #### Options
 
 option(KEYVI_C_BINDINGS "Keyvi: Build C binding" ${PROJECT_IS_TOP_LEVEL})
+option(KEYVI_PYTHON_BINDINGS "Keyvi: Build Python module" OFF)
 option(KEYVI_TESTS "Keyvi: Build unit tests" ${PROJECT_IS_TOP_LEVEL})
 option(KEYVI_BINARIES "Keyvi: Build keyvi binaries" ${PROJECT_IS_TOP_LEVEL})
 option(KEYVI_CLANG_TIDY "Keyvi: Build with clang tidy" OFF)
 option(KEYVI_DOCS "Keyvi: Build docs" ${PROJECT_IS_TOP_LEVEL})
 
+
 #### Linting
 if(KEYVI_CLANG_TIDY)
   find_program(CLANGTIDY clang-tidy)
@@ -275,6 +277,10 @@ target_include_directories(keyvi INTERFACE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>
 target_compile_definitions(keyvi INTERFACE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
 target_link_libraries(keyvi INTERFACE ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${ZSTD_LIBRARIES} ${_OS_LIBRARIES})
 
+if (KEYVI_PYTHON_BINDINGS)
+    add_subdirectory(python-pybind)
+endif ()
+
 ### docs
 
 # don't run it as part of a non-toplevel build, e.g. python

keyvi/include/keyvi/dictionary/completion/prefix_completion.h

@@ -27,6 +27,7 @@
 
 #include <algorithm>
 #include <cstddef>
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <utility>
@@ -129,8 +130,10 @@ class PrefixCompletion final {
     return MatchIterator::EmptyIteratorPair();
   }
 
-  MatchIterator::MatchIteratorPair GetFuzzyCompletions(const std::string& query, const int32_t max_edit_distance,
-                                                       const size_t minimum_exact_prefix = 2) {
+  // NOLINTBEGIN(readability-function-cognitive-complexity,bugprone-easily-swappable-parameters)
+  [[nodiscard]] MatchIterator::MatchIteratorPair GetFuzzyCompletions(const std::string& query,
+                                                                     const int32_t max_edit_distance,
+                                                                     const size_t minimum_exact_prefix = 2) const {
     uint64_t state = fsa_->GetStartState();
     size_t depth = 0;
     std::vector<uint32_t> codepoints;
@@ -223,6 +226,7 @@ class PrefixCompletion final {
 
     return MatchIterator::MakeIteratorPair(tfunc, std::move(first_match));
   }
+  // NOLINTEND(readability-function-cognitive-complexity,bugprone-easily-swappable-parameters)
 
  private:
   fsa::automata_t fsa_;

pyproject.toml

@@ -0,0 +1,96 @@
+[build-system]
+requires = ["scikit-build-core>=0.10", "pybind11"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "keyvi"
+# 0.9.6 corresponds to 0.7.6 (Cython), using 0.9.x for the pybind11 beta series
+version = "0.9.6b1"
+dependencies = [
+    "msgpack>=1.0.0",
+]
+
+[dependency-groups]
+lint = [
+    'ruff>=0.5.0',
+]
+test = [
+    'pytest>=8.3.5',
+    'pytest-env>=1.1.5',
+    'python-snappy>=0.7.3',
+    'zstd>=1.5.7.2'
+]
+dev = [
+    { include-group = "test" },
+    { include-group = "lint" },
+    "tox",
+]
+
+
+[tool.scikit-build]
+wheel.expand-macos-universal-tags = true
+minimum-version = "build-system.requires"
+
+[tool.scikit-build.cmake.define]
+KEYVI_PYTHON_BINDINGS = "ON"
+KEYVI_C_BINDINGS = "OFF"
+KEYVI_TESTS = "OFF"
+KEYVI_BINARIES = "OFF"
+KEYVI_CLANG_TIDY = "OFF"
+KEYVI_DOCS = "OFF"
+
+[tool.pytest.ini_options]
+minversion = "7.1.1"
+addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"]
+xfail_strict = true
+log_cli_level = "INFO"
+filterwarnings = [
+  "error",
+  "ignore::pytest.PytestCacheWarning",
+  # zstd C extension lacks Py_mod_gil declaration, triggers RuntimeWarning on Python 3.14t free-threaded builds
+  "ignore:The global interpreter lock.*:RuntimeWarning",
+]
+testpaths = ["python/tests"]
+
+[tool.cibuildwheel]
+build-frontend = "default"
+test-groups = ["test"]
+test-command = "pytest {project}/python/tests"
+manylinux-x86_64-image = "keyvidev/manylinux-builder-x86_64"
+musllinux-x86_64-image = "keyvidev/musllinux-builder-x86_64"
+manylinux-aarch64-image = "keyvidev/manylinux-builder-aarch64"
+musllinux-aarch64-image = "keyvidev/musllinux-builder-aarch64"
+test-environment = { KEYVI_SKIP_TEST_DEPRECATIONS="1"}
+
+[tool.ruff.lint]
+extend-select = [
+  "B",           # flake8-bugbear
+  "I",           # isort
+  "ARG",         # flake8-unused-arguments
+  "C4",          # flake8-comprehensions
+  "EM",          # flake8-errmsg
+  "ICN",         # flake8-import-conventions
+  "G",           # flake8-logging-format
+  "PGH",         # pygrep-hooks
+  "PIE",         # flake8-pie
+  "PL",          # pylint
+  "PT",          # flake8-pytest-style
+  "PTH",         # flake8-use-pathlib
+  "RET",         # flake8-return
+  "RUF",         # Ruff-specific
+  "SIM",         # flake8-simplify
+  "T20",         # flake8-print
+  "UP",          # pyupgrade
+  "YTT",         # flake8-2020
+  "EXE",         # flake8-executable
+  "NPY",         # NumPy specific rules
+  "PD",          # pandas-vet
+]
+ignore = [
+  "PLR09",    # Too many X
+  "PLR2004",  # Magic comparison
+]
+isort.required-imports = ["from __future__ import annotations"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["T20"]

python-pybind/CMakeLists.txt

@@ -0,0 +1,28 @@
+cmake_minimum_required(VERSION 3.15...3.27)
+
+# Scikit-build-core sets these values for you, or you can just hard-code the
+# name and version.
+project(
+  ${SKBUILD_PROJECT_NAME}
+  VERSION ${SKBUILD_PROJECT_VERSION}
+  LANGUAGES CXX)
+
+# Find the module development requirements (requires FindPython from 3.17 or
+# scikit-build-core's built-in backport)
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
+find_package(pybind11 CONFIG REQUIRED)
+
+# Use _keyvi_pybind as internal target name to avoid conflict with the keyvi C++ library target
+FILE(GLOB_RECURSE KEYVI_PYBIND_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} src/*.cpp)
+pybind11_add_module(_keyvi_pybind ${KEYVI_PYBIND_SOURCES})
+
+set_target_properties(_keyvi_pybind PROPERTIES OUTPUT_NAME keyvi)
+target_include_directories(_keyvi_pybind PRIVATE src)
+target_link_libraries(_keyvi_pybind PRIVATE keyvi)
+
+# This is passing in the version as a define just as an example
+target_compile_definitions(_keyvi_pybind PRIVATE VERSION_INFO=${PROJECT_VERSION})
+
+# The install directory is the output (wheel) directory
+install(TARGETS _keyvi_pybind DESTINATION .)
+install(FILES keyvimerger.py DESTINATION .)