diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index fc228356e8..f3d29e373e 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -93,8 +93,7 @@ jobs: VCPKG_DEFAULT_TRIPLET: x64-windows-dynamic-release VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg-bincache CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - CIBW_BUILD: "cp39-win_amd64" - CIBW_TEST_REQUIRES: pytest adbc_driver_manager geoarrow-pyarrow geopandas + CIBW_BUILD: "cp310-win_amd64" # Test command exits 0 even on failure so CIBW continues building all wheels; # failures are recorded in .test_failed and checked after upload. # We use Python here to be absolutely sure there are no shell escaping issues. @@ -113,7 +112,7 @@ jobs: - uses: actions/upload-artifact@v7 with: name: release-wheels-windows-x86_64 - path: python/sedonadb/dist/*-cp39-abi3-*.whl + path: python/sedonadb/dist/*-cp310-abi3-*.whl if-no-files-found: error - name: Fail if tests failed @@ -179,7 +178,7 @@ jobs: - uses: actions/upload-artifact@v7 with: name: release-wheels-macOS-arm64 - path: python/sedonadb/dist/*-cp39-abi3-*.whl + path: python/sedonadb/dist/*-cp310-abi3-*.whl if-no-files-found: error - name: Fail if tests failed @@ -246,7 +245,7 @@ jobs: - uses: actions/upload-artifact@v7 with: name: release-wheels-macOS-amd64 - path: python/sedonadb/dist/*-cp39-abi3-*.whl + path: python/sedonadb/dist/*-cp310-abi3-*.whl if-no-files-found: error - name: Fail if tests failed @@ -312,7 +311,7 @@ jobs: - uses: actions/upload-artifact@v7 with: name: release-wheels-${{ matrix.config.label }} - path: python/sedonadb/dist/*-cp39-abi3-*.whl + path: python/sedonadb/dist/*-cp310-abi3-*.whl if-no-files-found: error - name: Fail if tests failed diff --git a/.github/workflows/wheels-zarr.yml b/.github/workflows/wheels-zarr.yml index 483d13fc98..cdd02f3621 100644 --- a/.github/workflows/wheels-zarr.yml +++ b/.github/workflows/wheels-zarr.yml @@ -58,7 +58,7 @@ jobs: cd ci/scripts .\wheels-build-zarr-windows.ps1 env: - CIBW_BUILD: "cp39-win_amd64" + CIBW_BUILD: "cp310-win_amd64" # Install the sedonadb dev wheel downloaded into the package dir # above: first sedonadb itself (--no-deps --pre, since the dev # version is a prerelease), then its dependencies without --pre so diff --git a/ci/scripts/wheels-build-linux.sh b/ci/scripts/wheels-build-linux.sh index 8bda4e9d27..3cc7162201 100755 --- a/ci/scripts/wheels-build-linux.sh +++ b/ci/scripts/wheels-build-linux.sh @@ -53,7 +53,7 @@ BEFORE_ALL_MANYLINUX="yum install -y curl zip unzip tar clang perl" # add quite a bit of complexity but could save time if we build wheels for linux frequently. # The native and Rust builds are cached on each image such that compile work is effectively # cached between Python versions (just not between invocations of this script). -export CIBW_ENVIRONMENT_LINUX="VCPKG_ROOT=/vcpkg VCPKG_REF=$VCPKG_REF VCPKG_DEFAULT_TRIPLET=$VCPKG_DEFAULT_TRIPLET CMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake PKG_CONFIG_PATH=/vcpkg/installed/$VCPKG_DEFAULT_TRIPLET/lib/pkgconfig LD_LIBRARY_PATH=/vcpkg/installed/$VCPKG_DEFAULT_TRIPLET/lib $SEDONADB_RELEASE_PROFILE_ENV MATURIN_PEP517_ARGS='--features s2geography,pyo3/extension-module,pyo3/abi3-py39'" +export CIBW_ENVIRONMENT_LINUX="VCPKG_ROOT=/vcpkg VCPKG_REF=$VCPKG_REF VCPKG_DEFAULT_TRIPLET=$VCPKG_DEFAULT_TRIPLET CMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake PKG_CONFIG_PATH=/vcpkg/installed/$VCPKG_DEFAULT_TRIPLET/lib/pkgconfig LD_LIBRARY_PATH=/vcpkg/installed/$VCPKG_DEFAULT_TRIPLET/lib $SEDONADB_RELEASE_PROFILE_ENV MATURIN_PEP517_ARGS='--features s2geography,pyo3/extension-module,pyo3/abi3-py310'" # Host directories to bind-mount into the build container. cibuildwheel copies # the project into the container rather than bind-mounting it, so anything the diff --git a/ci/scripts/wheels-build-macos.sh b/ci/scripts/wheels-build-macos.sh index bd73e8c4a7..56a7bb0a91 100755 --- a/ci/scripts/wheels-build-macos.sh +++ b/ci/scripts/wheels-build-macos.sh @@ -47,7 +47,7 @@ source ./wheels-bootstrap-vcpkg.sh export CIBW_REPAIR_WHEEL_COMMAND_MACOS="DYLD_LIBRARY_PATH=$VCPKG_INSTALL_NAME_DIR delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$VCPKG_INSTALL_NAME_DIR delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" # Pass on environment variables specifically for the build -export CIBW_ENVIRONMENT_MACOS="$CIBW_ENVIRONMENT_MACOS _PYTHON_HOST_PLATFORM=macosx-12.0-${SEDONADB_MACOS_ARCH} MACOSX_DEPLOYMENT_TARGET=12.0 CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} MATURIN_PEP517_ARGS='--features s2geography,pyo3/extension-module,pyo3/abi3-py39'" +export CIBW_ENVIRONMENT_MACOS="$CIBW_ENVIRONMENT_MACOS _PYTHON_HOST_PLATFORM=macosx-12.0-${SEDONADB_MACOS_ARCH} MACOSX_DEPLOYMENT_TARGET=12.0 CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} MATURIN_PEP517_ARGS='--features s2geography,pyo3/extension-module,pyo3/abi3-py310'" pushd "${SEDONADB_DIR}" python -m cibuildwheel --output-dir python/$1/dist python/$1 diff --git a/ci/scripts/wheels-build-windows.ps1 b/ci/scripts/wheels-build-windows.ps1 index bee971998c..0d238db6a6 100644 --- a/ci/scripts/wheels-build-windows.ps1 +++ b/ci/scripts/wheels-build-windows.ps1 @@ -73,7 +73,7 @@ $env:PATH += ";$scriptDirectory\windows" $env:GEOS_LIB_DIR = "$vcpkgLibDirectory" $env:GEOS_VERSION = "3.13.0" -$env:MATURIN_PEP517_ARGS='--features s2geography,pyo3/extension-module,pyo3/abi3-py39' +$env:MATURIN_PEP517_ARGS='--features s2geography,pyo3/extension-module,pyo3/abi3-py310' # Some CMake configurations needs this separately from the toolchain file $env:CMAKE_PREFIX_PATH="$vcpkgInstalledDirectory" diff --git a/docs/postgis.ipynb b/docs/postgis.ipynb index db46117dd9..d7f2882c9e 100644 --- a/docs/postgis.ipynb +++ b/docs/postgis.ipynb @@ -50,7 +50,7 @@ "This notebook assumes:\n", "\n", "- A running PostgreSQL instance with PostGIS enabled\n", - "- Python 3.9+\n", + "- Python 3.10+\n", "- The following Python packages available:\n", " - `geopandas`\n", " - `sqlalchemy`\n", diff --git a/docs/postgis.md b/docs/postgis.md index e60ce60f19..d6c808a338 100644 --- a/docs/postgis.md +++ b/docs/postgis.md @@ -33,7 +33,7 @@ Two approaches are covered: This notebook assumes: - A running PostgreSQL instance with PostGIS enabled -- Python 3.9+ +- Python 3.10+ - The following Python packages available: - `geopandas` - `sqlalchemy` diff --git a/python/sedonadb-zarr/pyproject.toml b/python/sedonadb-zarr/pyproject.toml index 2def92ec29..2d38c241fb 100644 --- a/python/sedonadb-zarr/pyproject.toml +++ b/python/sedonadb-zarr/pyproject.toml @@ -48,7 +48,8 @@ test = [ [tool.cibuildwheel] # PyO3's stable ABI support makes this one wheel per platform compatible with # every supported non-free-threaded CPython version. -build = "cp39-*" +# Test against the current sedonadb wheel, which requires Python 3.10. +build = "cp310-*" [tool.maturin] features = ["pyo3/extension-module", "pyo3/abi3-py39"] diff --git a/python/sedonadb/pyproject.toml b/python/sedonadb/pyproject.toml index ce10679a1c..1014b99c12 100644 --- a/python/sedonadb/pyproject.toml +++ b/python/sedonadb/pyproject.toml @@ -22,7 +22,7 @@ build-backend = "maturin" [project] name = "sedonadb" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", @@ -42,6 +42,8 @@ test = [ "geopandas", "pandas", "polars", + # Raster export tests need the inline binary view fix in Arrow 25. + "pyarrow>=25.0.0", "pytest", "pyyaml", "rasterio", @@ -57,13 +59,13 @@ geopandas = [ [tool.cibuildwheel] # PyO3's stable ABI support makes this one wheel per platform compatible with # every supported CPython version, rather than producing one wheel per version. -build = "cp39-*" +build = "cp310-*" # Set here rather than via CIBW_TEST_REQUIRES in the workflow: environment # variables take precedence over the overrides below. -test-requires = ["pytest", "adbc_driver_manager", "geoarrow-pyarrow", "geopandas", "duckdb"] +test-requires = ["pyarrow>=25.0.0", "pytest", "adbc_driver_manager", "geoarrow-pyarrow", "geopandas", "duckdb"] [tool.maturin] -features = ["pyo3/extension-module", "pyo3/abi3-py39"] +features = ["pyo3/extension-module", "pyo3/abi3-py310"] module-name = "sedonadb._lib" python-source = "python" strip = true diff --git a/python/sedonadb/python/sedonadb/raster.py b/python/sedonadb/python/sedonadb/raster.py index 7db719dfc7..713e9fc36e 100644 --- a/python/sedonadb/python/sedonadb/raster.py +++ b/python/sedonadb/python/sedonadb/raster.py @@ -106,7 +106,8 @@ def from_numpy( The mirror of `lazy`: identical dimension/CRS conventions, but the pixel data is carried in the raster rather than referenced by URI. - This operation is zero-copy for contiguous arrays. + Contiguous arrays with more than 12 bytes share their pixel buffer. + Smaller arrays are copied into Arrow's inline binary view storage. Args: array: The pixel data. The trailing two axes are the spatial `(y, x)` @@ -709,8 +710,8 @@ def _get_binary_view_buffer( def _wrap_data_zero_copy(data) -> pa.BinaryViewArray: """Wrap data as a single-element BinaryViewArray without copying. - For numpy arrays and memoryviews, this creates a zero-copy view. - For bytes, pyarrow handles the wrapping efficiently. + For contiguous buffers larger than 12 bytes, this creates a zero-copy + view. Smaller buffers are copied into the binary view's inline storage. """ import numpy as np diff --git a/python/sedonadb/src/raster_loader.rs b/python/sedonadb/src/raster_loader.rs index cff283c0fc..cf51cdd134 100644 --- a/python/sedonadb/src/raster_loader.rs +++ b/python/sedonadb/src/raster_loader.rs @@ -134,7 +134,7 @@ impl AsyncRasterLoader for PyRasterLoader { // The buffer protocol did not join Python's Limited API // until 3.11. Convert through the built-in bytes type so - // this extension can use abi3 while retaining Python 3.9 + // this extension can use abi3 while retaining Python 3.10 // support. `Bytes` takes ownership of the resulting Vec, // so this performs one copy at the Python/Rust boundary. let py_bytes = py diff --git a/python/sedonadb/tests/test_raster.py b/python/sedonadb/tests/test_raster.py index 9befc2a660..5f2a92f7ae 100644 --- a/python/sedonadb/tests/test_raster.py +++ b/python/sedonadb/tests/test_raster.py @@ -382,6 +382,24 @@ def test_raster_from_numpy_2d(): assert np.shares_memory(b.to_numpy(), arr) +@pytest.mark.parametrize("nbytes", [0, 1, 6, 12, 13, 1024]) +@pytest.mark.parametrize("strided", [False, True]) +def test_raster_from_numpy_arrow_roundtrip(nbytes, strided): + source = np.arange(nbytes * (2 if strided else 1), dtype=np.uint8) + pixels = (source[::2] if strided else source).reshape(1, nbytes) + raster = Raster.from_numpy(pixels) + + # Exporting an inline binary view cast from binary crashes older PyArrow. + # Exercise the C Data protocol, not just Arrow's array validation. + schema, array = raster.__arrow_c_array__() + result = pa.Array._import_from_c_capsule(schema, array) + result.validate(full=True) + restored = Raster(result).to_numpy()[0] + np.testing.assert_array_equal(restored, pixels) + if nbytes > 12: + assert np.shares_memory(restored, pixels) == (not strided) + + def test_raster_from_numpy_nd_with_crs(): arr = np.arange(2 * 2 * 3, dtype="float32").reshape(2, 2, 3) r = Raster.from_numpy(arr, dim_names=["time", "y", "x"], crs="EPSG:4326")