From 37c9a659886fcabc4150916f8227d0cb8cc12d9e Mon Sep 17 00:00:00 2001 From: Conor worthington <45695214+ConorWorthington@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:38:18 +0000 Subject: [PATCH] feat: support Spark 4 and numpy 2 Relax dependency bounds to pyspark>=3.4.0,<5.0.0 and numpy>=1.22.0 (no upper bound), fixing installs on Databricks runtimes that ship numpy 2. Spark 4 enables ANSI mode by default, under which division/modulo by zero and out of bounds element_at raise instead of returning null. Guard these explicitly so results match Spark 3 and the Keras layers: - Divide, CosineSimilarity, PairwiseCosineSimilarity: zero divisor -> 0.0 - Modulo: zero divisor -> null - SubStringDelimAtIndex: out of bounds index -> defaultValue - ListMedian: empty list -> nanFillValue Tests: - Add test_ansi_mode.py covering the above with ANSI on and off. - Fix test helper scalar conversion for numpy 2.5. - Patch the concrete DataFrame class in the checkpoint test, since pyspark.sql.DataFrame is abstract in Spark 4. - Check JIT-incompatible layers via experimental_get_compiler_ir, avoiding a TF 2.21 abort in the XLA compilation cache on repeated failed compiles of string ops. CI: add pyspark 4.0.1 lane, set up Java 17, keep a numpy 1.26 lane. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yaml | 16 ++- pyproject.toml | 4 +- .../spark/transformers/cosine_similarity.py | 15 ++- src/kamae/spark/transformers/divide.py | 6 +- src/kamae/spark/transformers/list_median.py | 9 +- src/kamae/spark/transformers/modulo.py | 20 ++-- .../pairwise_cosine_similarity.py | 7 +- .../transformers/sub_string_delim_at_index.py | 21 +++- tests/kamae/keras/test_jit_compatibility.py | 10 +- tests/kamae/spark/pipeline/test_pipeline.py | 8 +- tests/kamae/spark/test_helpers.py | 6 +- .../spark/transformers/test_ansi_mode.py | 106 ++++++++++++++++++ uv.lock | 50 ++++----- 13 files changed, 217 insertions(+), 61 deletions(-) create mode 100644 tests/kamae/spark/transformers/test_ansi_mode.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 81e77296..317a2ebd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,8 +17,14 @@ jobs: strategy: matrix: python-version: ["3.10", "3.11", "3.12"] - pyspark-version: ["3.4.1", "3.5.0"] + pyspark-version: ["3.4.1", "3.5.0", "4.0.1"] keras-version: ["3.3.0", "3.7.0", "3.10.0", "3.12.0"] + include: + # numpy is otherwise resolved to 2.x, so keep coverage of numpy 1.x. + - python-version: "3.10" + pyspark-version: "3.5.0" + keras-version: "3.10.0" + numpy-version: "1.26.4" steps: - name: Setup Local Repo uses: actions/checkout@v4 @@ -26,6 +32,12 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Setup Java + # Spark 4 requires Java 17+, which Spark 3.4 and 3.5 also support. + uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: "17" - name: Install uv run: | pip install --upgrade pip @@ -34,7 +46,7 @@ jobs: run: | uv venv --python ${{ matrix.python-version }} uv pip install -e ".[tensorflow]" - uv pip install pyspark==${{ matrix.pyspark-version }} keras==${{ matrix.keras-version }} + uv pip install pyspark==${{ matrix.pyspark-version }} keras==${{ matrix.keras-version }} ${{ matrix.numpy-version && format('numpy=={0}', matrix.numpy-version) || '' }} - name: Run tests run: uv run -p ${{ matrix.python-version }} python -m pytest -n auto . formatting: diff --git a/pyproject.toml b/pyproject.toml index 0c175c4e..5186263b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,14 +10,14 @@ license = "Apache-2.0" license-files = ["LICENSE.txt"] requires-python = ">=3.10,<3.13" dependencies = [ - "pyspark>=3.4.0,<4.0.0", + "pyspark>=3.4.0,<5.0.0", "pandas>=1.3.4,<3.0.0", "pyarrow>=4.0.0", "networkx>=2.6.3,<3.0.0", "pyfarmhash>=0.3.2,<0.4.0", "keras>=3.0.0,<4.0.0", "keras-tuner>=1.4.0,<2.0.0", - "numpy>=1.22.0,<2.0.0", + "numpy>=1.22.0", "tensorflow>=2.16.0,<3.0.0", "dill>=0.3.0,<1.0.0", ] diff --git a/src/kamae/spark/transformers/cosine_similarity.py b/src/kamae/spark/transformers/cosine_similarity.py index 9db45583..359b38bb 100644 --- a/src/kamae/spark/transformers/cosine_similarity.py +++ b/src/kamae/spark/transformers/cosine_similarity.py @@ -133,15 +133,20 @@ def norm(x: Column, col_name: str) -> Column: ) # If the norms are zero, then we match the tensorflow behavior and return 0.0. + # The zero check is explicit since with ANSI mode enabled (the default in + # Spark 4) division by zero raises an error rather than returning null. + def cosine_similarity(x: Column) -> Column: + norm_product = norm(x, input_col_names[0]) * norm(x, input_col_names[1]) + return F.coalesce( + F.when(norm_product != F.lit(0.0), dot_product(x) / norm_product), + F.lit(0.0), + ) + output_col = multi_input_single_output_array_transform( input_cols=input_cols, input_col_datatypes=datatypes, input_col_names=input_col_names, - func=lambda x: F.coalesce( - dot_product(x) - / (norm(x, input_col_names[0]) * norm(x, input_col_names[1])), - F.lit(0.0), - ), + func=cosine_similarity, ) return dataset.withColumn(self.getOutputCol(), output_col) diff --git a/src/kamae/spark/transformers/divide.py b/src/kamae/spark/transformers/divide.py index 74791fdc..fd1999d9 100644 --- a/src/kamae/spark/transformers/divide.py +++ b/src/kamae/spark/transformers/divide.py @@ -121,7 +121,11 @@ def divide_no_nan(column1: Column, column2: Column) -> Column: """ Divide two columns, and if the result is NaN, return 0.0 instead """ - return F.coalesce(column1 / column2, F.lit(0.0)) + # Guard the divisor explicitly, since with ANSI mode enabled (the default + # in Spark 4) division by zero raises an error rather than returning null. + return F.coalesce( + F.when(column2 != F.lit(0), column1 / column2), F.lit(0.0) + ) output_col = multi_input_single_output_scalar_transform( input_cols=input_cols, diff --git a/src/kamae/spark/transformers/list_median.py b/src/kamae/spark/transformers/list_median.py index 5d10a86f..fb4bb2c4 100644 --- a/src/kamae/spark/transformers/list_median.py +++ b/src/kamae/spark/transformers/list_median.py @@ -162,12 +162,17 @@ def _transform(self, dataset: DataFrame) -> DataFrame: dataset = dataset.withColumn("array_size", F.size(F.col("sorted_values"))) mid_index_1 = (F.col("array_size") / 2).cast("int") mid_index_2 = (F.col("array_size") / 2 - 1).cast("int") + # Empty arrays give a null median (filled below). This is explicit since with + # ANSI mode enabled (the default in Spark 4) element_at raises an error for + # an out of bounds index rather than returning null. dataset = dataset.withColumn( self.getOutputCol(), - F.when( + F.when(F.col("array_size") <= 0, F.lit(None)) + .when( F.col("array_size") % 2 == 1, F.element_at(F.col("sorted_values"), mid_index_1 + 1), - ).otherwise( + ) + .otherwise( ( F.element_at(F.col("sorted_values"), mid_index_1 + 1) + F.element_at(F.col("sorted_values"), mid_index_2 + 1) diff --git a/src/kamae/spark/transformers/modulo.py b/src/kamae/spark/transformers/modulo.py index 3b105883..406cbf96 100644 --- a/src/kamae/spark/transformers/modulo.py +++ b/src/kamae/spark/transformers/modulo.py @@ -179,14 +179,20 @@ def _transform(self, dataset: DataFrame) -> DataFrame: input_cols=input_cols, input_col_names=input_col_names, input_col_datatypes=input_col_datatypes, + # A zero divisor returns null. This is explicit since with ANSI mode + # enabled (the default in Spark 4) modulo by zero raises an error. func=lambda x: F.when( - # If the result of the modulo operation is negative, - # add the divisor to the result. This is to match the behavior of the - # tensorflow modulo layer. - x[input_col_names[0]] % x[input_col_names[1]] >= 0, - x[input_col_names[0]] % x[input_col_names[1]], - ).otherwise( - (x[input_col_names[0]] % x[input_col_names[1]]) + x[input_col_names[1]] + x[input_col_names[1]] != F.lit(0), + F.when( + # If the result of the modulo operation is negative, + # add the divisor to the result. This is to match the behavior of + # the tensorflow modulo layer. + x[input_col_names[0]] % x[input_col_names[1]] >= 0, + x[input_col_names[0]] % x[input_col_names[1]], + ).otherwise( + (x[input_col_names[0]] % x[input_col_names[1]]) + + x[input_col_names[1]] + ), ), ) return dataset.withColumn(self.getOutputCol(), output_col) diff --git a/src/kamae/spark/transformers/pairwise_cosine_similarity.py b/src/kamae/spark/transformers/pairwise_cosine_similarity.py index f56eecb8..586b8f06 100644 --- a/src/kamae/spark/transformers/pairwise_cosine_similarity.py +++ b/src/kamae/spark/transformers/pairwise_cosine_similarity.py @@ -126,7 +126,12 @@ def cosine_sim_at_index(idx: Column) -> Column: lambda acc, x: acc + (x * x).cast("double"), ) ) - return F.coalesce(dot / (query_norm * cand_norm), F.lit(0.0)) + # Guard against zero norms explicitly, since with ANSI mode enabled (the + # default in Spark 4) division by zero raises rather than returning null. + norm_product = query_norm * cand_norm + return F.coalesce( + F.when(norm_product != F.lit(0.0), dot / norm_product), F.lit(0.0) + ) similarities = F.transform(indices, cosine_sim_at_index) return dataset.withColumn(self.getOutputCol(), similarities) diff --git a/src/kamae/spark/transformers/sub_string_delim_at_index.py b/src/kamae/spark/transformers/sub_string_delim_at_index.py index f8e87204..a2587b0b 100644 --- a/src/kamae/spark/transformers/sub_string_delim_at_index.py +++ b/src/kamae/spark/transformers/sub_string_delim_at_index.py @@ -23,7 +23,7 @@ import tensorflow as tf from pyspark import keyword_only from pyspark.ml.param import Param, Params, TypeConverters -from pyspark.sql import DataFrame +from pyspark.sql import Column, DataFrame from pyspark.sql.types import DataType, StringType from kamae.keras.core.backend import TENSORFLOW_ONLY @@ -198,13 +198,24 @@ def _transform(self, dataset: DataFrame) -> DataFrame: # Since element_at is a 1-based index , we need to add 1 to the index if it # is non-negative. one_based_index = index + 1 if index >= 0 else index + + # The bounds check is explicit since with ANSI mode enabled (the default in + # Spark 4) element_at raises an error for an out of bounds index rather than + # returning null. + def sub_string_at_index(x: Column) -> Column: + split_col = F.split(x, delimiter) + return F.coalesce( + F.when( + F.size(split_col) >= F.lit(abs(one_based_index)), + F.element_at(split_col, one_based_index), + ), + F.lit(default_value), + ) + output_col = single_input_single_output_scalar_transform( input_col=F.col(self.getInputCol()), input_col_datatype=input_datatype, - func=lambda x: F.coalesce( - F.element_at(F.split(x, delimiter), one_based_index), - F.lit(default_value), - ), + func=sub_string_at_index, ) return dataset.withColumn(self.getOutputCol(), output_col) diff --git a/tests/kamae/keras/test_jit_compatibility.py b/tests/kamae/keras/test_jit_compatibility.py index 2ec2d600..e4cd1701 100644 --- a/tests/kamae/keras/test_jit_compatibility.py +++ b/tests/kamae/keras/test_jit_compatibility.py @@ -504,12 +504,12 @@ def jit_call(*inputs): return layer(inputs[0]) return layer(list(inputs)) - # Must raise Exception when trying to JIT compile + # Must raise Exception when trying to JIT compile. We lower to HLO rather than + # executing, since executing goes through the XLA compilation cache, which in + # TF 2.21 aborts the process when hashing string constants after a prior failed + # compile (Check failed: DataTypeCanUseMemcpy(dtype())). with pytest.raises(Exception): - result = jit_call(*input_tensors) - # Force evaluation if result is symbolic - if hasattr(result, "numpy"): - result.numpy() + jit_call.experimental_get_compiler_ir(*input_tensors)(stage="hlo") def test_all_layers_define_jit_compatible_and_supported_backends(): diff --git a/tests/kamae/spark/pipeline/test_pipeline.py b/tests/kamae/spark/pipeline/test_pipeline.py index f910f192..2d78d0d6 100644 --- a/tests/kamae/spark/pipeline/test_pipeline.py +++ b/tests/kamae/spark/pipeline/test_pipeline.py @@ -18,7 +18,6 @@ import pytest import tensorflow as tf -from pyspark.sql import DataFrame from pyspark.sql.types import DoubleType from kamae.spark.estimators import ( @@ -619,10 +618,13 @@ def test_spark_pipeline_checkpoint_invocation( """ checkpoint must be invoked during fit only when checkpointInterval > 0. """ - original_checkpoint = DataFrame.checkpoint + # Patch the concrete class: on Spark 4 pyspark.sql.DataFrame is an abstract + # parent of the classic DataFrame, so patching it would not intercept calls. + dataframe_cls = type(example_dataframe) + original_checkpoint = dataframe_cls.checkpoint with patch.object( - DataFrame, + dataframe_cls, "checkpoint", autospec=True, side_effect=original_checkpoint, diff --git a/tests/kamae/spark/test_helpers.py b/tests/kamae/spark/test_helpers.py index 980d252d..0e748d95 100644 --- a/tests/kamae/spark/test_helpers.py +++ b/tests/kamae/spark/test_helpers.py @@ -17,10 +17,10 @@ def tensor_to_python_type(x): if x.dtype.name == "string": return x.numpy().decode("utf-8") elif x.dtype.name == "bool": - return bool(x.numpy()) + return bool(x.numpy().item()) elif x.dtype.is_floating: - return float(x.numpy()) + return float(x.numpy().item()) elif x.dtype.is_integer: - return int(x.numpy()) + return int(x.numpy().item()) else: raise ValueError(f"Unknown dtype: {x.dtype}") diff --git a/tests/kamae/spark/transformers/test_ansi_mode.py b/tests/kamae/spark/transformers/test_ansi_mode.py new file mode 100644 index 00000000..ad7bb1b8 --- /dev/null +++ b/tests/kamae/spark/transformers/test_ansi_mode.py @@ -0,0 +1,106 @@ +# Copyright [2024] Expedia, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Transformers must give the same results whether or not ANSI mode is enabled, since +it is enabled by default from Spark 4 and raises on division by zero and out of +bounds array access, where non-ANSI mode returns null.""" + +import pytest + +from kamae.spark.transformers import ( + CosineSimilarityTransformer, + DivideTransformer, + ListMedianTransformer, + ModuloTransformer, + PairwiseCosineSimilarityTransformer, + SubStringDelimAtIndexTransformer, +) + + +@pytest.fixture(params=["true", "false"], ids=["ansi", "non_ansi"]) +def ansi_spark_session(request, spark_session): + original = spark_session.conf.get("spark.sql.ansi.enabled") + spark_session.conf.set("spark.sql.ansi.enabled", request.param) + yield spark_session + spark_session.conf.set("spark.sql.ansi.enabled", original) + + +class TestAnsiMode: + def test_divide_by_zero(self, ansi_spark_session): + df = ansi_spark_session.createDataFrame( + [(1.0, 0.0), (0.0, 0.0), (6.0, 2.0)], ["col1", "col2"] + ) + transformer = DivideTransformer(inputCols=["col1", "col2"], outputCol="out") + actual = [r.out for r in transformer.transform(df).collect()] + assert actual == [0.0, 0.0, 3.0] + + def test_modulo_by_zero(self, ansi_spark_session): + df = ansi_spark_session.createDataFrame( + [(10, 0), (-7, 3), (7, 3)], ["col1", "col2"] + ) + transformer = ModuloTransformer(inputCols=["col1", "col2"], outputCol="out") + actual = [r.out for r in transformer.transform(df).collect()] + assert actual == [None, 2, 1] + + def test_cosine_similarity_zero_norm(self, ansi_spark_session): + df = ansi_spark_session.createDataFrame( + [([0.0, 0.0], [1.0, 2.0]), ([1.0, 0.0], [2.0, 0.0])], ["col1", "col2"] + ) + transformer = CosineSimilarityTransformer( + inputCols=["col1", "col2"], outputCol="out" + ) + actual = [r.out for r in transformer.transform(df).collect()] + assert actual == [0.0, 1.0] + + def test_pairwise_cosine_similarity_zero_norm(self, ansi_spark_session): + df = ansi_spark_session.createDataFrame( + [([1.0, 0.0], [0.0, 0.0, 2.0, 0.0])], ["query", "candidates"] + ) + transformer = PairwiseCosineSimilarityTransformer( + inputCols=["query", "candidates"], outputCol="out", embeddingDim=2 + ) + actual = transformer.transform(df).collect()[0].out + assert actual == [0.0, 1.0] + + @pytest.mark.parametrize( + "index, expected", + [(1, ["b", "default"]), (-1, ["c", "a"]), (5, ["default", "default"])], + ) + def test_sub_string_delim_at_index_out_of_bounds( + self, ansi_spark_session, index, expected + ): + df = ansi_spark_session.createDataFrame([("a_b_c",), ("a",)], ["col1"]) + transformer = SubStringDelimAtIndexTransformer( + inputCol="col1", + outputCol="out", + delimiter="_", + index=index, + defaultValue="default", + ) + actual = [r.out for r in transformer.transform(df).collect()] + assert actual == expected + + def test_list_median_all_values_filtered(self, ansi_spark_session): + df = ansi_spark_session.createDataFrame( + [(1, 5.0), (1, 3.0), (2, -1.0), (2, -1.0)], ["search_id", "value"] + ) + transformer = ListMedianTransformer( + inputCol="value", + outputCol="out", + queryIdCol="search_id", + minFilterValue=0.0, + nanFillValue=-99.0, + ) + actual = {(r.search_id, r.out) for r in transformer.transform(df).collect()} + assert actual == {(1, 4.0), (2, -99.0)} diff --git a/uv.lock b/uv.lock index 1159c334..63b3ce64 100644 --- a/uv.lock +++ b/uv.lock @@ -927,11 +927,11 @@ requires-dist = [ { name = "keras", specifier = ">=3.0.0,<4.0.0" }, { name = "keras-tuner", specifier = ">=1.4.0,<2.0.0" }, { name = "networkx", specifier = ">=2.6.3,<3.0.0" }, - { name = "numpy", specifier = ">=1.22.0,<2.0.0" }, + { name = "numpy", specifier = ">=1.22.0" }, { name = "pandas", specifier = ">=1.3.4,<3.0.0" }, { name = "pyarrow", specifier = ">=4.0.0" }, { name = "pyfarmhash", specifier = ">=0.3.2,<0.4.0" }, - { name = "pyspark", specifier = ">=3.4.0,<4.0.0" }, + { name = "pyspark", specifier = ">=3.4.0,<5.0.0" }, { name = "tensorflow", specifier = ">=2.16.0,<3.0.0" }, { name = "torch", marker = "extra == 'torch'", specifier = ">=2.0.0" }, ] @@ -1849,29 +1849,29 @@ wheels = [ name = "pyarrow" version = "25.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3d/e3/27f57f80141379d60defe6703eb50a707325706f07fedfd1312c7a751995/pyarrow-25.0.1.tar.gz", hash = "sha256:9150a83248bfed9813ea3c3af74c3856c1984d444aa28e58bf7733b9750ddf6a", size = 1201653 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/3e/5cd70becb51e1d044c54ba5e627424a6e87df5b98008cbd22cc6abd409ca/pyarrow-25.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0b1edbb2f385a6a65e9711b62ba86ac54a7816a3f8d17bb3e8a5929d65fb2485", size = 35954271 }, - { url = "https://files.pythonhosted.org/packages/64/be/17599e086df264ea7dc221d1101e3131e181e00da428a2f9bd0358f0d06b/pyarrow-25.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:a4dd8bf99a8fac133efc0ed6a92f5fddbe2adba0d0f6dd720e39ba9855cea85c", size = 37647543 }, - { url = "https://files.pythonhosted.org/packages/42/34/e138b451fd3970a6eda4599f68ae3b2b32b661bc958de3239d54a0bf6575/pyarrow-25.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bddd0c4f7630c2a3ddf6347c1bdaa79d97bcf6bd445f9e60c816b7d77c85a5ae", size = 46837120 }, - { url = "https://files.pythonhosted.org/packages/57/5c/f8fc0eb2de03464a557d5a4d0c15e972d73362414696618833b771f7eddd/pyarrow-25.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a4d6d5e9a3d1879a97c08ded0c797579b7965eafd0f0c26c30b45ccc06db939b", size = 50066460 }, - { url = "https://files.pythonhosted.org/packages/3f/d1/0dd64fd06de0333b808a02f60981635f067b71aad3a30698a9a104fae778/pyarrow-25.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:514ddb60285631af068875550c90eddc181db3e8e63a032b1559be189e82f056", size = 49937892 }, - { url = "https://files.pythonhosted.org/packages/cb/3c/f89d1bd76d5f3284c2a44d7d7ebbd8204535e5ae2b41f4077069b4ff2ec6/pyarrow-25.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cab40b1edfef0262e0e5251aa2c58d75630f24d06dd7794480243acc001a1d7d", size = 53107240 }, - { url = "https://files.pythonhosted.org/packages/67/67/b554a8e09f3f3decccf405eb8fbe86696321cbcb5b62d18b4a5057a4c113/pyarrow-25.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:60e89d8f13861a1f7f8d950fa54aebb8023b30734d0ac51ffa80beabe2df4bba", size = 27848683 }, - { url = "https://files.pythonhosted.org/packages/ee/8b/0d23b47702fcfe8b3618d5292035099675c5a1c48258932350c08020f7b5/pyarrow-25.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:51093dd9e10325fbdb3c10a2ae7c4806e5c822d94e74ae4938b26524a3323fee", size = 35946180 }, - { url = "https://files.pythonhosted.org/packages/d8/17/707d17a5476c55a9541fde0db8213ac30979a792864d72415f176ba50c45/pyarrow-25.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:eb6203482ff3746a5632303a7279ae0b5a304c46985b49ed1378cb350ea6728d", size = 37644787 }, - { url = "https://files.pythonhosted.org/packages/c1/b2/cdc98ecf1a6408280bc3a6a07054cdd99a3f4670acc0545d383ce113e87d/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:880523be3d29efcf83d3998835d206118ccf35e3871dbd2fb60408cf6b007a80", size = 46834633 }, - { url = "https://files.pythonhosted.org/packages/c8/6e/d3fafc41f378b2c65be43b827798c0fae42049a641c8526633ed3eb573e2/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:25f8720bf6387d5dc2ebd2622112de630760419e4b66134405dd24110d15f37e", size = 50065507 }, - { url = "https://files.pythonhosted.org/packages/d5/12/8d0698954b8c3001844a898e0a6900bebe83d7ee40c11195174c5122f324/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4facd65742a024a4a366328a1d2292062d72d6e023c1b7dda8d4c37544933a25", size = 49955690 }, - { url = "https://files.pythonhosted.org/packages/d3/0b/1ecb936ac6409e90a34d58eea1c7cec09a9ae6d2141b9e49ad01a2b1ea47/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa0559502e1cd6254d6814614085dd9c5a3dd0419362978a936a3f68a9e5c3df", size = 53128198 }, - { url = "https://files.pythonhosted.org/packages/8e/1c/5236033550633c9b7377b2a53660b2bbb06cb06dc09c4356332d67643ca1/pyarrow-25.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:62cd0d785b8aa6675ee355f9fc02252a340f4441257c42674937826fd7594325", size = 27857263 }, - { url = "https://files.pythonhosted.org/packages/a6/e2/9ab15b88cbfac28e16419ce5439ec29234c5172cb8259301b4ba639bdec0/pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:df961f2e7ae9cf496459259d798652c70625f6c080650d6952f8c04053c58ee9", size = 35861559 }, - { url = "https://files.pythonhosted.org/packages/58/79/a0036dbe1eabe1f73127427342f1d99982584c4a2cde2651d6c93499c6f6/pyarrow-25.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:cc4aa407fde9fc660be3939e49ea31f50f3e9fec17c0ec63159f7711edd3efc9", size = 37628383 }, - { url = "https://files.pythonhosted.org/packages/13/49/d93a57d375f4bf0cf82913dd6bb54acafde83dd993be2282c81ac5616cad/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:4340f0ba6c1d2e13f21658de1d7c662ca2545018568d0030a1e9afca159d87e3", size = 46820190 }, - { url = "https://files.pythonhosted.org/packages/60/c9/711ca85d79f1ec98f29a5eae2b051e25b4ecec5de3e3c0e2d5c5dcb15664/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5389cdf79447ed1515c9e31620e6e1e2302249564d603f2ad727d4f6d313e4c3", size = 50102437 }, - { url = "https://files.pythonhosted.org/packages/80/53/8fb8359ff17cfb6263a1cf3ebf7caec9fe197de118719e84fcb1d0618026/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d51592cb7561e87877c506113e7adbf1342ab579e6c21f0ef44b8ba41cb74c80", size = 49942424 }, - { url = "https://files.pythonhosted.org/packages/e8/83/4e5ae02a9341571b18a6fca380ac7a58ce6ddae7ab3c060208c0a1e79f02/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6109c94d8b9f3b17a041daca16cacb2f651ad8f1ef70a4232c2c0f37a23da2a8", size = 53144206 }, - { url = "https://files.pythonhosted.org/packages/65/ee/197cbf47e49f83e6ebeb946a5259a48a638dea27ac774db42fe78022179d/pyarrow-25.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:8858d7bfc22e3f51529aeaa4077225029724623e4595dc9eff8c793935c34140", size = 27953934 }, +sdist = { url = "https://files.pythonhosted.org/packages/3d/e3/27f57f80141379d60defe6703eb50a707325706f07fedfd1312c7a751995/pyarrow-25.0.1.tar.gz", hash = "sha256:9150a83248bfed9813ea3c3af74c3856c1984d444aa28e58bf7733b9750ddf6a", size = 1201653, upload-time = "2026-08-10T12:40:53.904Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/3e/5cd70becb51e1d044c54ba5e627424a6e87df5b98008cbd22cc6abd409ca/pyarrow-25.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0b1edbb2f385a6a65e9711b62ba86ac54a7816a3f8d17bb3e8a5929d65fb2485", size = 35954271, upload-time = "2026-08-10T12:36:33.857Z" }, + { url = "https://files.pythonhosted.org/packages/64/be/17599e086df264ea7dc221d1101e3131e181e00da428a2f9bd0358f0d06b/pyarrow-25.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:a4dd8bf99a8fac133efc0ed6a92f5fddbe2adba0d0f6dd720e39ba9855cea85c", size = 37647543, upload-time = "2026-08-10T12:36:39.486Z" }, + { url = "https://files.pythonhosted.org/packages/42/34/e138b451fd3970a6eda4599f68ae3b2b32b661bc958de3239d54a0bf6575/pyarrow-25.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bddd0c4f7630c2a3ddf6347c1bdaa79d97bcf6bd445f9e60c816b7d77c85a5ae", size = 46837120, upload-time = "2026-08-10T12:36:46.58Z" }, + { url = "https://files.pythonhosted.org/packages/57/5c/f8fc0eb2de03464a557d5a4d0c15e972d73362414696618833b771f7eddd/pyarrow-25.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a4d6d5e9a3d1879a97c08ded0c797579b7965eafd0f0c26c30b45ccc06db939b", size = 50066460, upload-time = "2026-08-10T12:36:53.702Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d1/0dd64fd06de0333b808a02f60981635f067b71aad3a30698a9a104fae778/pyarrow-25.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:514ddb60285631af068875550c90eddc181db3e8e63a032b1559be189e82f056", size = 49937892, upload-time = "2026-08-10T12:37:00.349Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3c/f89d1bd76d5f3284c2a44d7d7ebbd8204535e5ae2b41f4077069b4ff2ec6/pyarrow-25.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cab40b1edfef0262e0e5251aa2c58d75630f24d06dd7794480243acc001a1d7d", size = 53107240, upload-time = "2026-08-10T12:37:07.205Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/b554a8e09f3f3decccf405eb8fbe86696321cbcb5b62d18b4a5057a4c113/pyarrow-25.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:60e89d8f13861a1f7f8d950fa54aebb8023b30734d0ac51ffa80beabe2df4bba", size = 27848683, upload-time = "2026-08-10T12:37:12.058Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8b/0d23b47702fcfe8b3618d5292035099675c5a1c48258932350c08020f7b5/pyarrow-25.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:51093dd9e10325fbdb3c10a2ae7c4806e5c822d94e74ae4938b26524a3323fee", size = 35946180, upload-time = "2026-08-10T12:37:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/d8/17/707d17a5476c55a9541fde0db8213ac30979a792864d72415f176ba50c45/pyarrow-25.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:eb6203482ff3746a5632303a7279ae0b5a304c46985b49ed1378cb350ea6728d", size = 37644787, upload-time = "2026-08-10T12:37:25.795Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b2/cdc98ecf1a6408280bc3a6a07054cdd99a3f4670acc0545d383ce113e87d/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:880523be3d29efcf83d3998835d206118ccf35e3871dbd2fb60408cf6b007a80", size = 46834633, upload-time = "2026-08-10T12:37:33.604Z" }, + { url = "https://files.pythonhosted.org/packages/c8/6e/d3fafc41f378b2c65be43b827798c0fae42049a641c8526633ed3eb573e2/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:25f8720bf6387d5dc2ebd2622112de630760419e4b66134405dd24110d15f37e", size = 50065507, upload-time = "2026-08-10T12:37:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/d5/12/8d0698954b8c3001844a898e0a6900bebe83d7ee40c11195174c5122f324/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4facd65742a024a4a366328a1d2292062d72d6e023c1b7dda8d4c37544933a25", size = 49955690, upload-time = "2026-08-10T12:37:46.644Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0b/1ecb936ac6409e90a34d58eea1c7cec09a9ae6d2141b9e49ad01a2b1ea47/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa0559502e1cd6254d6814614085dd9c5a3dd0419362978a936a3f68a9e5c3df", size = 53128198, upload-time = "2026-08-10T12:37:52.531Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/5236033550633c9b7377b2a53660b2bbb06cb06dc09c4356332d67643ca1/pyarrow-25.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:62cd0d785b8aa6675ee355f9fc02252a340f4441257c42674937826fd7594325", size = 27857263, upload-time = "2026-08-10T12:37:56.943Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e2/9ab15b88cbfac28e16419ce5439ec29234c5172cb8259301b4ba639bdec0/pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:df961f2e7ae9cf496459259d798652c70625f6c080650d6952f8c04053c58ee9", size = 35861559, upload-time = "2026-08-10T12:38:02.567Z" }, + { url = "https://files.pythonhosted.org/packages/58/79/a0036dbe1eabe1f73127427342f1d99982584c4a2cde2651d6c93499c6f6/pyarrow-25.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:cc4aa407fde9fc660be3939e49ea31f50f3e9fec17c0ec63159f7711edd3efc9", size = 37628383, upload-time = "2026-08-10T12:38:09.083Z" }, + { url = "https://files.pythonhosted.org/packages/13/49/d93a57d375f4bf0cf82913dd6bb54acafde83dd993be2282c81ac5616cad/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:4340f0ba6c1d2e13f21658de1d7c662ca2545018568d0030a1e9afca159d87e3", size = 46820190, upload-time = "2026-08-10T12:38:15.458Z" }, + { url = "https://files.pythonhosted.org/packages/60/c9/711ca85d79f1ec98f29a5eae2b051e25b4ecec5de3e3c0e2d5c5dcb15664/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5389cdf79447ed1515c9e31620e6e1e2302249564d603f2ad727d4f6d313e4c3", size = 50102437, upload-time = "2026-08-10T12:38:22.487Z" }, + { url = "https://files.pythonhosted.org/packages/80/53/8fb8359ff17cfb6263a1cf3ebf7caec9fe197de118719e84fcb1d0618026/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d51592cb7561e87877c506113e7adbf1342ab579e6c21f0ef44b8ba41cb74c80", size = 49942424, upload-time = "2026-08-10T12:38:28.755Z" }, + { url = "https://files.pythonhosted.org/packages/e8/83/4e5ae02a9341571b18a6fca380ac7a58ce6ddae7ab3c060208c0a1e79f02/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6109c94d8b9f3b17a041daca16cacb2f651ad8f1ef70a4232c2c0f37a23da2a8", size = 53144206, upload-time = "2026-08-10T12:38:34.862Z" }, + { url = "https://files.pythonhosted.org/packages/65/ee/197cbf47e49f83e6ebeb946a5259a48a638dea27ac774db42fe78022179d/pyarrow-25.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:8858d7bfc22e3f51529aeaa4077225029724623e4595dc9eff8c793935c34140", size = 27953934, upload-time = "2026-08-10T12:38:39.808Z" }, ] [[package]]