From 9e97fc745c2aec65bb462163dd53bca813dddc34 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 6 Nov 2026 17:10:35 -0800 Subject: [PATCH 1/4] [SPARK-60686][CONNECT] Preserve custom pandas timestamp normalization Decline the timestamp Arrow fast path when pandas normalization methods are customized, preserving their values, errors and configuration ordering. Co-authored-by: Isaac --- python/pyspark/sql/_pandas_input.py | 21 +++++++- python/tests/test_arrow_handoff.py | 82 +++++++++++++++++++++++++---- 2 files changed, 90 insertions(+), 13 deletions(-) diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index 8cf6d98a..134958f0 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -286,10 +286,11 @@ def timestamp_columns(frame, schema): buffers separately so NaT's sentinel never participates in alignment tests. """ import sys - from types import GetSetDescriptorType, WrapperDescriptorType + from types import FunctionType, GetSetDescriptorType, WrapperDescriptorType import numpy as np import pandas as pd + from pandas.core import frame as frame_module, generic as generic_module try: from pandas.core.arrays.datetimes import DatetimeArray from pandas.core.dtypes.dtypes import DatetimeTZDtype @@ -305,7 +306,23 @@ def timestamp_columns(frame, schema): from pyspark.sql import types as sql_types if (pd.DataFrame is not DataFrame or type(frame) is not DataFrame - or type(schema) is not NativeStructType or not len(frame)): + or type(schema) is not NativeStructType or not len(frame) + or any(name in vars(frame) for name in ("astype", "notna", "where"))): + return None + # The row path calls these normalization methods before reading cells. + # Preserve customized transformations and errors before any Arrow Config RPC. + for name, module, qualname in ( + ("astype", generic_module, "NDFrame.astype"), + ("notna", frame_module, "DataFrame.notna"), + ("where", generic_module, "NDFrame.where"), + ): + method = getattr(DataFrame, name) + if (type(method) is not FunctionType or method.__name__ != name + or method.__qualname__ != qualname or method.__module__ != module.__name__ + or method.__globals__ is not vars(module)): + return None + if (DataFrame.astype is not generic_module.NDFrame.astype + or DataFrame.where is not generic_module.NDFrame.where): return None if not _has_canonical_columns(frame.columns): return None diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 9424ac5e..020e1d49 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -2021,10 +2021,6 @@ def test_pandas_timestamp_ipc_preserves_every_microsecond(protocol, monkeypatch, schema = T.StructType([T.StructField("renamed_a", T.TimestampType(), True, {"source": "test"}), T.StructField("b", T.TimestampType())]) - def no_rows(*args, **kwargs): - raise AssertionError("supported timestamps must not materialize Python rows") - - monkeypatch.setattr(pd.DataFrame, "astype", no_rows) with _arrow_input_service(protocol, capture_tables=True, **{"spark.sql.session.timeZone": zone}) as (spark, state): result = spark.createDataFrame(frame, schema) # Aware columns carry their zone: only the importer's settings are read. @@ -2051,7 +2047,6 @@ def test_pandas_naive_timestamp_utc_ipc(protocol, monkeypatch): frame = pd.DataFrame({"a": pd.to_datetime(["1969-12-31 23:59:59.999999", None, "2020-01-01 00:00:00.123456"])}) schema = T.StructType([T.StructField("a", T.TimestampType())]) - monkeypatch.setattr(pd.DataFrame, "astype", lambda *a, **k: pytest.fail("row conversion")) with _arrow_input_service(protocol, capture_tables=True) as (spark, state): result = spark.createDataFrame(frame, schema) # A naive column reads the session time zone, then the importer its settings. @@ -2142,6 +2137,75 @@ def test_timestamp_nonnullable_nat_keeps_local_error_before_config(protocol, con assert state["uploads"] == 0 +@pytest.mark.parametrize("unit,aware", [("ns", True), ("us", False)]) +@pytest.mark.parametrize("location,method", [ + ("instance", "notna"), ("instance", "astype"), + ("class", "notna"), ("class", "astype"), ("class", "where"), +]) +@pytest.mark.parametrize("behavior", ["mutate", "raise"]) +def test_timestamp_custom_normalization_hooks_keep_native_row_path( + protocol, monkeypatch, unit, aware, location, method, behavior +): + import json + from types import MethodType + import pyspark.sql._pandas_input as helper + + values = pd.to_datetime([-2000000, -1000000, 0, 1000000, None], unit="us", utc=aware) + frame = pd.DataFrame({"original": values.as_unit(unit)}) + schema = T.StructType([ + T.StructField("renamed", T.TimestampType(), True, {"source": "normalization"}) + ]) + original = getattr(pd.DataFrame, method) + calls, tables = [], [] + expected = [None if pd.isna(value) else round(value.to_pydatetime().timestamp() * 1000000) + for value in frame["original"]] + expected[0] = None if method == "notna" else 7000000 + message = f"custom timestamp {method} precedes configuration" + + def customized(self, *args, **kwargs): + calls.append(method) + if behavior == "raise": + raise RuntimeError(message) + result = original(self, *args, **kwargs) + result.iat[0, 0] = (False if method == "notna" + else datetime(1970, 1, 1, 0, 0, 7, tzinfo=timezone.utc)) + return result + + if location == "class": + monkeypatch.setattr(pd.DataFrame, method, customized) + else: + monkeypatch.setitem(vars(frame), method, MethodType(customized, frame)) + + # Compare with native row conversion, then restore ordinary dispatch. A failing + # configuration service makes any premature Arrow configuration read observable. + for bypass in (True, False): + calls.clear() + with monkeypatch.context() as dispatch: + if bypass: + dispatch.setattr(helper, "timestamp_columns", lambda *_: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + if behavior == "raise": + with pytest.raises(RuntimeError, match=message): + spark.createDataFrame(frame, schema) + assert state["relations"] == [] + assert state["input_tables"] == [] + else: + spark.createDataFrame(frame, schema).toArrow() + assert len(state["input_tables"]) == 1 + table = state["input_tables"][0] + assert state["column_names"] == schema.names + assert table.column(0).cast(pa.int64()).to_pylist() == expected + assert table.column(0).is_null().to_pylist() == [ + value is None for value in expected] + assert json.loads(state["relations"][0].local_relation.schema) == json.loads(schema.json()) + tables.append(table) + assert calls == [method] + assert state["config_requests"] == [] + assert state["uploads"] == 0 + if behavior == "mutate": + assert tables[0].equals(tables[1], check_metadata=True) + + @pytest.mark.parametrize("custom", ["dtype", "array"]) def test_timestamp_custom_datetime_array_keeps_row_values(protocol, custom): from pandas.core.arrays.datetimes import DatetimeArray @@ -2351,8 +2415,8 @@ def prepare_table(table, schema, *configs): spark.createDataFrame(pa.table({"warm": [1]})).toArrow() monkeypatch.setitem(sys.modules, importer.__name__, substitute) assert timestamp_columns(frame, schema) is not None - monkeypatch.setattr(pd.DataFrame, "astype", lambda *a, **k: pytest.fail("row conversion")) spark.createDataFrame(frame, schema).toArrow() + assert len(state["config_requests"]) == 2 table = state["tables"][-1] assert table.column_names == ["renamed"] assert table.column(0).cast(pa.int64()).to_pylist() == [0, 1000, None] @@ -2593,7 +2657,7 @@ def set_data(index, value): @pytest.mark.parametrize("unit", ["ns", "us"]) @pytest.mark.parametrize("storage", ["object", "default", "python", "pyarrow", "python_nan", "pyarrow_nan"]) -def test_timestamp_canonical_columns_storage_stays_columnar(protocol, monkeypatch, unit, storage): +def test_timestamp_canonical_columns_storage_stays_columnar(protocol, unit, storage): from pyspark.sql.pandas.types import to_arrow_schema import json @@ -2606,10 +2670,6 @@ def test_timestamp_canonical_columns_storage_stays_columnar(protocol, monkeypatc if storage.endswith("_nan") else pd.StringDtype(storage=kind)) frame.columns = pd.Index(pd.array(["original"], dtype=dtype)) - def no_rows(*args, **kwargs): - raise AssertionError("canonical timestamp labels must retain columnar construction") - - monkeypatch.setattr(pd.DataFrame, "astype", no_rows) with _arrow_input_service(protocol) as (spark, state): spark.createDataFrame(frame, schema).toArrow() assert state["config_requests"] From 0627d493a590fe94936c4e6da27d572dd7da9113 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 6 Nov 2026 17:17:08 -0800 Subject: [PATCH 2/4] [SPARK-60686][CONNECT] Preserve custom pandas timestamp normalization Decline the timestamp Arrow fast path when pandas normalization methods are customized, preserving their values, errors and configuration ordering. Co-authored-by: Isaac --- python/pyspark/sql/_pandas_input.py | 22 +++++--- python/tests/test_arrow_handoff.py | 78 ++++++++++++++++++++++++++--- 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index 134958f0..bab354de 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -407,6 +407,10 @@ def timestamp_columns(frame, schema): if unit == "ns" and np.any(column.array.asi8[column.notna()] % 1000): return None columns.append(column) + # Reject unsupported host zones before Rust queries the session time zone. + # Their row fallback builds a local relation without any configuration RPC. + if not timestamp_columns_are_aware(columns) and not _timestamp_host_is_utc(): + return None return columns @@ -417,20 +421,26 @@ def timestamp_columns_are_aware(columns): return all(type(column.dtype) is DatetimeTZDtype for column in columns) +def _timestamp_host_is_utc(): + """Require an explicit UTC host setting and an initialized UTC runtime zone.""" + import os + import time + + return os.environ.get("TZ") == "UTC" and time.tzname == ("UTC", "UTC") + + def prepare_timestamp_table(columns, schema, timezone): """Preserve instants without row materialization or a host-zone conversion. Aware columns carry their own timezone. Naive columns are deliberately - restricted to UTC host/session pairs: the existing row path uses the host - zone, unlike Python Connect's session-zone conversion (and DST fold policy). + restricted to literal ``TZ=UTC`` and session ``UTC``, with a UTC runtime + zone: the existing row path uses the host zone, unlike Python Connect's + session-zone conversion (and DST fold policy). """ - import os - import time import pyarrow as pa if not timestamp_columns_are_aware(columns): - if (timezone != "UTC" or os.environ.get("TZ") != "UTC" - or time.tzname != ("UTC", "UTC")): + if timezone != "UTC" or not _timestamp_host_is_utc(): return None return pa.Table.from_arrays( [pa.array(column, type=pa.timestamp("us", tz="UTC"), from_pandas=True) diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 020e1d49..a46cbb78 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -2061,9 +2061,13 @@ def test_pandas_naive_timestamp_utc_ipc(protocol, monkeypatch): @pytest.mark.parametrize("dtype", ["datetime64[ns]", "datetime64[us]", "datetime64[ms]", "datetime64[s]", "object"]) -def test_pandas_timestamp_eligibility(dtype): +def test_pandas_timestamp_eligibility(monkeypatch, dtype): + import time from pyspark.sql import types as T from pyspark.sql._pandas_input import timestamp_columns + + monkeypatch.setenv("TZ", "UTC") + monkeypatch.setattr(time, "tzname", ("UTC", "UTC")) frame = pd.DataFrame({"a": pd.Series([datetime(2020, 1, 1), None], dtype=dtype)}) schema = T.StructType([T.StructField("a", T.TimestampType())]) assert (timestamp_columns(frame, schema) is not None) == (dtype in ( @@ -2075,15 +2079,19 @@ def test_pandas_timestamp_eligibility(dtype): def test_pandas_timestamp_precision_and_timezone_fallback(monkeypatch): + import time from pyspark.sql import types as T from pyspark.sql._pandas_input import timestamp_columns, prepare_timestamp_table + + monkeypatch.setenv("TZ", "UTC") + monkeypatch.setattr(time, "tzname", ("UTC", "UTC")) schema = T.StructType([T.StructField("a", T.TimestampType())]) for nanos in (-1, 1, 1001): frame = pd.DataFrame({"a": pd.to_datetime([nanos, None], unit="ns")}) assert timestamp_columns(frame, schema) is None frame = pd.DataFrame({"a": pd.to_datetime([0, None], unit="ns")}) columns = timestamp_columns(frame, schema) - monkeypatch.setenv("TZ", "UTC") + assert columns is not None assert prepare_timestamp_table(columns, schema, "America/Los_Angeles") is None monkeypatch.setenv("TZ", "America/Los_Angeles") assert prepare_timestamp_table(columns, schema, "UTC") is None @@ -2091,6 +2099,56 @@ def test_pandas_timestamp_precision_and_timezone_fallback(monkeypatch): assert prepare_timestamp_table(columns, schema, "UTC") is None +@pytest.mark.parametrize("unit,mixed", [("ns", False), ("us", True)]) +@pytest.mark.parametrize("host_tz,host_names", [ + (None, ("UTC", "UTC")), + ("America/Los_Angeles", ("PST", "PDT")), + ("UTC", ("PST", "PDT")), +], ids=["unset-tz", "non-utc-tz", "mismatched-host-zone"]) +@pytest.mark.parametrize("config_error", [False, True]) +def test_timestamp_non_utc_host_keeps_row_path_without_config( + protocol, monkeypatch, unit, mixed, host_tz, host_names, config_error +): + import json + import time + + if host_tz is None: + monkeypatch.delenv("TZ", raising=False) + else: + monkeypatch.setenv("TZ", host_tz) + monkeypatch.setattr(time, "tzname", host_names) + micros = [-2000000, 0, 123456, None] + columns = {} + if mixed: + columns["aware"] = pd.to_datetime(micros, unit="us", utc=True).as_unit(unit) + columns["naive"] = pd.to_datetime(micros, unit="us").as_unit(unit) + frame = pd.DataFrame(columns) + schema = T.StructType([ + T.StructField(f"renamed_{name}", T.TimestampType(), True, {"source": name}) + for name in columns + ]) + # Match the row path's naive instants using datetime.timestamp(), independently + # of the mocked timezone eligibility inputs above. + expected = [ + [None if pd.isna(value) else round(value.to_pydatetime().timestamp() * 1000000) + for value in frame[name]] + for name in columns + ] + with _arrow_input_service(protocol, config_error=config_error) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["column_names"] == schema.names + assert len(state["input_tables"]) == 1 + table = state["input_tables"][0] + assert table.num_rows == len(micros) + for column, values in zip(table.columns, expected): + assert column.type == pa.timestamp("us", tz="UTC") + assert column.cast(pa.int64()).to_pylist() == values + assert column.is_null().to_pylist() == [value is None for value in values] + assert json.loads(state["relations"][0].local_relation.schema) == json.loads(schema.json()) + + @pytest.mark.parametrize("has_null", [False, True]) def test_pandas_timestamp_non_nullable_schema(protocol, has_null): from pyspark.sql import types as T @@ -2139,7 +2197,7 @@ def test_timestamp_nonnullable_nat_keeps_local_error_before_config(protocol, con @pytest.mark.parametrize("unit,aware", [("ns", True), ("us", False)]) @pytest.mark.parametrize("location,method", [ - ("instance", "notna"), ("instance", "astype"), + ("instance", "notna"), ("instance", "astype"), ("instance", "where"), ("class", "notna"), ("class", "astype"), ("class", "where"), ]) @pytest.mark.parametrize("behavior", ["mutate", "raise"]) @@ -2147,9 +2205,12 @@ def test_timestamp_custom_normalization_hooks_keep_native_row_path( protocol, monkeypatch, unit, aware, location, method, behavior ): import json + import time from types import MethodType import pyspark.sql._pandas_input as helper + monkeypatch.setenv("TZ", "UTC") + monkeypatch.setattr(time, "tzname", ("UTC", "UTC")) values = pd.to_datetime([-2000000, -1000000, 0, 1000000, None], unit="us", utc=aware) frame = pd.DataFrame({"original": values.as_unit(unit)}) schema = T.StructType([ @@ -2157,9 +2218,12 @@ def test_timestamp_custom_normalization_hooks_keep_native_row_path( ]) original = getattr(pd.DataFrame, method) calls, tables = [], [] + # astype returns a new frame, so the row path never calls an instance where. + invokes_hook = (location, method) != ("instance", "where") expected = [None if pd.isna(value) else round(value.to_pydatetime().timestamp() * 1000000) for value in frame["original"]] - expected[0] = None if method == "notna" else 7000000 + if invokes_hook: + expected[0] = None if method == "notna" else 7000000 message = f"custom timestamp {method} precedes configuration" def customized(self, *args, **kwargs): @@ -2184,7 +2248,7 @@ def customized(self, *args, **kwargs): if bypass: dispatch.setattr(helper, "timestamp_columns", lambda *_: None) with _arrow_input_service(protocol, config_error=True) as (spark, state): - if behavior == "raise": + if behavior == "raise" and invokes_hook: with pytest.raises(RuntimeError, match=message): spark.createDataFrame(frame, schema) assert state["relations"] == [] @@ -2199,10 +2263,10 @@ def customized(self, *args, **kwargs): value is None for value in expected] assert json.loads(state["relations"][0].local_relation.schema) == json.loads(schema.json()) tables.append(table) - assert calls == [method] + assert calls == ([method] if invokes_hook else []) assert state["config_requests"] == [] assert state["uploads"] == 0 - if behavior == "mutate": + if behavior == "mutate" or not invokes_hook: assert tables[0].equals(tables[1], check_metadata=True) From 96d768c11546697d92ef4dcc86c315c7fa7a4db9 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Sat, 7 Nov 2026 16:53:38 +0900 Subject: [PATCH 3/4] Expect PySpark's conversion of the frames that the timestamp fast path declines Since SPARK-61079 such frames convert as PySpark's Arrow conversion does, which reads the configs first and takes naive values in the session time zone, rather than the row path, which read none and took the host zone. Co-authored-by: Isaac --- python/tests/test_arrow_handoff.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 92abaac7..88fe3327 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -2172,9 +2172,12 @@ def test_pandas_timestamp_precision_and_timezone_fallback(monkeypatch): ("UTC", ("PST", "PDT")), ], ids=["unset-tz", "non-utc-tz", "mismatched-host-zone"]) @pytest.mark.parametrize("config_error", [False, True]) -def test_timestamp_non_utc_host_keeps_row_path_without_config( +def test_timestamp_non_utc_host_declines_and_converts_as_pyspark( protocol, monkeypatch, unit, mixed, host_tz, host_names, config_error ): + # A host zone other than a literal UTC declines the timestamp fast path before its + # session time zone RPC; the frame then converts as PySpark's Arrow conversion does, + # which reads the configs first and takes naive values in the session time zone (UTC). import json import time @@ -2193,16 +2196,18 @@ def test_timestamp_non_utc_host_keeps_row_path_without_config( T.StructField(f"renamed_{name}", T.TimestampType(), True, {"source": name}) for name in columns ]) - # Match the row path's naive instants using datetime.timestamp(), independently - # of the mocked timezone eligibility inputs above. - expected = [ - [None if pd.isna(value) else round(value.to_pydatetime().timestamp() * 1000000) - for value in frame[name]] - for name in columns - ] + from pyspark.sql._pandas_input import timestamp_columns + + assert timestamp_columns(frame, schema) is None + expected = [micros for _ in columns] with _arrow_input_service(protocol, config_error=config_error) as (spark, state): + if config_error: + with pytest.raises(Exception, match="configuration unavailable"): + spark.createDataFrame(frame, schema).toArrow() + assert state["input_tables"] == [] + return spark.createDataFrame(frame, schema).toArrow() - assert state["config_requests"] == [] + assert state["config_requests"] assert state["uploads"] == 0 assert state["column_names"] == schema.names assert len(state["input_tables"]) == 1 @@ -2212,7 +2217,6 @@ def test_timestamp_non_utc_host_keeps_row_path_without_config( assert column.type == pa.timestamp("us", tz="UTC") assert column.cast(pa.int64()).to_pylist() == values assert column.is_null().to_pylist() == [value is None for value in values] - assert json.loads(state["relations"][0].local_relation.schema) == json.loads(schema.json()) @pytest.mark.parametrize("has_null", [False, True]) From 572edf8b3930554b77f05cf1e3841b535a65a346 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Sat, 7 Nov 2026 18:56:01 +0900 Subject: [PATCH 4/4] Drop the unavailable-configuration variant of the declined timestamp test The client retries UNAVAILABLE configuration RPCs under its policy, so expecting the error took minutes per case; the declined frames read their configs as PySpark's conversion does, which the remaining cases check. Co-authored-by: Isaac --- python/tests/test_arrow_handoff.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 88fe3327..fb6cd012 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -2171,9 +2171,8 @@ def test_pandas_timestamp_precision_and_timezone_fallback(monkeypatch): ("America/Los_Angeles", ("PST", "PDT")), ("UTC", ("PST", "PDT")), ], ids=["unset-tz", "non-utc-tz", "mismatched-host-zone"]) -@pytest.mark.parametrize("config_error", [False, True]) def test_timestamp_non_utc_host_declines_and_converts_as_pyspark( - protocol, monkeypatch, unit, mixed, host_tz, host_names, config_error + protocol, monkeypatch, unit, mixed, host_tz, host_names ): # A host zone other than a literal UTC declines the timestamp fast path before its # session time zone RPC; the frame then converts as PySpark's Arrow conversion does, @@ -2200,12 +2199,7 @@ def test_timestamp_non_utc_host_declines_and_converts_as_pyspark( assert timestamp_columns(frame, schema) is None expected = [micros for _ in columns] - with _arrow_input_service(protocol, config_error=config_error) as (spark, state): - if config_error: - with pytest.raises(Exception, match="configuration unavailable"): - spark.createDataFrame(frame, schema).toArrow() - assert state["input_tables"] == [] - return + with _arrow_input_service(protocol) as (spark, state): spark.createDataFrame(frame, schema).toArrow() assert state["config_requests"] assert state["uploads"] == 0