diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index f4891c23..27e1cd12 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -286,10 +286,11 @@ def timestamp_columns(frame, schema): buffers separately so NaT's sentinel never participates in alignment tests. """ import sys - from types import GetSetDescriptorType, WrapperDescriptorType + from types import FunctionType, GetSetDescriptorType, WrapperDescriptorType import numpy as np import pandas as pd + from pandas.core import frame as frame_module, generic as generic_module try: from pandas.core.arrays.datetimes import DatetimeArray from pandas.core.dtypes.dtypes import DatetimeTZDtype @@ -305,7 +306,23 @@ def timestamp_columns(frame, schema): from pyspark.sql import types as sql_types if (pd.DataFrame is not DataFrame or type(frame) is not DataFrame - or type(schema) is not NativeStructType or not len(frame)): + or type(schema) is not NativeStructType or not len(frame) + or any(name in vars(frame) for name in ("astype", "notna", "where"))): + return None + # The row path calls these normalization methods before reading cells. + # Preserve customized transformations and errors before any Arrow Config RPC. + for name, module, qualname in ( + ("astype", generic_module, "NDFrame.astype"), + ("notna", frame_module, "DataFrame.notna"), + ("where", generic_module, "NDFrame.where"), + ): + method = getattr(DataFrame, name) + if (type(method) is not FunctionType or method.__name__ != name + or method.__qualname__ != qualname or method.__module__ != module.__name__ + or method.__globals__ is not vars(module)): + return None + if (DataFrame.astype is not generic_module.NDFrame.astype + or DataFrame.where is not generic_module.NDFrame.where): return None if not _has_canonical_columns(frame.columns): return None @@ -390,6 +407,10 @@ def timestamp_columns(frame, schema): if unit == "ns" and np.any(column.array.asi8[column.notna()] % 1000): return None columns.append(column) + # Reject unsupported host zones before Rust queries the session time zone. + # Their row fallback builds a local relation without any configuration RPC. + if not timestamp_columns_are_aware(columns) and not _timestamp_host_is_utc(): + return None return columns @@ -400,20 +421,26 @@ def timestamp_columns_are_aware(columns): return all(type(column.dtype) is DatetimeTZDtype for column in columns) +def _timestamp_host_is_utc(): + """Require an explicit UTC host setting and an initialized UTC runtime zone.""" + import os + import time + + return os.environ.get("TZ") == "UTC" and time.tzname == ("UTC", "UTC") + + def prepare_timestamp_table(columns, schema, timezone): """Preserve instants without row materialization or a host-zone conversion. Aware columns carry their own timezone. Naive columns are deliberately - restricted to UTC host/session pairs: the existing row path uses the host - zone, unlike Python Connect's session-zone conversion (and DST fold policy). + restricted to literal ``TZ=UTC`` and session ``UTC``, with a UTC runtime + zone: the existing row path uses the host zone, unlike Python Connect's + session-zone conversion (and DST fold policy). """ - import os - import time import pyarrow as pa if not timestamp_columns_are_aware(columns): - if (timezone != "UTC" or os.environ.get("TZ") != "UTC" - or time.tzname != ("UTC", "UTC")): + if timezone != "UTC" or not _timestamp_host_is_utc(): return None return pa.Table.from_arrays( [pa.array(column, type=pa.timestamp("us", tz="UTC"), from_pandas=True) diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index af993c41..fb6cd012 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -2087,10 +2087,6 @@ def test_pandas_timestamp_ipc_preserves_every_microsecond(protocol, monkeypatch, schema = T.StructType([T.StructField("renamed_a", T.TimestampType(), True, {"source": "test"}), T.StructField("b", T.TimestampType())]) - def no_rows(*args, **kwargs): - raise AssertionError("supported timestamps must not materialize Python rows") - - monkeypatch.setattr(pd.DataFrame, "astype", no_rows) with _arrow_input_service(protocol, capture_tables=True, **{"spark.sql.session.timeZone": zone}) as (spark, state): result = spark.createDataFrame(frame, schema) # Aware columns carry their zone: only the importer's settings are read. @@ -2117,7 +2113,6 @@ def test_pandas_naive_timestamp_utc_ipc(protocol, monkeypatch): frame = pd.DataFrame({"a": pd.to_datetime(["1969-12-31 23:59:59.999999", None, "2020-01-01 00:00:00.123456"])}) schema = T.StructType([T.StructField("a", T.TimestampType())]) - monkeypatch.setattr(pd.DataFrame, "astype", lambda *a, **k: pytest.fail("row conversion")) with _arrow_input_service(protocol, capture_tables=True) as (spark, state): result = spark.createDataFrame(frame, schema) # A naive column reads the session time zone, then the importer its settings. @@ -2132,9 +2127,13 @@ def test_pandas_naive_timestamp_utc_ipc(protocol, monkeypatch): @pytest.mark.parametrize("dtype", ["datetime64[ns]", "datetime64[us]", "datetime64[ms]", "datetime64[s]", "object"]) -def test_pandas_timestamp_eligibility(dtype): +def test_pandas_timestamp_eligibility(monkeypatch, dtype): + import time from pyspark.sql import types as T from pyspark.sql._pandas_input import timestamp_columns + + monkeypatch.setenv("TZ", "UTC") + monkeypatch.setattr(time, "tzname", ("UTC", "UTC")) frame = pd.DataFrame({"a": pd.Series([datetime(2020, 1, 1), None], dtype=dtype)}) schema = T.StructType([T.StructField("a", T.TimestampType())]) assert (timestamp_columns(frame, schema) is not None) == (dtype in ( @@ -2146,15 +2145,19 @@ def test_pandas_timestamp_eligibility(dtype): def test_pandas_timestamp_precision_and_timezone_fallback(monkeypatch): + import time from pyspark.sql import types as T from pyspark.sql._pandas_input import timestamp_columns, prepare_timestamp_table + + monkeypatch.setenv("TZ", "UTC") + monkeypatch.setattr(time, "tzname", ("UTC", "UTC")) schema = T.StructType([T.StructField("a", T.TimestampType())]) for nanos in (-1, 1, 1001): frame = pd.DataFrame({"a": pd.to_datetime([nanos, None], unit="ns")}) assert timestamp_columns(frame, schema) is None frame = pd.DataFrame({"a": pd.to_datetime([0, None], unit="ns")}) columns = timestamp_columns(frame, schema) - monkeypatch.setenv("TZ", "UTC") + assert columns is not None assert prepare_timestamp_table(columns, schema, "America/Los_Angeles") is None monkeypatch.setenv("TZ", "America/Los_Angeles") assert prepare_timestamp_table(columns, schema, "UTC") is None @@ -2162,6 +2165,54 @@ def test_pandas_timestamp_precision_and_timezone_fallback(monkeypatch): assert prepare_timestamp_table(columns, schema, "UTC") is None +@pytest.mark.parametrize("unit,mixed", [("ns", False), ("us", True)]) +@pytest.mark.parametrize("host_tz,host_names", [ + (None, ("UTC", "UTC")), + ("America/Los_Angeles", ("PST", "PDT")), + ("UTC", ("PST", "PDT")), +], ids=["unset-tz", "non-utc-tz", "mismatched-host-zone"]) +def test_timestamp_non_utc_host_declines_and_converts_as_pyspark( + protocol, monkeypatch, unit, mixed, host_tz, host_names +): + # A host zone other than a literal UTC declines the timestamp fast path before its + # session time zone RPC; the frame then converts as PySpark's Arrow conversion does, + # which reads the configs first and takes naive values in the session time zone (UTC). + import json + import time + + if host_tz is None: + monkeypatch.delenv("TZ", raising=False) + else: + monkeypatch.setenv("TZ", host_tz) + monkeypatch.setattr(time, "tzname", host_names) + micros = [-2000000, 0, 123456, None] + columns = {} + if mixed: + columns["aware"] = pd.to_datetime(micros, unit="us", utc=True).as_unit(unit) + columns["naive"] = pd.to_datetime(micros, unit="us").as_unit(unit) + frame = pd.DataFrame(columns) + schema = T.StructType([ + T.StructField(f"renamed_{name}", T.TimestampType(), True, {"source": name}) + for name in columns + ]) + from pyspark.sql._pandas_input import timestamp_columns + + assert timestamp_columns(frame, schema) is None + expected = [micros for _ in columns] + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] + assert state["uploads"] == 0 + assert state["column_names"] == schema.names + assert len(state["input_tables"]) == 1 + table = state["input_tables"][0] + assert table.num_rows == len(micros) + for column, values in zip(table.columns, expected): + assert column.type == pa.timestamp("us", tz="UTC") + assert column.cast(pa.int64()).to_pylist() == values + assert column.is_null().to_pylist() == [value is None for value in values] + + @pytest.mark.parametrize("has_null", [False, True]) def test_pandas_timestamp_non_nullable_schema(protocol, has_null): from pyspark.sql import types as T @@ -2218,6 +2269,81 @@ def test_timestamp_nonnullable_nat_reads_config_before_cast_error(protocol, conf assert state["uploads"] == 0 +@pytest.mark.parametrize("unit,aware", [("ns", True), ("us", False)]) +@pytest.mark.parametrize("location,method", [ + ("instance", "notna"), ("instance", "astype"), ("instance", "where"), + ("class", "notna"), ("class", "astype"), ("class", "where"), +]) +@pytest.mark.parametrize("behavior", ["mutate", "raise"]) +def test_timestamp_custom_normalization_hooks_keep_native_row_path( + protocol, monkeypatch, unit, aware, location, method, behavior +): + import json + import time + from types import MethodType + import pyspark.sql._pandas_input as helper + + monkeypatch.setenv("TZ", "UTC") + monkeypatch.setattr(time, "tzname", ("UTC", "UTC")) + values = pd.to_datetime([-2000000, -1000000, 0, 1000000, None], unit="us", utc=aware) + frame = pd.DataFrame({"original": values.as_unit(unit)}) + schema = T.StructType([ + T.StructField("renamed", T.TimestampType(), True, {"source": "normalization"}) + ]) + original = getattr(pd.DataFrame, method) + calls, tables = [], [] + # astype returns a new frame, so the row path never calls an instance where. + invokes_hook = (location, method) != ("instance", "where") + expected = [None if pd.isna(value) else round(value.to_pydatetime().timestamp() * 1000000) + for value in frame["original"]] + if invokes_hook: + expected[0] = None if method == "notna" else 7000000 + message = f"custom timestamp {method} precedes configuration" + + def customized(self, *args, **kwargs): + calls.append(method) + if behavior == "raise": + raise RuntimeError(message) + result = original(self, *args, **kwargs) + result.iat[0, 0] = (False if method == "notna" + else datetime(1970, 1, 1, 0, 0, 7, tzinfo=timezone.utc)) + return result + + if location == "class": + monkeypatch.setattr(pd.DataFrame, method, customized) + else: + monkeypatch.setitem(vars(frame), method, MethodType(customized, frame)) + + # Compare with native row conversion, then restore ordinary dispatch. A failing + # configuration service makes any premature Arrow configuration read observable. + for bypass in (True, False): + calls.clear() + with monkeypatch.context() as dispatch: + if bypass: + dispatch.setattr(helper, "timestamp_columns", lambda *_: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + if behavior == "raise" and invokes_hook: + with pytest.raises(RuntimeError, match=message): + spark.createDataFrame(frame, schema) + assert state["relations"] == [] + assert state["input_tables"] == [] + else: + spark.createDataFrame(frame, schema).toArrow() + assert len(state["input_tables"]) == 1 + table = state["input_tables"][0] + assert state["column_names"] == schema.names + assert table.column(0).cast(pa.int64()).to_pylist() == expected + assert table.column(0).is_null().to_pylist() == [ + value is None for value in expected] + assert json.loads(state["relations"][0].local_relation.schema) == json.loads(schema.json()) + tables.append(table) + assert calls == ([method] if invokes_hook else []) + assert state["config_requests"] == [] + assert state["uploads"] == 0 + if behavior == "mutate" or not invokes_hook: + assert tables[0].equals(tables[1], check_metadata=True) + + @pytest.mark.parametrize("custom", ["dtype", "array"]) def test_timestamp_custom_datetime_array_keeps_row_values(protocol, custom): from pandas.core.arrays.datetimes import DatetimeArray @@ -2427,8 +2553,8 @@ def prepare_table(table, schema, *configs): spark.createDataFrame(pa.table({"warm": [1]})).toArrow() monkeypatch.setitem(sys.modules, importer.__name__, substitute) assert timestamp_columns(frame, schema) is not None - monkeypatch.setattr(pd.DataFrame, "astype", lambda *a, **k: pytest.fail("row conversion")) spark.createDataFrame(frame, schema).toArrow() + assert len(state["config_requests"]) == 2 table = state["tables"][-1] assert table.column_names == ["renamed"] assert table.column(0).cast(pa.int64()).to_pylist() == [0, 1000, None] @@ -2669,7 +2795,7 @@ def set_data(index, value): @pytest.mark.parametrize("unit", ["ns", "us"]) @pytest.mark.parametrize("storage", ["object", "default", "python", "pyarrow", "python_nan", "pyarrow_nan"]) -def test_timestamp_canonical_columns_storage_stays_columnar(protocol, monkeypatch, unit, storage): +def test_timestamp_canonical_columns_storage_stays_columnar(protocol, unit, storage): from pyspark.sql.pandas.types import to_arrow_schema import json @@ -2682,10 +2808,6 @@ def test_timestamp_canonical_columns_storage_stays_columnar(protocol, monkeypatc if storage.endswith("_nan") else pd.StringDtype(storage=kind)) frame.columns = pd.Index(pd.array(["original"], dtype=dtype)) - def no_rows(*args, **kwargs): - raise AssertionError("canonical timestamp labels must retain columnar construction") - - monkeypatch.setattr(pd.DataFrame, "astype", no_rows) with _arrow_input_service(protocol) as (spark, state): spark.createDataFrame(frame, schema).toArrow() assert state["config_requests"]