From e62bf012ad308d11e272c18936dd76f8f6d8bbaa Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 19 Nov 2026 01:07:12 +0000 Subject: [PATCH] [SPARK-62058][CONNECT][PYTHON] Log pipeline events at the exact times of their timestamps, as PySpark's ToDatetime, and pin the rounding of pre-1970 nanoseconds of createDataFrame and literals - handle_pipeline_events of pyspark.pipelines.spark_connect_pipeline converts the microseconds of an event's Timestamp exactly, epoch + timedelta, as PySpark's event.timestamp.ToDatetime().replace(tzinfo=timezone.utc), instead of datetime.fromtimestamp of a float of its seconds, which rounded an event far from 1970 into the next second (0001-01-01 00:00:00.999999 logged 00:00:01) and failed with "year 10000 is out of range" for the last microsecond of 9999. - A test pins the rounding of times before 1970 with nanoseconds in each conversion of createDataFrame and of a literal, as PySpark 4.3.0's: local rows of pandas Timestamps, an object column of them, a pyarrow Table and a literal round down to the microsecond, and a datetime64[ns] column of a pandas DataFrame rounds toward 1970 when the safe cast is off and raises when it is on. - known-differences.md says how the row path of pandas frames without PyArrow drops nanoseconds: to the microsecond below, as local rows do, where PySpark's unsafe cast of a datetime64[ns] column rounds a time before 1970 toward 1970. Co-authored-by: Isaac --- dev/parity/known-differences.md | 5 +- .../pipelines/spark_connect_pipeline.py | 15 +++- python/tests/test_dropin_offline.py | 77 +++++++++++++++++++ 3 files changed, 92 insertions(+), 5 deletions(-) diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index 4c333ee4..f33f84eb 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -28,7 +28,10 @@ difference is intentional or tracked for a fix. local rows are typed, where PySpark's conversion takes Arrow's types: a named Row is a struct, not an array, a dict a map, not a struct, nested Decimals take precision 38 (see below), an object column of naive datetimes is a timestamp, not a timestamp_ntz, nanoseconds are - truncated without the error of a safe cast, and its errors are its own. This fallback covers + dropped as those of local rows are, to the microsecond below, without the error of a safe cast + (PySpark's unsafe cast of a `datetime64[ns]` column rounds a time before 1970 toward 1970: + `1969-12-31 23:59:59.999999999` is `1970-01-01 00:00:00` there, and `1969-12-31 23:59:59.999999` + here and in local rows), and its errors are its own. This fallback covers unavailable modules detected by `pyarrow_available`, not an installed PyArrow module that fails during import. The row path also converts a frame, with PyArrow, when the type names that the conversion and the importer resolve are rebound, or when the class of the struct of diff --git a/python/pyspark/pipelines/spark_connect_pipeline.py b/python/pyspark/pipelines/spark_connect_pipeline.py index 66d9d3f8..017a17c6 100644 --- a/python/pyspark/pipelines/spark_connect_pipeline.py +++ b/python/pyspark/pipelines/spark_connect_pipeline.py @@ -21,7 +21,7 @@ ``PipelineCommand`` construction + execution live in the Rust core (``pyspark._pyspark.pipeline_*``); these functions are thin wrappers over them. """ -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any, Iterator, Mapping, Optional, Sequence from pyspark.sql import SparkSession @@ -50,14 +50,21 @@ def create_dataflow_graph( ) +# The epoch of the microseconds of the timestamps of the events of a run. +_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) + + def handle_pipeline_events(iter: Iterator[Any]) -> None: """Print out the pipeline events received from the Spark Connect server. - The Rust run stream yields ``(message, timestamp_micros)`` per pipeline event and - raises the mapped pyspark exception (e.g. ``AnalysisException``) if the run fails. + The Rust run stream yields ``(message, timestamp_micros)`` per pipeline event, the + microseconds of the event's Timestamp since the epoch, of its nanoseconds truncated, and + raises the mapped pyspark exception (e.g. ``AnalysisException``) if the run fails. The time + of an event is exact, as PySpark's ``event.timestamp.ToDatetime()`` in UTC: a float of its + seconds would round the microseconds of a time far from 1970, into the next second too. """ for message, timestamp_micros in iter: - dt = datetime.fromtimestamp(timestamp_micros / 1_000_000, tz=timezone.utc) + dt = _EPOCH + timedelta(microseconds=timestamp_micros) log_with_provided_timestamp(message, dt) diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 9f5b2afb..d20c582a 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -27525,6 +27525,59 @@ def micros(column): time.tzset() +def test_pre_1970_nanoseconds_round_as_each_conversion_of_pyspark(mock_connect): + # As PySpark 4.3.0, of times before 1970 with nanoseconds: each of its conversions drops the + # digits below the microsecond in its own direction, and the drop-in's send the same + # microseconds. Local rows of pandas Timestamps and a literal of one take the fields of the + # datetime, of the microsecond below (floor), and so does an object column of a pandas + # DataFrame; a pyarrow Table is floored by pc.floor_temporal; a datetime64[ns] column is cast + # by PyArrow toward 1970 when the safe cast is off, and raises when it is on. + import pyarrow as pa + from pyspark.errors import PySparkValueError + from pyspark.sql import functions as F + + pd = pytest.importorskip("pandas") + spark, service = mock_connect + stamps = [pd.Timestamp("1969-12-31 23:59:59.999999999", tz="UTC"), + pd.Timestamp("1960-06-15 12:34:56.123456789", tz="UTC")] + floor, toward_1970 = [-1, -301231503876544], [0, -301231503876543] + ts = T.StructType([T.StructField("ts", T.TimestampType())]) + ntz = T.StructType([T.StructField("ts", T.TimestampNTZType())]) + + def micros(df): + # The Arrow data of the LocalRelation sent (Relation.local_relation=11, data=1), under + # the ToDF of names (Relation.to_df=18, input=1). + df.toArrow() + request = parse_message(service.execute_requests[-1]) + relation = parse_message(parse_message(request[3][0])[1][0]) + while 18 in relation: + relation = parse_message(parse_message(relation[18][0])[1][0]) + table = pa.ipc.open_stream(parse_message(relation[11][0])[1][0]).read_all() + return table.column(0).cast(pa.int64()).to_pylist() + + assert micros(spark.createDataFrame([(s,) for s in stamps], ts)) == floor + assert micros(spark.createDataFrame([(s.tz_localize(None),) for s in stamps], ntz)) == floor + objects = pd.DataFrame({"ts": pd.Series(stamps, dtype=object)}) + assert micros(spark.createDataFrame(objects, ts)) == floor + nanos = pa.array([s.value for s in stamps], pa.int64()) + for arrow_type in (pa.timestamp("ns", "UTC"), pa.timestamp("ns")): + table = pa.table({"ts": nanos.cast(arrow_type)}) + assert micros(spark.createDataFrame(table, ts)) == floor + frame = pd.DataFrame({"ts": pd.Series(stamps)}) + with pytest.raises(PySparkValueError, match="Exception thrown when converting pandas.Series"): + spark.createDataFrame(frame, ts) + service.confs = {"spark.sql.execution.pandas.convertToArrowArraySafely": "false"} + assert micros(spark.createDataFrame(frame, ts)) == toward_1970 + assert micros(spark.createDataFrame(frame)) == toward_1970 + # A literal, Literal.timestamp=17 of the Project's expression (Project.expressions=3). + for stamp, expected in zip(stamps, floor): + spark.range(1).select(F.lit(stamp)).collect() + plan = parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0] + expression = parse_message(parse_message(parse_message(plan)[3][0])[3][0]) + value = parse_message(expression[1][0])[17][0] + assert (value - (1 << 64) if value >= 1 << 63 else value) == expected + + def test_datetime_literals_follow_the_preferred_timestamp_type_as_pyspark(mock_connect): # As PySpark's LiteralExpression._infer_type, which reads spark.sql.timestampType for each # datetime literal: under TIMESTAMP_NTZ a datetime is a TIMESTAMP_NTZ of its wall clock (that @@ -29254,6 +29307,30 @@ def output(): assert len(service.execute_requests) == sent +def test_pipeline_events_are_logged_at_the_times_of_their_timestamps_as_pysparks(capsys): + # As PySpark's handle_pipeline_events, which logs each event at + # event.timestamp.ToDatetime() in UTC. The run stream gives the microseconds of the + # Timestamp, its nanoseconds truncated, which convert exactly, also before 1970 and far from + # it, where a float of the seconds rounded 0001-01-01 00:00:00.999999 into the next second + # and failed for the last microsecond of 9999. The lines are those of PySpark's + # ToDatetime of the same Timestamps. + from pyspark.pipelines.spark_connect_pipeline import handle_pipeline_events + + events = [ + ((-62135596800, 999_999_000), "first", "1-01-01 00:00:00: first"), + ((-29567770275, 976_787_301), "far", "1033-01-13 10:28:45: far"), + ((-1, 999_999_999), "before", "1969-12-31 23:59:59: before"), + ((0, 0), "epoch", "1970-01-01 00:00:00: epoch"), + ((1_763_510_400, 999_999_999), "now", "2025-11-19 00:00:00: now"), + ((253402300799, 999_999_999), "last", "9999-12-31 23:59:59: last"), + ] + # The (message, timestamp_micros) of each event, as pipeline_start_run's stream yields them. + handle_pipeline_events( + (message, seconds * 1_000_000 + nanos // 1_000) for (seconds, nanos), message, _ in events + ) + assert capsys.readouterr().out.splitlines() == [line for _, _, line in events] + + _REQUESTS_WITHOUT_THE_GIL = { "save": lambda spark: spark.range(1).write.format("parquet").save("/tmp/no_such_dir"), "saveAsTable": lambda spark: spark.range(1).write.saveAsTable("t"),