Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion dev/parity/known-differences.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ difference is intentional or tracked for a fix.
local rows are typed, where PySpark's conversion takes Arrow's types: a named Row is a struct,
not an array, a dict a map, not a struct, nested Decimals take precision 38 (see below), an
object column of naive datetimes is a timestamp, not a timestamp_ntz, nanoseconds are
truncated without the error of a safe cast, and its errors are its own. This fallback covers
dropped as those of local rows are, to the microsecond below, without the error of a safe cast
(PySpark's unsafe cast of a `datetime64[ns]` column rounds a time before 1970 toward 1970:
`1969-12-31 23:59:59.999999999` is `1970-01-01 00:00:00` there, and `1969-12-31 23:59:59.999999`
here and in local rows), and its errors are its own. This fallback covers
unavailable modules detected by `pyarrow_available`, not an installed PyArrow module that
fails during import. The row path also converts a frame, with PyArrow, when the type names
that the conversion and the importer resolve are rebound, or when the class of the struct of
Expand Down
15 changes: 11 additions & 4 deletions python/pyspark/pipelines/spark_connect_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
``PipelineCommand`` construction + execution live in the Rust core
(``pyspark._pyspark.pipeline_*``); these functions are thin wrappers over them.
"""
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from typing import Any, Iterator, Mapping, Optional, Sequence

from pyspark.sql import SparkSession
Expand Down Expand Up @@ -50,14 +50,21 @@ def create_dataflow_graph(
)


# The epoch of the microseconds of the timestamps of the events of a run.
_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)


def handle_pipeline_events(iter: Iterator[Any]) -> None:
"""Print out the pipeline events received from the Spark Connect server.

The Rust run stream yields ``(message, timestamp_micros)`` per pipeline event and
raises the mapped pyspark exception (e.g. ``AnalysisException``) if the run fails.
The Rust run stream yields ``(message, timestamp_micros)`` per pipeline event, the
microseconds of the event's Timestamp since the epoch, of its nanoseconds truncated, and
raises the mapped pyspark exception (e.g. ``AnalysisException``) if the run fails. The time
of an event is exact, as PySpark's ``event.timestamp.ToDatetime()`` in UTC: a float of its
seconds would round the microseconds of a time far from 1970, into the next second too.
"""
for message, timestamp_micros in iter:
dt = datetime.fromtimestamp(timestamp_micros / 1_000_000, tz=timezone.utc)
dt = _EPOCH + timedelta(microseconds=timestamp_micros)
log_with_provided_timestamp(message, dt)


Expand Down
77 changes: 77 additions & 0 deletions python/tests/test_dropin_offline.py
Original file line number Diff line number Diff line change
Expand Up @@ -27525,6 +27525,59 @@ def micros(column):
time.tzset()


def test_pre_1970_nanoseconds_round_as_each_conversion_of_pyspark(mock_connect):
# As PySpark 4.3.0, of times before 1970 with nanoseconds: each of its conversions drops the
# digits below the microsecond in its own direction, and the drop-in's send the same
# microseconds. Local rows of pandas Timestamps and a literal of one take the fields of the
# datetime, of the microsecond below (floor), and so does an object column of a pandas
# DataFrame; a pyarrow Table is floored by pc.floor_temporal; a datetime64[ns] column is cast
# by PyArrow toward 1970 when the safe cast is off, and raises when it is on.
import pyarrow as pa
from pyspark.errors import PySparkValueError
from pyspark.sql import functions as F

pd = pytest.importorskip("pandas")
spark, service = mock_connect
stamps = [pd.Timestamp("1969-12-31 23:59:59.999999999", tz="UTC"),
pd.Timestamp("1960-06-15 12:34:56.123456789", tz="UTC")]
floor, toward_1970 = [-1, -301231503876544], [0, -301231503876543]
ts = T.StructType([T.StructField("ts", T.TimestampType())])
ntz = T.StructType([T.StructField("ts", T.TimestampNTZType())])

def micros(df):
# The Arrow data of the LocalRelation sent (Relation.local_relation=11, data=1), under
# the ToDF of names (Relation.to_df=18, input=1).
df.toArrow()
request = parse_message(service.execute_requests[-1])
relation = parse_message(parse_message(request[3][0])[1][0])
while 18 in relation:
relation = parse_message(parse_message(relation[18][0])[1][0])
table = pa.ipc.open_stream(parse_message(relation[11][0])[1][0]).read_all()
return table.column(0).cast(pa.int64()).to_pylist()

assert micros(spark.createDataFrame([(s,) for s in stamps], ts)) == floor
assert micros(spark.createDataFrame([(s.tz_localize(None),) for s in stamps], ntz)) == floor
objects = pd.DataFrame({"ts": pd.Series(stamps, dtype=object)})
assert micros(spark.createDataFrame(objects, ts)) == floor
nanos = pa.array([s.value for s in stamps], pa.int64())
for arrow_type in (pa.timestamp("ns", "UTC"), pa.timestamp("ns")):
table = pa.table({"ts": nanos.cast(arrow_type)})
assert micros(spark.createDataFrame(table, ts)) == floor
frame = pd.DataFrame({"ts": pd.Series(stamps)})
with pytest.raises(PySparkValueError, match="Exception thrown when converting pandas.Series"):
spark.createDataFrame(frame, ts)
service.confs = {"spark.sql.execution.pandas.convertToArrowArraySafely": "false"}
assert micros(spark.createDataFrame(frame, ts)) == toward_1970
assert micros(spark.createDataFrame(frame)) == toward_1970
# A literal, Literal.timestamp=17 of the Project's expression (Project.expressions=3).
for stamp, expected in zip(stamps, floor):
spark.range(1).select(F.lit(stamp)).collect()
plan = parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
expression = parse_message(parse_message(parse_message(plan)[3][0])[3][0])
value = parse_message(expression[1][0])[17][0]
assert (value - (1 << 64) if value >= 1 << 63 else value) == expected


def test_datetime_literals_follow_the_preferred_timestamp_type_as_pyspark(mock_connect):
# As PySpark's LiteralExpression._infer_type, which reads spark.sql.timestampType for each
# datetime literal: under TIMESTAMP_NTZ a datetime is a TIMESTAMP_NTZ of its wall clock (that
Expand Down Expand Up @@ -29254,6 +29307,30 @@ def output():
assert len(service.execute_requests) == sent


def test_pipeline_events_are_logged_at_the_times_of_their_timestamps_as_pysparks(capsys):
# As PySpark's handle_pipeline_events, which logs each event at
# event.timestamp.ToDatetime() in UTC. The run stream gives the microseconds of the
# Timestamp, its nanoseconds truncated, which convert exactly, also before 1970 and far from
# it, where a float of the seconds rounded 0001-01-01 00:00:00.999999 into the next second
# and failed for the last microsecond of 9999. The lines are those of PySpark's
# ToDatetime of the same Timestamps.
from pyspark.pipelines.spark_connect_pipeline import handle_pipeline_events

events = [
((-62135596800, 999_999_000), "first", "1-01-01 00:00:00: first"),
((-29567770275, 976_787_301), "far", "1033-01-13 10:28:45: far"),
((-1, 999_999_999), "before", "1969-12-31 23:59:59: before"),
((0, 0), "epoch", "1970-01-01 00:00:00: epoch"),
((1_763_510_400, 999_999_999), "now", "2025-11-19 00:00:00: now"),
((253402300799, 999_999_999), "last", "9999-12-31 23:59:59: last"),
]
# The (message, timestamp_micros) of each event, as pipeline_start_run's stream yields them.
handle_pipeline_events(
(message, seconds * 1_000_000 + nanos // 1_000) for (seconds, nanos), message, _ in events
)
assert capsys.readouterr().out.splitlines() == [line for _, _, line in events]


_REQUESTS_WITHOUT_THE_GIL = {
"save": lambda spark: spark.range(1).write.format("parquet").save("/tmp/no_such_dir"),
"saveAsTable": lambda spark: spark.range(1).write.saveAsTable("t"),
Expand Down
Loading