fix(dtypes): the report no longer calls a datetime parse "converted to object"

CHANGELOG.md

@@ -113,6 +113,34 @@ adheres to [Semantic Versioning](https://semver.org/).
   entirely of complex values is still declined before parsing and keeps its
   dtype.
 
+- The report no longer calls a datetime parse "converted to object". A column
+  of datetime strings carrying different UTC offsets — `"2021-01-05
+  00:00:00+01:00"`, `"2021-01-06 00:00:00+02:00"`, the ordinary result of a
+  daylight-saving changeover or of merging exports from two regions — is parsed
+  cell by cell into timestamps that each keep their own offset. No single
+  `datetime64` dtype can hold several offsets, so pandas returns them in an
+  `object` column. That parse is faithful and lossless, but `fix_dtypes` built
+  its action description from the resulting dtype alone and recorded it as
+  `converted to object`, risk low, which reads as though nothing was
+  converted. A column mixing values with and without an offset took the same
+  path and got the same record. The description now says what happened:
+  `parsed to timestamps; kept as object because the values carry different
+  UTC offsets, so no single datetime64 dtype can hold them` (or `... because
+  the values mix timezone-aware and timezone-naive timestamps, ...`), followed
+  by the existing ` (N unparseable value(s) set to missing)` suffix when cells
+  were coerced. The coercion warning for such a column likewise says the
+  values "could not be parsed as timestamps" instead of "as object". Affects
+  every supported pandas version (pandas 1.x delivers `datetime.datetime`
+  cells, pandas 2 `Timestamp` cells; the record is the same).
+
+  **Compatibility impact:** report text only. Cell values, dtypes, risk
+  levels and confidence are unchanged, and so is every other column's
+  `converted to <dtype>` description, including single-offset
+  (`datetime64[ns, UTC+01:00]`) and naive (`datetime64[ns]`) datetime
+  columns. Code that matched the literal string `converted to object` for
+  such a column needs to match the new description. `fd.profile` still
+  previews such a column as `would convert to object`.
+
 
 ### Documentation
 

src/freshdata/steps/dtypes.py

@@ -592,8 +592,56 @@ def _warn_type_contamination(col: str, s: pd.Series, config: CleanConfig,
 COERCED_CELLS_CAP = 1_000
 
 
+def _is_object_datetime(target: str, converted: pd.Series) -> bool:
+    """A datetime parse that pandas could only deliver as an object column.
+
+    Values carrying different UTC offsets, or mixing timezone-aware with
+    timezone-naive values, have no common ``datetime64`` dtype, so pandas
+    returns one timestamp object per cell in an ``object`` column. The cells
+    were parsed; only the dtype is not a datetime64 one.
+    """
+    return target == "datetime" and not is_datetime64_any_dtype(converted.dtype)
+
+
+def _object_datetime_reason(converted: pd.Series) -> str:
+    """Say why the parsed timestamps in *converted* share no datetime64 dtype."""
+    aware = naive = False
+    offsets = set()
+    for value in converted.dropna():
+        utcoffset = getattr(value, "utcoffset", None)
+        if utcoffset is None:
+            continue
+        offset = utcoffset()
+        if offset is None:
+            naive = True
+        else:
+            aware = True
+            offsets.add(offset)
+    if aware and naive:
+        why = "the values mix timezone-aware and timezone-naive timestamps"
+    elif len(offsets) > 1:
+        why = "the values carry different UTC offsets"
+    else:
+        return "no single datetime64 dtype can hold them"
+    return f"{why}, so no single datetime64 dtype can hold them"
+
+
+def _describe_conversion(target: str, converted: pd.Series) -> str:
+    """The audit description of what :func:`fix_dtypes` did to a column.
+
+    ``"converted to <dtype>"`` except when a datetime parse ended in an object
+    column: saying "converted to object" there reads as though nothing was
+    converted, when every cell was in fact parsed to a timestamp.
+    """
+    if _is_object_datetime(target, converted):
+        return (f"parsed to timestamps; kept as {converted.dtype} because "
+                f"{_object_datetime_reason(converted)}")
+    return f"converted to {converted.dtype}"
+
+
 def _record_coerced(col: str, before: pd.Series, converted: pd.Series,
-                    report: CleanReport, config: CleanConfig) -> None:
+                    report: CleanReport, config: CleanConfig,
+                    target: str = "") -> None:
     """Preserve the original value of every cell the conversion nulled.
 
     For declared ``sensitive_columns`` the row keys survive but every value is
@@ -615,9 +663,11 @@ def _record_coerced(col: str, before: pd.Series, converted: pd.Series,
         for i, v in list(originals.head(3).items()))
     truncated = "" if len(originals) <= COERCED_CELLS_CAP else (
         f"; first {COERCED_CELLS_CAP} recorded")
+    parsed_as = ("timestamps" if _is_object_datetime(target, converted)
+                 else str(converted.dtype))
     report.add_warning(
         f"column '{col}': {len(originals)} value(s) could not be parsed as "
-        f"{converted.dtype} and were set to missing — e.g. {examples}. "
+        f"{parsed_as} and were set to missing — e.g. {examples}. "
         f"Originals are preserved in report.coerced_cells{truncated}; these "
         "cells stay missing (never auto-imputed) so they can be reviewed."
     )
@@ -726,10 +776,10 @@ def fix_dtypes(df: pd.DataFrame, config: CleanConfig, report: CleanReport) -> pd
         if converted is None:
             _warn_type_contamination(str(col), s, config, report)
             continue
-        description = f"converted to {converted.dtype}"
+        description = _describe_conversion(target, converted)
         if n_coerced:
             description += f" ({n_coerced} unparseable value(s) set to missing)"
-            _record_coerced(str(col), s, converted, report, config)
+            _record_coerced(str(col), s, converted, report, config, target)
         report.add("fix_dtypes", description, column=str(col),
                    count=int(converted.notna().sum()) + n_coerced)
         df[col] = converted

tests/test_dtypes_threshold_mutants.py

@@ -482,20 +482,21 @@ def test_the_post_semantic_retry_tolerates_exactly_the_casualty_budget():
 # ---------------------------------------------------------------------------
 
 
-def test_mixed_utc_offsets_are_reported_as_a_conversion_to_object():
-    """Pins current behaviour found while covering bool#16 -- a defect.
+def test_mixed_utc_offsets_parse_to_an_object_column_of_timestamps():
+    """Pins current behaviour found while covering bool#16.
 
     Values carrying *different* UTC offsets cannot share a datetime64 column,
-    so pandas returns an object-dtype Series of Timestamps. ``_try_datetime``
-    measures that object Series against the threshold, passes it, and the
-    column is announced as a dtype conversion: the report says "converted to
-    object" and the frame keeps an object column. A conversion that ends in
-    ``object`` is not a conversion, and because the result is not
-    datetime64 the ambiguity note in ``_record_coerced`` can never fire for
-    such a column either.
-
-    Nothing here argues the behaviour is right; it is pinned so that fixing
-    it has to change this test deliberately.
+    so pandas returns an object-dtype Series of per-cell timestamps, each
+    keeping its own offset. ``_try_datetime`` measures that object Series
+    against the threshold, passes it, and the column keeps the parsed
+    timestamps in an object column.
+
+    The audit record used to say "converted to object", which reads as though
+    nothing was converted (FD2-015); it now says what happened. The exact
+    wording is pinned in ``tests/test_mixed_offset_audit_record.py``. Whether
+    such a column should be converted at all (``utc=True``, or declined) is
+    not decided here; the dtype is pinned so that changing it has to change
+    this test deliberately.
     """
     values = [
         "2021-01-05 00:00:00+01:00",
@@ -510,4 +511,5 @@ def test_mixed_utc_offsets_are_reported_as_a_conversion_to_object():
     report = CleanReport()
     frame = fix_dtypes(pd.DataFrame({"when": values}), CleanConfig(), report)
     assert str(frame["when"].dtype) == "object"
-    assert [a.description for a in report.actions] == ["converted to object"]
+    [description] = [a.description for a in report.actions]
+    assert description.startswith("parsed to timestamps; kept as object")