diff --git a/CHANGELOG.md b/CHANGELOG.md index b73efa20..ea193dcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -243,6 +243,21 @@ adheres to [Semantic Versioning](https://semver.org/). no column, instead of masking nothing and exiting 0 (#251). ### Fixed +- `fd.evaluate_quality_debt` reports the `duplicates` dimension as not assessed + when duplicate rows cannot be checked (object columns of lists or dicts, + nested Arrow list/struct/map columns), instead of scoring it a clean 0.0 with + "0 duplicate row(s) detected". The item serialises with `score` and + `over_threshold` as `None`, its detail names the unhashable columns, it is + listed in `QualityDebtGate.unassessed`, and it never counts toward the total, + the gate status or the ledger history. Frames where duplicates can be + detected score exactly as before. +- `compute_trust_score` reports uniqueness as unknown when duplicate rows cannot + be checked (the same unhashable columns), instead of a perfect 100. + `TrustScore.uniqueness` is `NaN` with `uniqueness_assessed` False, `to_dict()` + gives `None`, `str()` and the Markdown tables show `n/a`, the blocking columns + carry an "unhashable values: duplicate rows not checked" issue, and `overall` + blends the other three dimensions with their weights renormalised. Frames + where duplicates can be detected score exactly as before. - Trust-gate integrations now validate `on_low_score` policies at configuration boundaries, rejecting typos instead of silently skipping failure handling (#345). - `QualityReport.to_markdown()` now escapes every cell in the Actions table, so diff --git a/docs/decision-workflow.md b/docs/decision-workflow.md index 8b40013f..0440caa0 100644 --- a/docs/decision-workflow.md +++ b/docs/decision-workflow.md @@ -75,6 +75,16 @@ instability, outlier spikes, PII risk, category churn, failed repairs, human- review backlog), persists the history to SQLite, and **escalates warn→fail when an issue repeats or worsens** across runs. +The duplicates dimension counts duplicate rows left in the cleaned output (or +the rows removed, if more). When duplicates cannot be checked because a column +holds unhashable values (lists, dicts, or nested Arrow list/struct/map columns), +the dimension is **not assessed** rather than scored clean: `to_dict()` gives +`score` and `over_threshold` as `None`, the detail names the columns, +`gate.unassessed` lists it, and `summary()` prints a `? duplicates: not +assessed` line. An unassessed dimension adds nothing to the total, never +changes the gate status on its own, and is not written to the ledger, so +escalation compares against the last run that measured it. + ## Dirty-join assistant Reviewable fuzzy joins for messy keys — never a silent low-confidence join. diff --git a/docs/feature-overview.md b/docs/feature-overview.md index 1a2589d8..4506542c 100644 --- a/docs/feature-overview.md +++ b/docs/feature-overview.md @@ -31,7 +31,7 @@ dependencies stay lazy, so a plain `import freshdata` is unaffected. | Capability | API | |---|---| | Full enterprise pipeline | `clean_enterprise(df, *, enterprise=…)` → `EnterpriseResult` | -| Data Trust Score (0–100) | `compute_trust_score(df)` → completeness / validity / uniqueness / consistency | +| Data Trust Score (0–100) | `compute_trust_score(df)` → completeness / validity / uniqueness / consistency (uniqueness is unknown and left out of the overall when a column holds unhashable values such as lists) | | Fuzzy value clustering | `merge_clusters(df, cols)` / `cluster_column(df, col)` | | PII masking | `mask_dataframe(df, rules)` — hash / redact / partial / regex-scrub / drop | | Semantic validation | `run_semantic_validation(df, configs)` — reference / regex / API checks | diff --git a/src/freshdata/enterprise/metrics.py b/src/freshdata/enterprise/metrics.py index 63f1aab9..a069ab7c 100644 --- a/src/freshdata/enterprise/metrics.py +++ b/src/freshdata/enterprise/metrics.py @@ -6,7 +6,11 @@ - **validity** — share of *present* cells already in a clean, parseable form (no stray whitespace, no sentinel-as-value, parses to its inferred dtype, not a statistical outlier), -- **uniqueness** — share of rows that are not exact duplicates, +- **uniqueness** — share of rows that are not exact duplicates. When duplicate + rows cannot be checked (a column holds lists, dicts or nested Arrow + list/struct/map values) it is *unknown* (``NaN``, ``None`` in + :meth:`TrustScore.to_dict`) and the overall score is blended from the other + three dimensions with their weights renormalised, - **consistency** — share of columns free of structural defects that corruption can introduce (mixed types, duplicate labels). Constant columns are surfaced as per-column issues instead of lowering this dimension: @@ -21,7 +25,8 @@ from __future__ import annotations import json -from dataclasses import dataclass, field +import math +from dataclasses import dataclass, field, replace from datetime import datetime, timezone from typing import TYPE_CHECKING, Any @@ -42,6 +47,9 @@ #: Minimum non-null count before outliers are scored (matches profile.py). _MIN_FOR_OUTLIERS = 20 +#: Per-column issue for values that make duplicate rows impossible to check. +_UNHASHABLE_ISSUE = "unhashable values: duplicate rows not checked" + @dataclass(frozen=True) class ColumnTrust: @@ -60,6 +68,9 @@ class TrustScore: Render with ``print(score)``, export with :meth:`to_dict` / :meth:`to_markdown`. ``grade`` maps the overall score to an A-F letter. + + ``uniqueness`` is ``NaN`` when duplicate rows could not be checked (see + :attr:`uniqueness_assessed`); ``overall`` then excludes it. """ overall: float @@ -71,6 +82,14 @@ class TrustScore: n_cols: int columns: tuple[ColumnTrust, ...] = () + @property + def uniqueness_assessed(self) -> bool: + """False when duplicate rows could not be checked (unhashable values).""" + return not math.isnan(self.uniqueness) + + def _uniqueness_text(self, fmt: str) -> str: + return format(self.uniqueness, fmt) if self.uniqueness_assessed else "n/a" + @property def grade(self) -> str: """Letter grade: A (>=90), B (>=80), C (>=70), D (>=60), else F.""" @@ -86,7 +105,7 @@ def to_dict(self) -> dict[str, Any]: "dimensions": { "completeness": round(self.completeness, 2), "validity": round(self.validity, 2), - "uniqueness": round(self.uniqueness, 2), + "uniqueness": round(self.uniqueness, 2) if self.uniqueness_assessed else None, "consistency": round(self.consistency, 2), }, "n_rows": self.n_rows, @@ -106,12 +125,12 @@ def to_dict(self) -> dict[str, Any]: def to_markdown(self) -> str: """Markdown table of the score and its dimensions.""" rows = [ - ("Completeness", self.completeness), - ("Validity", self.validity), - ("Uniqueness", self.uniqueness), - ("Consistency", self.consistency), + ("Completeness", f"{self.completeness:.1f}"), + ("Validity", f"{self.validity:.1f}"), + ("Uniqueness", self._uniqueness_text(".1f")), + ("Consistency", f"{self.consistency:.1f}"), ] - body = [_md_table_row((dim, f"{val:.1f}")) for dim, val in rows] + body = [_md_table_row(row) for row in rows] lines = [ f"### Data Trust Score: **{self.overall:.1f} / 100** (grade {self.grade})", "", @@ -126,7 +145,7 @@ def __str__(self) -> str: return ( f"Data Trust Score {self.overall:.1f}/100 (grade {self.grade}) — " f"completeness {self.completeness:.0f}, validity {self.validity:.0f}, " - f"uniqueness {self.uniqueness:.0f}, consistency {self.consistency:.0f}" + f"uniqueness {self._uniqueness_text('.0f')}, consistency {self.consistency:.0f}" ) def __repr__(self) -> str: @@ -211,6 +230,21 @@ def _is_constant(s: pd.Series, n_rows: int) -> bool: return False +def _unhashable_positions(frame: pd.DataFrame) -> list[int]: + """Positions of columns that make ``DataFrame.duplicated`` impossible. + + ``duplicated`` factorizes every column, which is what raises; a lone + ``Series.duplicated()`` on a list column does not. + """ + positions = [] + for i in range(frame.shape[1]): + try: + pd.factorize(frame.iloc[:, i]) + except (TypeError, NotImplementedError): + positions.append(i) + return positions + + def compute_trust_score( df: pd.DataFrame, *, @@ -219,7 +253,10 @@ def compute_trust_score( ) -> TrustScore: """Profile *df* (pandas or polars) and compute its Data Trust Score. - Read-only: the input is never modified. + Read-only: the input is never modified. If duplicate rows cannot be + checked, uniqueness is ``NaN`` and the overall score uses the remaining + dimensions with their weights renormalised (equal weights if all of the + weight was on uniqueness). """ frame = to_pandas(df) cfg = config or CleanConfig() @@ -274,18 +311,33 @@ def compute_trust_score( dup_rows = int(frame.duplicated().sum()) uniqueness = 100.0 * (1.0 - dup_rows / n_rows) if n_rows else 100.0 except (TypeError, NotImplementedError): # lists/dicts, nested Arrow: undetectable - uniqueness = 100.0 + # "Cannot check" is not "no duplicates": leave uniqueness unknown and + # name the columns that block the check. + uniqueness = math.nan + for i in _unhashable_positions(frame): + col_trust[i] = replace(col_trust[i], issues=(*col_trust[i].issues, _UNHASHABLE_ISSUE)) dup_labels = int(frame.columns.duplicated().sum()) flagged = min(n_cols, inconsistent_cols + dup_labels) consistency = 100.0 * (1.0 - flagged / n_cols) if n_cols else 100.0 - overall = ( - w["completeness"] * completeness - + w["validity"] * validity - + w["uniqueness"] * uniqueness - + w["consistency"] * consistency - ) + if not math.isnan(uniqueness): + overall = ( + w["completeness"] * completeness + + w["validity"] * validity + + w["uniqueness"] * uniqueness + + w["consistency"] * consistency + ) + else: + measured = w["completeness"] + w["validity"] + w["consistency"] + if measured > 0: + overall = ( + w["completeness"] * completeness + + w["validity"] * validity + + w["consistency"] * consistency + ) / measured + else: # all weight was on uniqueness: fall back to equal weights + overall = (completeness + validity + consistency) / 3.0 return TrustScore( overall=overall, completeness=completeness, @@ -358,8 +410,8 @@ def to_markdown(self) -> str: _md_table_row(("Completeness", f"{before.completeness:.1f}", f"{after.completeness:.1f}")), _md_table_row(("Validity", f"{before.validity:.1f}", f"{after.validity:.1f}")), - _md_table_row(("Uniqueness", f"{before.uniqueness:.1f}", - f"{after.uniqueness:.1f}")), + _md_table_row(("Uniqueness", before._uniqueness_text(".1f"), + after._uniqueness_text(".1f"))), _md_table_row(("Consistency", f"{before.consistency:.1f}", f"{after.consistency:.1f}")), _md_table_row(("**Overall**", f"**{before.overall:.1f}**", diff --git a/src/freshdata/quality.py b/src/freshdata/quality.py index 2977221c..3a702e25 100644 --- a/src/freshdata/quality.py +++ b/src/freshdata/quality.py @@ -48,21 +48,27 @@ class DebtItem: threshold: float detail: str previous: float | None = None + #: False when the dimension could not be measured on this run (for example + #: duplicate rows in a frame with unhashable cells). An unassessed item is + #: neither over threshold nor evidence of a clean result: it serialises + #: with ``score`` and ``over_threshold`` as ``None``, adds nothing to the + #: total, never drives the gate status and is not written to the ledger. + assessed: bool = True @property def over(self) -> bool: - return self.score > self.threshold + return self.assessed and self.score > self.threshold @property def worsening(self) -> bool: - return self.previous is not None and self.score > self.previous + return self.assessed and self.previous is not None and self.score > self.previous def to_dict(self) -> dict[str, Any]: return { "dimension": self.dimension, - "score": round(self.score, 4), + "score": round(self.score, 4) if self.assessed else None, "threshold": self.threshold, - "over_threshold": self.over, + "over_threshold": self.over if self.assessed else None, "previous": None if self.previous is None else round(self.previous, 4), "worsening": self.worsening, "detail": self.detail, @@ -89,6 +95,11 @@ def passed(self) -> bool: def warned(self) -> list[DebtItem]: return [i for i in self.items if i.over] + @property + def unassessed(self) -> list[DebtItem]: + """Dimensions that could not be measured this run (no evidence either way).""" + return [i for i in self.items if not i.assessed] + def to_frame(self) -> pd.DataFrame: return pd.DataFrame( [i.to_dict() for i in self.items], @@ -123,6 +134,8 @@ def summary(self) -> str: f"(threshold {i.threshold:.2f}){flag} — {i.detail}") if not self.warned: lines.append(" no quality debt over threshold") + for i in self.unassessed: + lines.append(f" ? {i.dimension}: not assessed — {i.detail}") return "\n".join(lines) def __str__(self) -> str: @@ -146,6 +159,10 @@ def _html_sections(self) -> list[str]: ]) rows = [] for i in sorted(self.items, key=lambda x: x.score, reverse=True): + if not i.assessed: + rows.append([i.dimension, "n/a", f"{i.threshold:.2f}", + H.risk_badge("unknown"), "", i.detail]) + continue sev = "high" if (i.over and i.worsening) else "medium" if i.over else "low" rows.append([i.dimension, f"{i.score:.2f}", f"{i.threshold:.2f}", H.risk_badge(sev), @@ -162,11 +179,15 @@ def _html_sections(self) -> list[str]: def _score_debt( df: pd.DataFrame, cleaned: pd.DataFrame, report: CleanReport, baseline: pd.DataFrame | None, -) -> dict[str, tuple[float, str]]: - """Return ``{dimension: (score, detail)}`` for the current run.""" +) -> dict[str, tuple[float | None, str]]: + """Return ``{dimension: (score, detail)}`` for the current run. + + A ``None`` score means the dimension could not be measured; *detail* then + says why. + """ rows = max(1, report.rows_after) cells = max(1, report.rows_after * max(1, report.cols_after)) - out: dict[str, tuple[float, str]] = {} + out: dict[str, tuple[float | None, str]] = {} miss = report.missing_after / cells out["missingness"] = (miss, f"{report.missing_after:,} missing cell(s) remain") @@ -178,10 +199,19 @@ def _score_debt( try: dup_remaining = int(cleaned.duplicated().sum()) except (TypeError, NotImplementedError): # lists/dicts, nested Arrow: undetectable - dup_remaining = 0 - n_dup = max(dup_removed, dup_remaining) - out["duplicates"] = (n_dup / max(1, report.rows_before), - f"{n_dup:,} duplicate row(s) detected ({dup_removed:,} removed)") + # Detection is impossible, which is not the same as "no duplicates". + # Rows already removed (duplicate_subset avoiding the column) are + # still a known lower bound; otherwise the dimension is unassessed. + why = f"duplicate rows could not be checked: {_unhashable_note(cleaned)}" + if dup_removed: + out["duplicates"] = (dup_removed / max(1, report.rows_before), + f"{dup_removed:,} duplicate row(s) removed; remaining {why}") + else: + out["duplicates"] = (None, why) + else: + n_dup = max(dup_removed, dup_remaining) + out["duplicates"] = (n_dup / max(1, report.rows_before), + f"{n_dup:,} duplicate row(s) detected ({dup_removed:,} removed)") outl = report.outliers_handled / rows out["outlier_spikes"] = (outl, f"{report.outliers_handled:,} outlier(s) flagged") @@ -237,6 +267,24 @@ def _score_debt( return out +def _unhashable_note(frame: pd.DataFrame, limit: int = 5) -> str: + """Name the columns whose values block duplicate detection.""" + names = [] + for i in range(frame.shape[1]): + # DataFrame.duplicated factorizes every column, which is what raises; + # a lone Series.duplicated() on a list column does not. + try: + pd.factorize(frame.iloc[:, i]) + except (TypeError, NotImplementedError): + names.append(repr(str(frame.columns[i]))) + if not names: + return "the frame holds unhashable values" + shown = ", ".join(names[:limit]) + if len(names) > limit: + shown += f" and {len(names) - limit} more" + return f"column(s) {shown} hold unhashable values" + + def _category_churn(baseline: pd.DataFrame, current: pd.DataFrame) -> float: """Mean fraction of new categorical values across shared low-cardinality cols.""" shared = [c for c in baseline.columns if c in current.columns] @@ -262,11 +310,16 @@ def _ledger_setup(conn: sqlite3.Connection) -> None: def _previous_scores(conn: sqlite3.Connection) -> dict[str, float]: - row = conn.execute("SELECT run_id FROM debt_runs ORDER BY run_id DESC LIMIT 1").fetchone() - if not row: - return {} + """Each dimension's score from the last run that recorded it. + + Unassessed dimensions are not written, so the latest run may lack a + dimension; reading per dimension keeps ``previous`` (and ``worsening``) + anchored to the last run that actually measured it. + """ return {d: float(s) for d, s in conn.execute( - "SELECT dimension, score FROM debt_items WHERE run_id=?", (row[0],)).fetchall()} + "SELECT i.dimension, i.score FROM debt_items i " + "WHERE i.run_id = (SELECT MAX(run_id) FROM debt_items WHERE dimension = i.dimension)" + ).fetchall()} def _over_threshold_history(conn: sqlite3.Connection, dimension: str, n: int = 1) -> int: @@ -329,7 +382,8 @@ def evaluate_quality_debt( items: list[DebtItem] = [] for dim in DEBT_DIMENSIONS: score, detail = scores.get(dim, (0.0, "not assessed")) - items.append(DebtItem(dim, score, thr[dim], detail, previous.get(dim))) + items.append(DebtItem(dim, 0.0 if score is None else score, thr[dim], detail, + previous.get(dim), assessed=score is not None)) over = [i for i in items if i.over] # Decide status under the policy. @@ -348,7 +402,7 @@ def evaluate_quality_debt( break status = "fail" if escalate else "warn" - total = round(sum(i.score for i in items), 4) + total = round(sum(i.score for i in items if i.assessed), 4) run_at = datetime.now(timezone.utc).isoformat() trust = None @@ -365,9 +419,11 @@ def evaluate_quality_debt( "INSERT INTO debt_runs(run_at, policy, status, total_score) VALUES(?,?,?,?)", (run_at, debt_policy, status, total)) run_id = cur.lastrowid + # Unassessed dimensions are left out, so the next run compares against + # (and escalates from) the last run that actually measured them. conn.executemany( "INSERT INTO debt_items(run_id, dimension, score, over_threshold) VALUES(?,?,?,?)", - [(run_id, i.dimension, i.score, int(i.over)) for i in items]) + [(run_id, i.dimension, i.score, int(i.over)) for i in items if i.assessed]) conn.commit() conn.close() diff --git a/src/freshdata/render/normalize.py b/src/freshdata/render/normalize.py index a117ca72..f4ca4791 100644 --- a/src/freshdata/render/normalize.py +++ b/src/freshdata/render/normalize.py @@ -543,13 +543,17 @@ def normalize_copilot_report(rep: Any) -> PeelView: if rep.pii_warning else "no PII detected — nothing was masked" ) + # Uniqueness is unknown when duplicate rows could not be checked. + uniqueness = ( + f"{trust.uniqueness:.0f}" if getattr(trust, "uniqueness_assessed", True) else "n/a" + ) metrics = ( Metric("privacy", pii_state), Metric("trust", f"{trust.overall:.0f}/100 ({trust.grade}) — data quality only"), Metric( "dimensions", f"completeness {trust.completeness:.0f} · validity {trust.validity:.0f} · " - f"uniqueness {trust.uniqueness:.0f} · consistency {trust.consistency:.0f}", + f"uniqueness {uniqueness} · consistency {trust.consistency:.0f}", ), ) diff --git a/tests/test_enterprise_metrics_nested.py b/tests/test_enterprise_metrics_nested.py index aaea69ab..5006d7d6 100644 --- a/tests/test_enterprise_metrics_nested.py +++ b/tests/test_enterprise_metrics_nested.py @@ -5,19 +5,29 @@ large_list, struct, map) raise ``pyarrow.lib.ArrowNotImplementedError``, a ``NotImplementedError`` subclass, because they cannot be hashed or dictionary-encoded. Both mean "cannot compute": the column is not reported as -constant and uniqueness falls back to 100 instead of crashing. +constant, and uniqueness is unknown (``NaN``, ``None`` when serialised) and +left out of the overall score instead of counting as a perfect 100. """ from __future__ import annotations import json +import math import pandas as pd import pytest -from freshdata.enterprise import clean_enterprise, compute_trust_score +import freshdata as fd +from freshdata.enterprise import ( + TrustScoreWeights, + build_quality_report, + clean_enterprise, + compute_trust_score, +) from freshdata.enterprise.metrics import _is_constant +_UNHASHABLE = "unhashable values: duplicate rows not checked" + def _arrow_nested_series(kind: str, values: list) -> pd.Series: pa = pytest.importorskip("pyarrow") @@ -53,6 +63,30 @@ def _nested_frame(kind: str) -> pd.DataFrame: }) +def _renormalised(score, weights: TrustScoreWeights | None = None) -> float: + w = (weights or TrustScoreWeights()).normalized() + measured = w["completeness"] + w["validity"] + w["consistency"] + return ( + w["completeness"] * score.completeness + + w["validity"] * score.validity + + w["consistency"] * score.consistency + ) / measured + + +def _assert_uniqueness_unknown(score) -> None: + assert math.isnan(score.uniqueness) + assert not score.uniqueness_assessed + assert score.overall == pytest.approx(_renormalised(score)) + payload = score.to_dict() + assert payload["dimensions"]["uniqueness"] is None + assert json.loads(json.dumps(payload)) == payload # strict JSON, no NaN + assert "uniqueness n/a" in str(score) + assert "| Uniqueness | n/a |" in score.to_markdown() + by_name = {c.name: c for c in score.columns} + assert _UNHASHABLE in by_name["tags"].issues + assert _UNHASHABLE not in by_name["a"].issues + + def test_trust_score_arrow_list_column_repro(): pa = pytest.importorskip("pyarrow") if not hasattr(pd, "ArrowDtype"): @@ -63,8 +97,7 @@ def test_trust_score_arrow_list_column_repro(): score = compute_trust_score(df) assert score.n_rows == 3 assert score.completeness == 100.0 - assert score.uniqueness == 100.0 # duplicate detection impossible - assert json.dumps(score.to_dict()) + _assert_uniqueness_unknown(score) # duplicate detection impossible @pytest.mark.parametrize("kind", _KINDS) @@ -75,12 +108,10 @@ def test_trust_score_arrow_nested_column(kind): assert (score.n_rows, score.n_cols) == (3, 2) assert score.completeness == 100.0 assert score.validity == 100.0 - assert score.uniqueness == 100.0 + _assert_uniqueness_unknown(score) assert 0.0 <= score.overall <= 100.0 by_name = {c.name: c for c in score.columns} assert "constant column" not in by_name["tags"].issues - assert str(score) - assert json.dumps(score.to_dict()) pd.testing.assert_frame_equal(df, snapshot) # scoring never mutates @@ -96,13 +127,38 @@ def test_object_list_column_matches_nested_fallback(): score = compute_trust_score(df) assert score.completeness == 100.0 assert score.validity == 100.0 - assert score.uniqueness == 100.0 + assert score.consistency == 50.0 # "tags" infers as mixed + _assert_uniqueness_unknown(score) + # (0.3 * 100 + 0.3 * 100 + 0.2 * 50) / 0.8; previously 90.0 with uniqueness 100. + assert score.overall == pytest.approx(87.5) by_name = {c.name: c for c in score.columns} assert "constant column" not in by_name["tags"].issues same = pd.Series([["x"], ["x"], ["x"]]) assert _is_constant(same, len(same)) is False +def test_unknown_uniqueness_carries_no_weight(): + df = pd.DataFrame({"a": [1, None, 2], "tags": [["x"], ["y"], ["y"]]}) + heavy = TrustScoreWeights(completeness=1.0, validity=0.0, uniqueness=5.0, consistency=0.0) + score = compute_trust_score(df, weights=heavy) + assert score.overall == pytest.approx(score.completeness) + only_uniqueness = TrustScoreWeights(completeness=0.0, validity=0.0, uniqueness=1.0, + consistency=0.0) + score = compute_trust_score(df, weights=only_uniqueness) + assert not score.uniqueness_assessed + assert score.overall == pytest.approx( + (score.completeness + score.validity + score.consistency) / 3) + + +def test_quality_report_markdown_marks_unknown_uniqueness(): + df = pd.DataFrame({"a": [1, 2, 2], "tags": [["x"], ["y", "z"], ["y", "z"]]}) + cleaned, report = fd.clean(df, return_report=True, verbose=False) + quality = build_quality_report(df, cleaned, report) + assert "| Uniqueness | n/a | n/a |" in quality.to_markdown() + payload = json.loads(quality.to_json()) + assert payload["trust_before"]["dimensions"]["uniqueness"] is None + + def test_plain_frame_scores_unchanged(): df = pd.DataFrame({ "a": [1, 2, 2, None], @@ -113,7 +169,18 @@ def test_plain_frame_scores_unchanged(): assert score.completeness == pytest.approx(100.0 * 11 / 12) assert score.validity == 100.0 assert score.uniqueness == 75.0 # row 3 repeats row 2 + assert score.uniqueness_assessed assert score.consistency == pytest.approx(100.0 * 2 / 3) # "m" is mixed + w = TrustScoreWeights().normalized() + assert score.overall == ( # the exact four-way blend, bit for bit + w["completeness"] * score.completeness + + w["validity"] * score.validity + + w["uniqueness"] * score.uniqueness + + w["consistency"] * score.consistency + ) + assert score.to_dict()["dimensions"]["uniqueness"] == 75.0 + assert "uniqueness 75," in str(score) + assert "| Uniqueness | 75.0 |" in score.to_markdown() by_name = {c.name: c for c in score.columns} assert by_name["b"].issues == ("constant column",) assert by_name["m"].issues == ("mixed types",) @@ -124,6 +191,8 @@ def test_plain_frame_scores_unchanged(): def test_clean_enterprise_arrow_nested_column(kind): df = _nested_frame(kind) result = clean_enterprise(df, verbose=False) - assert result.trust_before.uniqueness == 100.0 + assert not result.trust_before.uniqueness_assessed assert result.trust_after.n_rows == 3 - assert json.loads(result.to_json())["trust_after"]["n_rows"] == 3 + payload = json.loads(result.to_json()) + assert payload["trust_after"]["n_rows"] == 3 + assert payload["trust_before"]["dimensions"]["uniqueness"] is None diff --git a/tests/test_peel_copilot.py b/tests/test_peel_copilot.py index c5479a69..d0ac3159 100644 --- a/tests/test_peel_copilot.py +++ b/tests/test_peel_copilot.py @@ -115,6 +115,18 @@ def test_trust_metric_carries_scope_note(self): assert "data quality only" in trust_metric.value assert "71/100 (C)" in trust_metric.value + def test_dimensions_metric_shows_unknown_uniqueness_as_na(self): + view = normalize(make_report(grade_dims=(62.0, 78.0, float("nan"), 45.0))) + dims = next(m for m in view.metrics if m.label == "dimensions") + assert "uniqueness n/a" in dims.value + assert "nan" not in dims.value + assert "completeness 62" in dims.value + + def test_dimensions_metric_shows_measured_uniqueness(self): + view = normalize(make_report()) + dims = next(m for m in view.metrics if m.label == "dimensions") + assert "uniqueness 99" in dims.value + class TestExperimentalAndProvider: def test_experimental_banner_always_present(self): diff --git a/tests/test_quality_debt_dimensions.py b/tests/test_quality_debt_dimensions.py index 5eadfcae..05f15cc1 100644 --- a/tests/test_quality_debt_dimensions.py +++ b/tests/test_quality_debt_dimensions.py @@ -2,12 +2,15 @@ from __future__ import annotations +import json +import sqlite3 + import pandas as pd import pytest import freshdata as fd from freshdata.enterprise import privacy -from freshdata.quality import DebtItem, QualityDebtGate +from freshdata.quality import DebtItem, QualityDebtGate, _unhashable_note def _item(gate: QualityDebtGate, dimension: str) -> DebtItem: @@ -51,6 +54,185 @@ def test_no_duplicates_scores_zero() -> None: assert dup.detail == "0 duplicate row(s) detected (0 removed)" +_DUP_KEYS = ["dimension", "score", "threshold", "over_threshold", "previous", "worsening", + "detail"] + + +def test_plain_frame_duplicates_serialisation_unchanged() -> None: + """Detectable duplicates keep the exact prior scores and output.""" + _, gate = fd.evaluate_quality_debt( + _half_duplicated(), debt_policy="fail", ledger=None, verbose=False + ) + dup = _item(gate, "duplicates") + assert dup.assessed + assert list(dup.to_dict()) == _DUP_KEYS + assert dup.to_dict() == { + "dimension": "duplicates", "score": 0.5, "threshold": 0.1, "over_threshold": True, + "previous": None, "worsening": False, + "detail": "50 duplicate row(s) detected (0 removed)", + } + assert gate.unassessed == [] + assert "not assessed" not in gate.summary() + assert gate.total_score == pytest.approx(sum(i.score for i in gate.items)) + + _, clean_gate = fd.evaluate_quality_debt( + pd.DataFrame({"a": range(20), "b": list("xy" * 10)}), ledger=None, verbose=False + ) + assert _item(clean_gate, "duplicates").to_dict() == { + "dimension": "duplicates", "score": 0.0, "threshold": 0.1, "over_threshold": False, + "previous": None, "worsening": False, + "detail": "0 duplicate row(s) detected (0 removed)", + } + assert clean_gate.unassessed == [] + + +def _list_frame() -> pd.DataFrame: + # Rows 2 and 3 are real duplicates that cannot be detected. + return pd.DataFrame({"id": [1, 2, 2], "tags": [["x"], ["y", "z"], ["y", "z"]]}) + + +def _arrow_frame(kind: str) -> pd.DataFrame: + pa = pytest.importorskip("pyarrow") + if not hasattr(pd, "ArrowDtype"): + pytest.skip("pd.ArrowDtype is not available in this pandas version") + types = { + "list": (pa.list_(pa.string()), [["x"], ["y", "z"], ["y", "z"]]), + "struct": (pa.struct([("k", pa.int64()), ("v", pa.string())]), + [{"k": 1, "v": "x"}, {"k": 2, "v": "y"}, {"k": 2, "v": "y"}]), + } + dtype, values = types[kind] + try: + nested = pd.array(values, dtype=pd.ArrowDtype(dtype)) + except (TypeError, ValueError, NotImplementedError) as exc: # pragma: no cover + pytest.skip(f"nested ArrowDtype {kind!r} unsupported here: {exc}") + return pd.DataFrame({"id": [1, 2, 2], "tags": nested}) + + +def _assert_duplicates_unassessed(gate: QualityDebtGate) -> None: + dup = _item(gate, "duplicates") + assert not dup.assessed + assert not dup.over + assert not dup.worsening + assert dup.detail == ( + "duplicate rows could not be checked: column(s) 'tags' hold unhashable values" + ) + payload = dup.to_dict() + assert list(payload) == _DUP_KEYS + assert payload["score"] is None + assert payload["over_threshold"] is None + assert gate.unassessed == [dup] + assert dup not in gate.warned + # No passing evidence: excluded from the total, flagged in every output. + assert gate.total_score == pytest.approx( + round(sum(i.score for i in gate.items if i.assessed), 4)) + assert f"? duplicates: not assessed — {dup.detail}" in gate.summary() + assert "n/a" in gate.to_html() + frame = gate.to_frame() + assert pd.isna(frame.loc[frame["dimension"] == "duplicates", "score"].iloc[0]) + decoded = json.loads(json.dumps(gate.to_dict())) + assert decoded == gate.to_dict() + assert next(i for i in decoded["items"] if i["dimension"] == "duplicates") == payload + + +@pytest.mark.parametrize("policy", ["warn", "fail", "warn_then_fail"]) +def test_object_list_duplicates_unassessed(policy: str) -> None: + _, gate = fd.evaluate_quality_debt( + _list_frame(), debt_policy=policy, ledger=None, verbose=False + ) + _assert_duplicates_unassessed(gate) + assert gate.status == "pass" # undetectable duplicates never gate on their own + + +@pytest.mark.parametrize("kind", ["list", "struct"]) +def test_nested_arrow_duplicates_unassessed(kind: str) -> None: + _, gate = fd.evaluate_quality_debt( + _arrow_frame(kind), debt_policy="fail", ledger=None, verbose=False + ) + _assert_duplicates_unassessed(gate) + assert gate.status == "pass" + + +def test_object_dict_column_named_in_note() -> None: + df = pd.DataFrame({"id": [1, 2], "meta": [{"a": 1}, {"a": 1}], "tags": [["x"], ["y"]]}) + _, gate = fd.evaluate_quality_debt(df, ledger=None, verbose=False) + assert _item(gate, "duplicates").detail == ( + "duplicate rows could not be checked: column(s) 'meta', 'tags' hold unhashable values" + ) + + +def test_unhashable_note_truncates_long_column_lists() -> None: + df = pd.DataFrame({f"c{i}": [[i], [i]] for i in range(7)}) + assert _unhashable_note(df) == ( + "column(s) 'c0', 'c1', 'c2', 'c3', 'c4' and 2 more hold unhashable values" + ) + + +def test_removed_duplicates_stay_a_known_lower_bound() -> None: + """Rows removed via a hashable duplicate_subset are still counted.""" + _, gate = fd.evaluate_quality_debt( + _list_frame(), debt_policy="warn", ledger=None, verbose=False, + drop_duplicates=True, duplicate_subset=["id"], + ) + dup = _item(gate, "duplicates") + assert dup.assessed + assert dup.score == pytest.approx(1 / 3) + assert dup.over + assert dup.detail == ( + "1 duplicate row(s) removed; remaining duplicate rows could not be checked: " + "column(s) 'tags' hold unhashable values" + ) + + +def test_unassessed_duplicates_skip_the_ledger(tmp_path) -> None: + ledger = str(tmp_path / "debt.sqlite") + _, g1 = fd.evaluate_quality_debt( + _half_duplicated(), debt_policy="warn_then_fail", ledger=ledger, verbose=False + ) + assert g1.status == "warn" and _item(g1, "duplicates").over + _, g2 = fd.evaluate_quality_debt( + _list_frame(), debt_policy="warn_then_fail", ledger=ledger, verbose=False + ) + dup2 = _item(g2, "duplicates") + assert dup2.previous == pytest.approx(0.5) + assert not dup2.assessed and not dup2.worsening + assert g2.status == "pass" + conn = sqlite3.connect(ledger) + try: + rows = conn.execute( + "SELECT run_id FROM debt_items WHERE dimension='duplicates'").fetchall() + n_items = conn.execute("SELECT COUNT(*) FROM debt_items").fetchone()[0] + finally: + conn.close() + assert [r[0] for r in rows] == [1] # the unassessed run recorded no score + assert n_items == 9 + 8 + # The next measured run escalates from, and compares against, the last run + # that measured duplicates. + _, g3 = fd.evaluate_quality_debt( + _half_duplicated(), debt_policy="warn_then_fail", ledger=ledger, verbose=False + ) + dup3 = _item(g3, "duplicates") + assert dup3.previous == pytest.approx(0.5) + assert not dup3.worsening + assert g3.status == "fail" + + +def test_worsening_compares_against_last_measured_run(tmp_path) -> None: + ledger = str(tmp_path / "debt.sqlite") + _, g1 = fd.evaluate_quality_debt( + _half_duplicated(), debt_policy="warn", ledger=ledger, verbose=False + ) + first = _item(g1, "duplicates").score + fd.evaluate_quality_debt(_list_frame(), debt_policy="warn", ledger=ledger, verbose=False) + mostly_duplicated = pd.DataFrame({"a": [1] * 99 + [2], "b": ["x"] * 100}) + _, g3 = fd.evaluate_quality_debt( + mostly_duplicated, debt_policy="warn", ledger=ledger, verbose=False + ) + dup3 = _item(g3, "duplicates") + assert dup3.previous == pytest.approx(first) + assert dup3.score > first + assert dup3.worsening + + def test_pii_risk_counts_columns_not_matches() -> None: """#286: ten matching cells in one column are one PII column.""" df = pd.DataFrame({"email": [f"user{i}@x.com" for i in range(10)], "v": range(10)})