From 75c26d8199f19781daa1ff2a397ebca6c3a23952 Mon Sep 17 00:00:00 2001 From: igerber Date: Sat, 5 Sep 2026 18:40:08 -0400 Subject: [PATCH 1/2] Add WooldridgeDiD opt-out for comparison-support filtering --- TODO.md | 1 - ...05-wooldridge-unsupported-period-action.md | 7 + diff_diff/guides/llms-autonomous.txt | 18 +- diff_diff/guides/llms-full.txt | 24 +- diff_diff/guides/llms.txt | 2 +- diff_diff/wooldridge.py | 35 +++ diff_diff/wooldridge_results.py | 14 +- docs/api/wooldridge_etwfe.rst | 23 +- docs/doc-deps.yaml | 3 + docs/methodology/REGISTRY.md | 3 +- docs/migration-4.0.md | 11 +- docs/tutorials/16_wooldridge_etwfe.ipynb | 20 +- docs/v4-deprecations.yaml | 15 +- docs/v4-design.md | 8 +- tests/test_v4_matrix.py | 21 +- tests/test_wooldridge.py | 228 ++++++++++++++++++ 16 files changed, 400 insertions(+), 33 deletions(-) create mode 100644 changelog.d/20260905-wooldridge-unsupported-period-action.md diff --git a/TODO.md b/TODO.md index ac97adcaf..e3ef56384 100644 --- a/TODO.md +++ b/TODO.md @@ -56,7 +56,6 @@ Related tracking surfaces: | `ContinuousDiD` CGBS-2024 remaining extensions (earlier phases — `covariates=` reg/dr, `treatment_type="discrete"`, single-cohort `control_group="lowest_dose"` with estimand `ATT(d)−ATT(d_L)` — are already supported; see REGISTRY Note #7). Remaining (all deferred `NotImplementedError`, documented): `estimation_method="ipw"` on the dose curve (scalar-adjustment / degenerate); `covariates=` × `survey_design=` (weighted OR + weighted nuisance IF); multi-cohort **heterogeneous-support** discrete aggregation (support-aware: average each dose only over the cohorts that observe it); **multi-cohort `lowest_dose`** (within-cohort `d_L` reference + support-aware cross-cohort aggregation); and **`covariates=` × `lowest_dose`** (conditional-PT-relative-to-`d_L` estimand). Single-cohort / 2-period / shared-support multi-cohort are supported. | `continuous_did.py` | CGBS-2024 | Heavy | Low | | `WooldridgeDiD` does not apply the W2025 Sec 5.4 `D_{G_max} x X` covariate normalization, and three sibling covariate rank deficiencies are pre-existing. Measured with the period range pinned and only the never-treated units toggled: (1) time-invariant `exovar` is absorbed by the unit FE, 4 of 26 columns, IDENTICALLY with and without never-treated units; (2) `xgvar`'s cell x covariate block, 19 of 41, identical on both panels; (3) `xtvar` under `demean_covariates=False` does exhibit the `sum_g D_g x = x` dependency that the default demeaning removes; (4) the newly-reachable case -- time-VARYING data passed through `exovar`, which its own docstring reserves for time-invariant covariates -- where the paper's `dT_i` rule would give a deterministic `D_{G_max} x X` drop instead of QR's arbitrary pick (coefficients unaffected, `1.35e-14`; `rank_deficient_action="error"` raises). REGISTRY's narrowed Sec 5.4 note cross-references this row. **Trap for whoever takes it:** `xtvar` under the DEFAULT `demean_covariates=True` is FULL RANK -- the raw block carries demeaned values while `D_g x X` carries raw ones -- and forcing the drop there moves `overall_att` 1.11903 -> 1.46269. Pinned as-is by `TestComparisonSupportFiltering::test_cells_derived_groups_did_not_leak_into_the_design`. | `diff_diff/wooldridge.py` | #729-followup | Heavy | Medium | | `WooldridgeDiD.n_control_units` counts never-treated UNITS on `control_group="never_treated"` regardless of method, but on the nonlinear paths (`logit`/`poisson`) treated units' pre-treatment rows ARE the identifying comparison -- only the OLS path absorbs them into their own cells. So the reported count under-states the comparison pool exactly where the REGISTRY control-pool asymmetry note applies. Widen to `not_yet_treated or (never_treated and method != "ols")`, or document the count as never-treated-units-by-definition. Behavior is PRE-EXISTING; documented for now in the REGISTRY control-pool Note rather than changed, because widening moves a public results field and wants its own ledger row and test matrix. | `diff_diff/wooldridge.py` | #729-followup | Mid | Low | -| `WooldridgeDiD` has no opt-out for comparison-support period filtering: a user who would rather see the refusal than a reduced sample cannot ask for it. Adding one means a constructor parameter (`get_params`/`set_params` propagation, transactional validation), a ledger row, and a test matrix across both predicate branches and all three `rank_deficient_action` modes -- deliberately out of scope for the change that introduced the filter. The always-on warning is the interim answer. | `diff_diff/wooldridge.py` | #729-followup | Mid | Low | | Bad-control imputation estimator (Caetano et al. 2026 Section 6.1, Eqs. 5-7, S8 influence function) on a CallawaySantAnna `estimation_method="reg"` host: two untreated-sample OLS fits per cell plus the generated-regressor IF line. | `staggered.py` | bad-controls PR-B | Heavy | Low | | Bad-control SC "parallel trends for X" variant (Caetano et al. 2026 Section 7 / S17): a linearity-based alternative to covariate unconfoundedness with its own estimand. | `dml_did.py` | bad-controls PR-B | Heavy | Low | | `ATT_X(g,t)` event-study aggregation + bootstrap replay for the bad-control pre-test (today analytical per-cell only; never aggregated). | `dml_did_results.py` | bad-controls PR-B | Mid | Low | diff --git a/changelog.d/20260905-wooldridge-unsupported-period-action.md b/changelog.d/20260905-wooldridge-unsupported-period-action.md new file mode 100644 index 000000000..b7ef6aefb --- /dev/null +++ b/changelog.d/20260905-wooldridge-unsupported-period-action.md @@ -0,0 +1,7 @@ +### Added +- **WooldridgeDiD comparison-support policy** ([M-147]): set + `unsupported_period_action="error"` to refuse periods lacking eligible comparison + support before removing them. The default `"drop"` preserves filtering and warnings. + The option works across OLS, logit and Poisson independently of + `rank_deficient_action`; results record the fit-time policy in `summary()` and + `to_dict()`. Existing survey and identification checks remain active. diff --git a/diff_diff/guides/llms-autonomous.txt b/diff_diff/guides/llms-autonomous.txt index b87729b8a..cdbbb663f 100644 --- a/diff_diff/guides/llms-autonomous.txt +++ b/diff_diff/guides/llms-autonomous.txt @@ -563,13 +563,14 @@ When `has_never_treated == False`: - `ChaisemartinDHaultfoeuille` - constructs switchers vs. non-switchers directly; no never-treated requirement. - TWFE / `MultiPeriodDiD` / `ImputationDiD` / `TwoStageDiD` / - `StackedDiD` / `WooldridgeDiD` - use the last-treated or untreated- + `StackedDiD` - use the last-treated or untreated- until-late units as implicit controls; estimators do not error, but consider whether the implicit control structure is what you want. - `WooldridgeDiD` specifically: use `control_group="not_yet_treated"` (the default). `control_group="never_treated"` raises when no - cohort-0 units exist. On an all-eventually-treated panel the last - cohort becomes the reference per W2025 Section 5.4, so periods at + cohort-0 units exist. With the default `unsupported_period_action="drop"`, + the last cohort becomes the reference on an all-eventually-treated panel + per W2025 Section 5.4, so periods at which every unit is treated carry no identified ATT(g, t) and are REMOVED from the estimation sample before the solve. The fit emits a `UserWarning` naming the dropped periods, the observation count and @@ -578,6 +579,17 @@ When `has_never_treated == False`: cohorts. If your agent surfaces warnings to a user, surface these: the estimate is computed on fewer rows than were supplied. Stata `jwdid` performs the same reduction but reports only a smaller `N`. + Use `WooldridgeDiD(unsupported_period_action="error")` when the user prefers + refusal to automatic period filtering: it raises `ValueError` naming the + unsupported periods and affected observation count before removal. A period + lacks support when no positive-weight eligible comparison is observed: + never-treated rows only on OLS + `never_treated`, and also rows before + `g - anticipation` on other paths. The policy applies to all methods, + independently of `rank_deficient_action`; other identification checks and + unidentified-cohort exclusion remain active. With `survey_design`, design + validation still runs first: `"error"` then raises `ValueError`, while + `"drop"` retains the survey-domain `NotImplementedError` when periods would + be removed. Fully supported survey fits are unaffected. SOME covariate specifications on such a panel are still rank-deficient (`exovar`, `xgvar`, and `xtvar` with `demean_covariates=False`), because `D_{G_max} x X` is not normalized; `rank_deficient_action="error"` diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 8195c2e19..c9cf99291 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1603,13 +1603,25 @@ WooldridgeDiD( # carries G-1 entries. Rejected with survey_design= and # with control_group="never_treated". df_convention: str = "residual", # OLS analytical t/p/CI df (3.9: default-hc1 arms moved z -> t(residual)): "residual" (default), "cluster" (G-1, hc1-clustered only), "normal" (z); survey/BM DOF keep precedence; GLM arms knob-independent (explicit non-default warns); flips at v4 + unsupported_period_action: str = "drop", # "drop" warns and filters; "error" refuses unsupported periods ) ``` `fit()` additionally accepts `survey_design=` (a `SurveyDesign`) on all three methods; see the Survey Support section. -**All-eventually-treated panels (no never-treated group).** Use +**Comparison-support policy.** `unsupported_period_action="drop"` preserves +period filtering and its warnings. Set `unsupported_period_action="error"` +to raise `ValueError` naming the unsupported periods and affected observation +count before removal. Support requires positive-weight never-treated rows on +OLS + `control_group="never_treated"`; elsewhere not-yet-treated rows before +`g - anticipation` also qualify. This is independent of `rank_deficient_action` +and does not control unidentified-cohort exclusion or bypass identification checks. +The results record the fit-time policy in `unsupported_period_action`, `to_dict()` +and `summary()`, including after aggregation or estimator reconfiguration. + +**All-eventually-treated panels (no never-treated group).** With the default +`unsupported_period_action="drop"`, use `control_group="not_yet_treated"` — `"never_treated"` raises when no cohort-0 units exist. Periods at which every unit is treated carry no identified ATT(g, t), so they are REMOVED from the estimation sample before the solve and @@ -1625,11 +1637,13 @@ unaffected either way. Default `xtvar` (`demean_covariates=True`) is FULL RANK and fits cleanly. **Survey designs refuse row-dropping paths.** `survey_design=` combined with -either comparison-support period filtering or unidentified-cohort exclusion -raises `NotImplementedError` rather than deleting rows, because deletion would +either comparison-support period filtering (`unsupported_period_action="drop"`) +or unidentified-cohort exclusion raises `NotImplementedError` rather than deleting rows, because deletion would remove their PSUs and strata from the TSL variance. Restrict the frame yourself -and re-fit. The refusals are conditional: survey fits that drop nothing are -unaffected. +and re-fit only after confirming every PSU and stratum survives that restriction. +With `unsupported_period_action="error"`, unsupported periods instead raise +`ValueError`, after existing input/design validation. The refusals are conditional: +survey fits that drop nothing are unaffected. **Alias:** `ETWFE` diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index 2ffd140af..517e43419 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -76,7 +76,7 @@ The site is organized into 5 sections, each with a landing page: - [EfficientDiD](https://diff-diff.readthedocs.io/en/stable/api/efficient_did.html): Chen, Sant'Anna & Xie (2025) efficient DiD with optimal weighting for tighter SEs - [TROP](https://diff-diff.readthedocs.io/en/stable/api/trop.html): Triply Robust Panel estimator (Athey et al. 2025) with nuclear norm factor adjustment (absorbing by default; `non_absorbing=True` for on/off treatment, method='local') - [StaggeredTripleDifference](https://diff-diff.readthedocs.io/en/stable/api/staggered.html#staggeredtripledifference): Ortiz-Villavicencio & Sant'Anna (2025) staggered DDD with group-time ATT. DEPRECATED in 3.9, removed in 4.0 - use `TripleDifference` with `first_treat=` (supplying the unit id, the calendar period column and `partition=`) - the same engine (`eligibility=` is `partition=` there; `control_group` takes the underscored values). Alias `SDDD` deprecated with it -- [WooldridgeDiD](https://diff-diff.readthedocs.io/en/stable/api/wooldridge_etwfe.html): Wooldridge (2023, 2025) ETWFE — saturated OLS, logit/Poisson QMLE (ASF-based ATT). Alias: ETWFE +- [WooldridgeDiD](https://diff-diff.readthedocs.io/en/stable/api/wooldridge_etwfe.html): Wooldridge (2023, 2025) ETWFE — saturated OLS, logit/Poisson QMLE (ASF-based ATT); `unsupported_period_action="error"` refuses comparison-support period filtering (default `"drop"`). Alias: ETWFE - [LPDiD](https://diff-diff.readthedocs.io/en/stable/api/lpdid.html): Dube, Girardi, Jorda & Taylor (2025) Local Projections DiD: per-horizon long-difference event study on clean controls (no negative weighting); variance- or equally-weighted ATT, premean differencing, pooled pre/post, fast. Absorbing by default; non-absorbing (reversible) treatment via `non_absorbing="first_entry"` (Eq. 12) or `"effect_stabilization"` (Eq. 13, window `L`). Complex-survey designs (pweight + stratified-PSU TSL SEs) on the default path via `fit(survey_design=...)`. - [ChangesInChanges](https://diff-diff.readthedocs.io/en/stable/api/changes_in_changes.html): Athey & Imbens (2006) nonlinear/distributional DiD for the 2x2 design: recovers the treated group's full counterfactual outcome distribution and quantile treatment effects (ATT + QTE grid) via the CDF transformation `F_10(F_00^{-1}(F_01(y)))`; invariant to monotone outcome transformations (unconditional fits; the covariate QR branch is not); bootstrap inference (panel or repeated cross-section resampling); point parity with R `qte::CiC()`, including its covariate branch (`covariates=` -> per-cell linear quantile regression, Melly-Santangelo-style conditional CiC). Continuous outcomes, numeric covariates. Alias `CiC`. - [QDiD](https://diff-diff.readthedocs.io/en/stable/api/changes_in_changes.html): **Deprecated 3.9, removed 4.0 - use `ChangesInChanges(method="qdid")`.** Athey & Imbens (2006) quantile DiD comparison estimator (additive quantile-by-quantile DiD, matching R `qte::QDiD()` including its covariate branch via `covariates=`); same bootstrap machinery as ChangesInChanges. The paper recommends CiC over QDiD (scale-dependent model with testable restrictions; a non-monotonicity warning fires when violated - unconditional fits only, the covariate-path counterfactual quantile curve is monotone by construction). diff --git a/diff_diff/wooldridge.py b/diff_diff/wooldridge.py index fac91a168..289fa6a48 100644 --- a/diff_diff/wooldridge.py +++ b/diff_diff/wooldridge.py @@ -971,6 +971,18 @@ class WooldridgeDiD(BaseEstimator): always take precedence; the logit/poisson arms are knob-independent (survey df or normal theory — an explicitly non-default value warns at fit time). The default flips to ``"cluster"`` at v4. + unsupported_period_action : {"drop", "error"}, default "drop" + How to handle periods lacking the required comparison support. + ``"drop"`` removes those periods before estimation and warns; + ``"error"`` raises ``ValueError`` before removing them. Support + requires a positive-weight never-treated observation on OLS with + ``control_group="never_treated"``; other paths also admit observations + before ``g - anticipation``. This policy is independent of + ``rank_deficient_action`` and does not control unidentified-cohort + exclusion. With ``survey_design``, ``"drop"`` still raises + ``NotImplementedError`` if periods would be removed, because survey + domain estimation is not supported; ``"error"`` raises ``ValueError`` + after the existing input and survey validation. """ def __init__( @@ -993,6 +1005,7 @@ def __init__( conley_kernel: str = "bartlett", conley_lag_cutoff: Optional[int] = None, df_convention: str = "residual", + unsupported_period_action: str = "drop", ) -> None: self._validate_constructor_args( method=method, @@ -1022,6 +1035,8 @@ def __init__( self.conley_kernel = conley_kernel self.conley_lag_cutoff = conley_lag_cutoff self.df_convention = df_convention + self._validate_unsupported_period_action(unsupported_period_action) + self.unsupported_period_action = unsupported_period_action # Track whether the user explicitly opted out of the "hc1" default. # The auto-cluster-at-unit default in `_fit_ols` is suppressed only # when the user explicitly opts into a one-way family (``hc2``, @@ -1033,6 +1048,12 @@ def __init__( self.is_fitted_: bool = False self._results: Optional[WooldridgeDiDResults] = None + @staticmethod + def _validate_unsupported_period_action(value: str) -> None: + """Validate the period policy without coercing non-string values.""" + if not isinstance(value, str) or value not in ("drop", "error"): + raise ValueError(f"unsupported_period_action must be 'drop' or 'error', got {value!r}") + @staticmethod def _validate_constructor_args( *, @@ -1152,6 +1173,7 @@ def fit( # mutated and passed the deprecated kwarg still sees the # FutureWarning before the raise. self.anticipation = validate_anticipation(self.anticipation) + self._validate_unsupported_period_action(self.unsupported_period_action) df = data.copy() df = _warn_and_fill_nan_cohort(df, cohort, stacklevel=2) @@ -1501,6 +1523,18 @@ def fit( } if _unsupported_periods: + if self.unsupported_period_action == "error": + _n_unsupported = int(sample[time].isin(_unsupported_periods).sum()) + _plabels = ", ".join(str(t) for t in _unsupported_periods) + raise ValueError( + f"Period(s) {_plabels} have no eligible comparison group " + f"and contain {_n_unsupported} of {len(sample)} observations. " + "unsupported_period_action='error' refuses the fit before " + "removing these unsupported periods. Use " + "unsupported_period_action='drop' to permit automatic " + "filtering (unavailable with survey_design), or supply data " + "with the required comparison support." + ) if survey_design is not None: # Same naive-subsetting problem the unidentified-cohort path # refuses below: deleting rows removes their PSUs and strata @@ -2015,6 +2049,7 @@ def _build(frame: pd.DataFrame, w: Optional[np.ndarray]): and _pre_filter_unit_counts[g] > results._n_g_per_cohort[g] } + results.unsupported_period_action = self.unsupported_period_action self._results = results self.is_fitted_ = True return results diff --git a/diff_diff/wooldridge_results.py b/diff_diff/wooldridge_results.py index 9612096d7..c03ceaf50 100644 --- a/diff_diff/wooldridge_results.py +++ b/diff_diff/wooldridge_results.py @@ -36,6 +36,8 @@ class WooldridgeDiDResults(BaseResults): ``aggregation_weights`` is keyed by aggregation type and records the active weighting scheme that wrote to each cached surface (surfaced in ``summary()`` / ``to_dataframe()`` / ``__repr__``). + ``unsupported_period_action`` records the fit's policy for periods + lacking comparison support (``"drop"`` or ``"error"``). """ # ------------------------------------------------------------------ # @@ -215,8 +217,12 @@ class WooldridgeDiDResults(BaseResults): df_convention: Optional[str] = None """The estimator's ``df_convention`` configuration echoed onto the results ("residual" | "cluster" | "normal"; added 3.9). Governs the OLS - analytical arms only; GLM inference is knob-independent. Appended LAST - (the generated ``__init__`` positional indexes are public API).""" + analytical arms only; GLM inference is knob-independent.""" + unsupported_period_action: str = "drop" + """Fit-time comparison-support period policy (``"drop"`` or ``"error"``). + Appended last to preserve generated constructor positional indexes. + ``"error"`` fits only when no period needs comparison-support filtering; + other identification guards, including cohort exclusion, still apply.""" # ------------------------------------------------------------------ # # Public methods # @@ -234,6 +240,7 @@ def __setstate__(self, state: Dict[str, Any]) -> None: classical/hc2 inference (legacy hc1 pickles carried None there, which restores the pre-3.9 normal-theory aggregation for them). ``df_convention`` (added 3.9) defaults to ``"residual"``. + Missing ``unsupported_period_action`` defaults to ``"drop"``. """ self.__dict__.update(state) # NOTE: check __dict__ membership, not hasattr - dataclass field @@ -243,6 +250,7 @@ def __setstate__(self, state: Dict[str, Any]) -> None: self.__dict__.pop("_df_one_way", None) if "df_convention" not in self.__dict__: self.__dict__["df_convention"] = "residual" + self.__dict__.setdefault("unsupported_period_action", "drop") # M-086: pickles written before the "event" -> "event_study" value # unification carry only the old aggregation_weights key; mirror it # so both spellings resolve during the 3.9 window. @@ -733,6 +741,7 @@ def summary(self, aggregation: Any = NOT_SUPPLIED, *, alpha: Optional[float] = N "=" * 70, f"Method: {self.method}", f"Control group: {self.control_group}", + f"Unsupported period action: {self.unsupported_period_action}", f"Observations: {self.n_obs}", f"Treated units: {self.n_treated_units}", f"Control units: {self.n_control_units}", @@ -862,6 +871,7 @@ def to_dict(self) -> Dict[str, Any]: "conf_int_upper": self.overall_conf_int[1], "method": self.method, "control_group": self.control_group, + "unsupported_period_action": self.unsupported_period_action, "n_obs": self.n_obs, "n_treated_units": self.n_treated_units, "n_control_units": self.n_control_units, diff --git a/docs/api/wooldridge_etwfe.rst b/docs/api/wooldridge_etwfe.rst index dbcb911fe..e4e2d28b5 100644 --- a/docs/api/wooldridge_etwfe.rst +++ b/docs/api/wooldridge_etwfe.rst @@ -43,6 +43,22 @@ WooldridgeDiD Main estimator class for Wooldridge ETWFE. +``unsupported_period_action="drop"`` (default) removes periods lacking the +required comparison support and warns. Use +``WooldridgeDiD(unsupported_period_action="error")`` to raise ``ValueError`` +before those periods are removed; the error names the periods and affected +observation count. This applies to all three methods, independently of +``rank_deficient_action``. It does not control unidentified-cohort exclusion +or relax other identification checks. + +Support requires a positive-weight never-treated observation for OLS with +``control_group="never_treated"``. All other paths also admit observations +before their cohort's ``g - anticipation`` threshold. Unsupported periods +can therefore occur even when never-treated units exist elsewhere in the panel. +With ``survey_design``, input/design validation runs first; ``"error"`` then +raises ``ValueError`` for unsupported periods, while ``"drop"`` retains the +existing ``NotImplementedError`` because survey domain estimation is unsupported. + .. autoclass:: diff_diff.WooldridgeDiD :no-index: :members: @@ -63,6 +79,10 @@ WooldridgeDiDResults Results container returned by ``WooldridgeDiD.fit()``. +``unsupported_period_action`` records the fit-time policy and is included in +``to_dict()`` and ``summary()``. It remains unchanged by post-fit aggregation +or subsequent estimator reconfiguration. + ``cohort_trend_coefs`` (populated under ``cohort_trends=True``, OLS path only): ``Dict[g → δ_g]`` keyed by treated cohort. The reported slopes are **relative to the baseline trend** absorbed by the design — the @@ -74,7 +94,8 @@ dict; its slope is the baseline (zero in deviation form). .. note:: - All-eventually-treated panels **estimate**. The paper's Section 5.4 + With the default ``unsupported_period_action="drop"``, all-eventually-treated + panels **estimate**. The paper's Section 5.4 rule is applied to the cohort × time cells as well as the trend columns: periods where no unit is untreated carry no identified ATT(g, t), so they are removed from the estimation sample before the diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index 66047f445..efaa0ab86 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -786,6 +786,9 @@ sources: - path: diff_diff/guides/llms-full.txt section: "WooldridgeDiD" type: user_guide + - path: diff_diff/guides/llms-autonomous.txt + section: "§4.4 No never-treated group" + type: user_guide - path: diff_diff/guides/llms.txt section: "Estimators" type: user_guide diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index a2718955f..2b33baf9f 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -2461,7 +2461,8 @@ The interaction coefficient `δ_{g,t}` identifies `ATT(g, t)` under parallel tre - **Note (reference-period normalization, W2025 Eq. 6.1/6.4 — `never_treated` OLS only):** this applies to the **lead-and-lag** specification, i.e. `control_group="never_treated"` on the OLS path, which is the only configuration that emits pre-treatment cells (`include_pre`). The default `not_yet_treated` (and both nonlinear paths) emit `t >= g − anticipation` cells only — the lag-only specification — so no cell is omitted from those designs and no placebo cells are produced; the reference period is still computed there, but solely to detect cohorts that have none (see the unidentified-cohort Note). For the lead-and-lag specification, for each treated cohort `g` the design OMITS one cell — the reference period — because the cohort's full cell block spans the cohort indicator `1{G_i = g}`, which the unit fixed effects absorb. The paper is explicit: Eq. 6.1 excludes `dg_i · f(g-1)_t` "so that `s = g − 1` is the reference period", and Eq. 6.4 describes the saturated regression as a collection of 2×2 DiDs using `g − 1` as reference. The library omits `ref(g) = max{t : t < g − anticipation AND cohort g is OBSERVED at t}` — `g − 1` on a balanced panel, and the cohort's own latest available pre-period otherwise. The support must be **per-cohort**: a panel-wide rule omits an identically-zero column whenever that cohort is unobserved at the panel's latest pre-period, leaving the collinearity intact. Anchored to Stata `jwdid ... never`, which omits the same cell for every cohort (`tests/test_etwfe_cs_stata_parity.py::TestNeverTreatedVsStataJwdid`). **Before v3.9 the reference was emitted and generic QR rank detection dropped an arbitrary column instead** — on `mpdta` that silently removed two genuine post-treatment effects, `(2004, 2004)` and `(2006, 2007)` (issue #724). - **Note (reference-period sensitivity, W2025 Section 6.1):** the choice of reference is a NORMALIZATION, not an identifying assumption. The paper notes any set of pre-treatment periods may serve, and the pre-trend `t`-test is numerically identical whichever is used. `g − 1` is the paper's canonical choice and the one this library implements. **The point estimates are NOT invariant to it** — only the two-sided pre-trend test is (Section 6.1). Different references give different finite-sample 2×2 contrasts, all consistent for ATT(g, t) under parallel trends: on `mpdta`, ATT(2007, 2007) computed by hand against references 2006/2005/2004/2003 gives −0.0261 / −0.0571 / −0.0599 / −0.0294. This is exactly why issue #724 mattered — QR dropped `g2007_t2005`, silently making 2005 the reference, and the estimator returned −0.0571: a correct 2×2 DiD against the wrong baseline. It also means the unbalanced-panel fallback (the cohort's latest AVAILABLE pre-period) yields a different contrast from `g − 1`, which is legitimate but should be read as such. - **Note (unidentified cohorts — library identification limit):** a cohort with NO observed period before `g − anticipation` has no reference cell, so none of its ATTs are identified. The library warns naming the cohort and **excludes its observations** from the estimation sample. *(Conditional since comparison-support filtering landed: when the filter removes that cohort's rows first, the cohort never reaches this path and is instead named by the zero-cell / fully-dropped warning above. The observations are excluded either way; only which warning names them differs.)* Excluding only its columns would be worse than the original bug: `_filter_sample` retains every treated row, so the cohort would join the omitted baseline beside the controls and load its treatment effect onto the time fixed effects — measured on `mpdta` at `anticipation=1`, that moved `ATT(2006, 2006)` by 0.0077 and `ATT(2006, 2007)` by 0.0055, past the 5e-3 this library treats as material. Exclusion can cascade (the removed rows may have been another cohort's not-yet-treated comparison), so the surviving sample is re-checked and `fit()` raises when no comparison observations remain rather than returning an all-NaN fit. This is a **library limit, not the paper's last-cohort result** — W2025's "the last cohort `T` plays the role of the never-treated group" is conditioned on there being no never-treated group at all, a different case. -- **Note (per-period comparison support, W2025 Section 5.4 — the cell half):** a period at which **no unit is untreated** carries no identified `ATT(g, t)`, because there is no untreated outcome to difference against. Such periods are removed from the estimation sample **before the solve**. The eligible set is keyed on the regression baseline, not on the method: on the lead-and-lag branch (`never_treated` + OLS) only never-treated rows qualify, because every `(g, t)` except each cohort's reference is emitted, so a later cohort's rows sit in their own indicator; elsewhere not-yet-treated rows qualify too. **The omitted reference cells do NOT count as support** even though they sit in the baseline: cohort `h`'s indicator is absorbed by the unit FE (`1{h,t} = D_h − Σ_{t'≠t} h_{t'}`), so the period dummy stays reproducible from the emitted cells and the design remains collinear — measured on a panel with never-treated units through `t=4` and cohorts 3, 6 through `t=6`, retaining `t=5` on the strength of `ref(6)=5` raises on the lost `(6,1)`, while dropping it fits at `overall_att = 1.0171`. On an all-eventually-treated panel this yields exactly Eq. 5.15's cell set: cohort `G_max` is the reference and receives nothing. Anchored to Stata `jwdid` (`tests/test_etwfe_cs_stata_parity.py::TestAllEventuallyTreatedVsStataJwdid`): identical cell set, identical `N` (764 of 955), ATTs agreeing to ~1e-15. +- **Note (per-period comparison support, W2025 Section 5.4 — the cell half):** a period at which **no unit is untreated** carries no identified `ATT(g, t)`, because there is no untreated outcome to difference against. With the default `unsupported_period_action="drop"`, such periods are removed from the estimation sample **before the solve**. The eligible set is keyed on the regression baseline, not on the method: on the lead-and-lag branch (`never_treated` + OLS) only never-treated rows qualify, because every `(g, t)` except each cohort's reference is emitted, so a later cohort's rows sit in their own indicator; elsewhere not-yet-treated rows qualify too. **The omitted reference cells do NOT count as support** even though they sit in the baseline: cohort `h`'s indicator is absorbed by the unit FE (`1{h,t} = D_h − Σ_{t'≠t} h_{t'}`), so the period dummy stays reproducible from the emitted cells and the design remains collinear — measured on a panel with never-treated units through `t=4` and cohorts 3, 6 through `t=6`, retaining `t=5` on the strength of `ref(6)=5` raises on the lost `(6,1)`, while dropping it fits at `overall_att = 1.0171`. On an all-eventually-treated panel this yields exactly Eq. 5.15's cell set: cohort `G_max` is the reference and receives nothing. Anchored to Stata `jwdid` (`tests/test_etwfe_cs_stata_parity.py::TestAllEventuallyTreatedVsStataJwdid`): identical cell set, identical `N` (764 of 955), ATTs agreeing to ~1e-15. +- **Note:** Comparison-support filtering is configurable via `unsupported_period_action="drop" | "error"` (ledger M-147). The default `"drop"` preserves the reduction and warnings described here. `"error"` raises `ValueError` before period removal or design construction, naming unsupported periods and their observation count; it does not fit an unidentified full-sample design. The same positive-weight, anticipation-aware predicate applies on OLS, logit and Poisson, independently of `rank_deficient_action`. Existing input/survey validation runs first; for valid survey designs with unsupported periods, `"error"` raises `ValueError` and `"drop"` retains the survey-domain `NotImplementedError`. Other identification guards and unidentified-cohort exclusion are unchanged. Successful results retain the configured policy through aggregation and serialization and expose it via `summary()` and `to_dict()`. - **Note (the reduction is always reported — deviation from `jwdid`):** Stata `jwdid` performs the same reduction **silently**, reporting only a smaller `N`. This library warns on every fit that drops rows, naming the periods, the observation count, and the branch-correct cause, and separately naming any cohort left with no estimated cells or stripped of every row. Neither warning is gated on `rank_deficient_action` — that setting governs how rank warnings surface, not whether the estimation sample changed. Being explicit is a deliberate improvement, not a numerical deviation: the estimates are identical. - **Note (reference movement under filtering):** on the `not_yet_treated` branch a reference period can never be filtered — reference eligibility (`t < g − anticipation`) IS the support predicate's own second disjunct evaluated at that cohort's rows — and the estimator asserts this at runtime, raising if it is ever violated (silent renormalization is the issue #724 defect class). On the `never_treated` + OLS branch it CAN move, because eligibility there does not imply never-treated presence: measured, dropping `t=5` moves `ref(6)` from 5 to 4. That is legitimate — the ATTs stay correctly labelled and validly identified, just normalized against a different baseline period — so it **warns** naming the cohort and both periods rather than raising. A reference that becomes `None` is not a move: that cohort is unidentified and the exclusion path already warns by name. - **Note (zero-cell cohorts are warned, not raised):** a cohort retained only as a control — cohort `G_max` under the Section 5.4 normalization — or one that loses every row to filtering produces no `ATT(g, t)`. `fit()` reports it by name and continues. `results.groups` and `_n_g_per_cohort` are derived from the **emitted cell set** so such a cohort is not advertised as estimated (`set(results.groups) == {g for (g, t) in group_time_effects}`). `cohort_trend_coefs` is keyed on PRESENT cohorts instead, because the trend baseline is exactly the zero-cell cohort; the two therefore differ by design, and the invariant on that dict is `set(cohort_trend_coefs) ⊆ present cohorts`. diff --git a/docs/migration-4.0.md b/docs/migration-4.0.md index dc34ddbfa..e39226496 100644 --- a/docs/migration-4.0.md +++ b/docs/migration-4.0.md @@ -223,8 +223,15 @@ each one with its target. Smaller items that do not fit the families above — two inert `SyntheticDiD` constructor parameters, the `covariates=` constructor-to-`fit()` move, a retired transition warning, the Bacon roster re-homing, and the family-wide `anticipation` validation below. The appendix lists -the ledger-derived removals/flips; [M-144], [M-145], and [M-146] are behavior tightenings with -no removal/deprecation fields and appear here only. +the ledger-derived removals/flips; [M-144], [M-145], and [M-146] are behavior tightenings, +and [M-147] is an additive policy. These have no removal/deprecation fields and appear here only. + +- `WooldridgeDiD(unsupported_period_action="error")` ([M-147]) refuses periods lacking + comparison support before removing them. The default `"drop"` preserves filtering and + warnings, so existing callers need no migration. The policy applies to all three methods + independently of `rank_deficient_action`; other identification checks still apply. + Survey validation runs first, then `"error"` raises `ValueError` for unsupported periods; + `"drop"` retains the survey-domain `NotImplementedError` when removal would be required. - `anticipation` is validated across the family ([M-144], landing at 4.0): whole-valued floats that previously fit identically to their integer now raise — pass the `int`; bool and diff --git a/docs/tutorials/16_wooldridge_etwfe.ipynb b/docs/tutorials/16_wooldridge_etwfe.ipynb index b0af71c66..d08e0e091 100644 --- a/docs/tutorials/16_wooldridge_etwfe.ipynb +++ b/docs/tutorials/16_wooldridge_etwfe.ipynb @@ -1198,11 +1198,24 @@ "\n", "Wooldridge (2025) Section 5.4 gives the answer: the **last cohort serves as the\n", "reference**, and \"all variables in regression (5.3) involving `dT_i` get\n", - "dropped\". `WooldridgeDiD` implements that — it removes the unsupported periods\n", + "dropped\". `WooldridgeDiD` implements that with the default\n", + "`unsupported_period_action=\"drop\"`: it removes the unsupported periods\n", "before estimating and tells you it did.\n", "\n", "Use `control_group=\"not_yet_treated\"` (the default); `\"never_treated\"` raises\n", - "when there are no never-treated units to use.\n" + "when there are no never-treated units to use.\n", + "\n", + "To refuse automatic period filtering, use\n", + "`WooldridgeDiD(unsupported_period_action=\"error\")`. It raises `ValueError`\n", + "with the unsupported periods and affected observation count before removal.\n", + "Support requires positive-weight never-treated observations on OLS with\n", + "`control_group=\"never_treated\"`; other paths also admit not-yet-treated\n", + "observations before `g - anticipation`. This applies to all three methods,\n", + "independently of `rank_deficient_action`, and leaves other identification\n", + "checks and unidentified-cohort exclusion active. With `survey_design`, design\n", + "validation runs first: `\"error\"` then raises `ValueError` for unsupported\n", + "periods; `\"drop\"` still raises `NotImplementedError` because domain estimation\n", + "is not supported.\n" ] }, { @@ -1302,7 +1315,7 @@ "3. **Nonlinear paths** (Poisson, Logit) use the ASF formula: E[f(η₁)] − E[f(η₀)] — the only valid ATT definition for nonlinear models\n", "4. **Four aggregations** mirror Stata's `estat` commands: event, group, calendar, simple\n", "5. **Delta-method SEs** for all aggregations, including nonlinear paths\n", - "6. **All-eventually-treated panels** estimate via the Section 5.4 normalization: the last cohort is the reference, unsupported periods are dropped, and the reduction is always reported\n", + "6. **All-eventually-treated panels** estimate under default `unsupported_period_action=\"drop\"` via the Section 5.4 normalization: the last cohort is the reference, unsupported periods are dropped, and the reduction is always reported\n", "7. **When to prefer ETWFE**: nonlinear outcomes, or when a single-regression framework is preferred\n", "8. **When to prefer CS/ImputationDiD**: covariate adjustment via IPW/DR, or multiplier bootstrap inference\n", "\n", @@ -1312,6 +1325,7 @@ "|-----------|---------|-------------|\n", "| `method` | `'ols'` | `'ols'`, `'poisson'`, or `'logit'` |\n", "| `control_group` | `'not_yet_treated'` | `'not_yet_treated'` or `'never_treated'` |\n", + "| `unsupported_period_action` | `'drop'` | `'drop'` filters unsupported periods and warns; `'error'` refuses before removal |\n", "| `anticipation` | `0` | Anticipation periods before treatment |\n", "| `alpha` | `0.05` | Significance level |\n", "| `cluster` | `None` | Column for clustering (default: unit variable) |\n", diff --git a/docs/v4-deprecations.yaml b/docs/v4-deprecations.yaml index 45b7fb7dd..a49c6a84d 100644 --- a/docs/v4-deprecations.yaml +++ b/docs/v4-deprecations.yaml @@ -1465,7 +1465,7 @@ rows: phase: 2 test_ref: tests/test_wooldridge.py code_refs: [diff_diff/wooldridge.py, docs/methodology/REGISTRY.md, tests/test_etwfe_cs_stata_parity.py] - notes: "Per-period COMPARISON-SUPPORT filtering: WooldridgeDiD.fit() now removes periods at which no unit is untreated from the estimation sample BEFORE the solve, instead of carrying them into a rank-deficient design. This is the cell half of W2025 Section 5.4 - with no never-treated group the last cohort serves as the reference, and 'all variables in regression (5.3) involving dT_i get dropped' - so all-eventually-treated panels ESTIMATE again, resolving the capability regression [M-124] recorded. Measured on a {3,5,8} panel over t=1..9 with a true ATT of 1.5: cells (3,3..7) and (5,5..7), overall_att=1.5502, cohort 8 receiving none. Anchored externally to Stata jwdid on the mpdta panel with never-treated counties dropped (golden block jwdid_alltreated): identical cell set, identical N (764 of 955), ATTs agreeing to ~1e-15. The eligible set is keyed on the regression BASELINE, not the method: on never_treated + OLS only never-treated rows qualify, because include_pre emits every (g,t) except each cohort's reference so a later cohort's rows sit in their own indicator; elsewhere not-yet-treated rows qualify too. Omitted reference cells deliberately do NOT count as support - they sit in the baseline but do not identify the period, since the cohort indicator is absorbed by the unit FE and the period dummy stays reproducible from the emitted cells (measured: retaining a period on the strength of ref(6)=5 raises on the lost (6,1), while dropping it fits at overall_att=1.0171). SAMPLE REDUCTION IS ALWAYS REPORTED, in two parts and neither gated on rank_deficient_action: at filter time the periods, observation count and branch-correct cause (emitted there so it survives a downstream raise), and after the final build any cohort left with no cells or stripped of every row (read from the FINAL cell set so a cohort that unidentified-cohort exclusion also removes is not double-reported). Stata jwdid performs the same reduction silently, reporting only a smaller N; being explicit is a deliberate improvement, not a numerical deviation. REFERENCE MOVEMENT: on the not_yet_treated branch a reference can never be filtered (eligibility t < g - anticipation IS the predicate's own second disjunct) and the estimator asserts this at runtime; on never_treated + OLS it CAN move, which renormalizes that cohort's ATTs against a different baseline period, so that case warns naming the cohort and both periods rather than raising. RESULTS METADATA: results.groups and _n_g_per_cohort are now derived from the emitted cell set so a cohort retained only as a control is not advertised as estimated; cohort_trend_coefs stays keyed on PRESENT cohorts because the trend baseline is exactly the zero-cell cohort. The design-side readers of the cohort list (covariate blocks, trend block) deliberately keep the present-cohort list - routing them through the cells-derived one would drop D_{G_max} x X and silently apply a covariate normalization this row does NOT include (measured: 8/8 ATTs move, max 0.0648). Under survey_design= the filter REFUSES rather than subsetting, conditional on rows actually dropping, for the same reason as [M-123]. COHORT-SHARE BOUNDARY: `_n_g_per_cohort` (N_g, Eqs. 7.4/7.6) is read off the FINAL sample, and this filter is the first thing in the estimator that can remove SOME units of a RETAINED cohort - `_filter_sample` keeps every row of every treated unit and unidentified-cohort exclusion removes whole cohorts, so before this N_g was always the full supplied cohort. On an UNBALANCED panel a unit observed only at unsupported periods vanishes with them and N_g then describes a strictly smaller cohort (measured: 90 of a cohort's 100 units observed only at a dropped period moves aggregate(weights='cohort_share') from 1.8078 to 3.8157). W2025 Section 7 assumes a balanced panel and does not say which reading applies, so aggregate() FAILS CLOSED naming the cohorts and unit counts rather than choosing silently, mirroring the existing survey + cohort_share refusal. weights='cell' never reads N_g and is unaffected; a BALANCED panel drops whole periods and no units, so the all-eventually-treated capability this row delivers cannot trip it. Defining the unbalanced estimand is tracked in TODO.md. SURVEY WORKAROUND IS CONDITIONAL: the refusal message points at explicit frame restriction, which is exact only when every PSU and stratum survives the restriction - true on a balanced panel, NOT in general (measured: restricting a 6-period unbalanced frame to its 5 supported periods removed one PSU and one stratum outright). REGISTRY, the survey roadmap and the error message all state that condition rather than claiming equivalence. NOT INCLUDED, tracked in TODO.md: the D_{G_max} x X covariate normalization, so covariates on such a panel remain rank-deficient (coefficients unaffected, but rank_deficient_action='error' raises); and an opt-out parameter for the filtering. introduced_in gates the 3.9 cut; deprecated_in stays null so the early-flip guard does not fire against the PR that ships it." + notes: "Per-period COMPARISON-SUPPORT filtering: WooldridgeDiD.fit() now removes periods at which no unit is untreated from the estimation sample BEFORE the solve, instead of carrying them into a rank-deficient design. This is the cell half of W2025 Section 5.4 - with no never-treated group the last cohort serves as the reference, and 'all variables in regression (5.3) involving dT_i get dropped' - so all-eventually-treated panels ESTIMATE again, resolving the capability regression [M-124] recorded. Measured on a {3,5,8} panel over t=1..9 with a true ATT of 1.5: cells (3,3..7) and (5,5..7), overall_att=1.5502, cohort 8 receiving none. Anchored externally to Stata jwdid on the mpdta panel with never-treated counties dropped (golden block jwdid_alltreated): identical cell set, identical N (764 of 955), ATTs agreeing to ~1e-15. The eligible set is keyed on the regression BASELINE, not the method: on never_treated + OLS only never-treated rows qualify, because include_pre emits every (g,t) except each cohort's reference so a later cohort's rows sit in their own indicator; elsewhere not-yet-treated rows qualify too. Omitted reference cells deliberately do NOT count as support - they sit in the baseline but do not identify the period, since the cohort indicator is absorbed by the unit FE and the period dummy stays reproducible from the emitted cells (measured: retaining a period on the strength of ref(6)=5 raises on the lost (6,1), while dropping it fits at overall_att=1.0171). SAMPLE REDUCTION IS ALWAYS REPORTED, in two parts and neither gated on rank_deficient_action: at filter time the periods, observation count and branch-correct cause (emitted there so it survives a downstream raise), and after the final build any cohort left with no cells or stripped of every row (read from the FINAL cell set so a cohort that unidentified-cohort exclusion also removes is not double-reported). Stata jwdid performs the same reduction silently, reporting only a smaller N; being explicit is a deliberate improvement, not a numerical deviation. REFERENCE MOVEMENT: on the not_yet_treated branch a reference can never be filtered (eligibility t < g - anticipation IS the predicate's own second disjunct) and the estimator asserts this at runtime; on never_treated + OLS it CAN move, which renormalizes that cohort's ATTs against a different baseline period, so that case warns naming the cohort and both periods rather than raising. RESULTS METADATA: results.groups and _n_g_per_cohort are now derived from the emitted cell set so a cohort retained only as a control is not advertised as estimated; cohort_trend_coefs stays keyed on PRESENT cohorts because the trend baseline is exactly the zero-cell cohort. The design-side readers of the cohort list (covariate blocks, trend block) deliberately keep the present-cohort list - routing them through the cells-derived one would drop D_{G_max} x X and silently apply a covariate normalization this row does NOT include (measured: 8/8 ATTs move, max 0.0648). Under survey_design= the filter REFUSES rather than subsetting, conditional on rows actually dropping, for the same reason as [M-123]. COHORT-SHARE BOUNDARY: `_n_g_per_cohort` (N_g, Eqs. 7.4/7.6) is read off the FINAL sample, and this filter is the first thing in the estimator that can remove SOME units of a RETAINED cohort - `_filter_sample` keeps every row of every treated unit and unidentified-cohort exclusion removes whole cohorts, so before this N_g was always the full supplied cohort. On an UNBALANCED panel a unit observed only at unsupported periods vanishes with them and N_g then describes a strictly smaller cohort (measured: 90 of a cohort's 100 units observed only at a dropped period moves aggregate(weights='cohort_share') from 1.8078 to 3.8157). W2025 Section 7 assumes a balanced panel and does not say which reading applies, so aggregate() FAILS CLOSED naming the cohorts and unit counts rather than choosing silently, mirroring the existing survey + cohort_share refusal. weights='cell' never reads N_g and is unaffected; a BALANCED panel drops whole periods and no units, so the all-eventually-treated capability this row delivers cannot trip it. Defining the unbalanced estimand is tracked in TODO.md. SURVEY WORKAROUND IS CONDITIONAL: the refusal message points at explicit frame restriction, which is exact only when every PSU and stratum survives the restriction - true on a balanced panel, NOT in general (measured: restricting a 6-period unbalanced frame to its 5 supported periods removed one PSU and one stratum outright). REGISTRY, the survey roadmap and the error message all state that condition rather than claiming equivalence. NOT INCLUDED, tracked in TODO.md: the D_{G_max} x X covariate normalization, so covariates on such a panel remain rank-deficient (coefficients unaffected, but rank_deficient_action='error' raises). The filtering opt-out is now implemented by [M-147]. introduced_in gates the 3.9 cut; deprecated_in stays null so the early-flip guard does not fire against the PR that ships it." - id: M-124 kind: behavior group: etwfe-reference-period @@ -1772,3 +1772,16 @@ rows: test_ref: tests/test_staggered.py code_refs: [diff_diff/results_base.py, diff_diff/staggered_results.py, diff_diff/staggered_triple_diff_results.py, diff_diff/chaisemartin_dhaultfoeuille_results.py, diff_diff/imputation_results.py, diff_diff/efficient_did_results.py, diff_diff/two_stage_results.py, diff_diff/stacked_did_results.py, diff_diff/sun_abraham.py, diff_diff/results.py, diff_diff/triple_diff.py, diff_diff/trop_results.py, diff_diff/continuous_did_results.py, diff_diff/synthetic_control_results.py, docs/methodology/REGISTRY.md] notes: "Staggered-family summary(alpha=)/print_summary(alpha=) tightening via the shared results_base._require_fit_alpha guard (the DMLDiDResults/EventStudyResults precedent): eight results classes (CallawaySantAnnaResults, StaggeredTripleDiffResults, ChaisemartinDHaultfoeuilleResults, ImputationDiDResults, EfficientDiDResults, TwoStageDiDResults, StackedDiDResults, SunAbrahamResults) previously did alpha = alpha or self.alpha and relabeled the confidence-interval header at the REQUESTED alpha while always printing the FIT-TIME stored intervals - silent coverage mislabeling on any non-fit alpha (bootstrap percentile intervals cannot be reconstructed from the SE at all). Behavior delta: accepted-and-mislabeled -> loud ValueError; alpha=0.0, previously swallowed by the falsy `or` idiom, now raises too. StaggeredTripleDiffResults is INCLUDED although its parent estimator is removed at 4.0 ([M-013]/[M-014]) - the mislabel is a live 3.x rendering bug, distinct from the estimator's frozen construction shape. Resolved - the guard is applied family-wide: the non-staggered summaries (DiDResults incl. SpilloverDiDResults by inheritance, MultiPeriodDiDResults, SyntheticDiDResults, TripleDifferenceResults, TROPResults, ContinuousDiDResults, SyntheticControlResults - the last previously a dead no-op alpha, now an honest raise) adopted it via the same helper (an optional message= override carries class-accurate wording where the staggered default's bootstrap-percentile rationale would be false; the staggered default message is byte-unchanged). Sibling behavioral evidence lives in tests/test_estimators.py, tests/test_triple_diff.py, tests/test_trop.py, tests/test_continuous_did.py, tests/test_methodology_synthetic_control.py, and tests/test_spillover.py (test_ref stays the staggered anchor: the schema's test_ref is a single path and this row remains the one logical contract). introduced_in 4.0 per the [M-144] rationale; status done is terminal." + - id: M-147 + kind: behavior + group: etwfe-reference-period + old: "diff_diff:WooldridgeDiD[unsupported_period_action]" + new: null + introduced_in: "4.0" + deprecated_in: null + removed_in: null + status: done + phase: 5 + test_ref: tests/test_wooldridge.py + code_refs: [diff_diff/wooldridge.py, diff_diff/wooldridge_results.py, docs/methodology/REGISTRY.md, diff_diff/guides/llms-full.txt, diff_diff/guides/llms-autonomous.txt] + notes: "Additive comparison-support policy: unsupported_period_action='drop' preserves [M-125] filtering and warnings; 'error' raises ValueError naming unsupported periods and their observation count before deleting periods or constructing the interaction matrix. The positive-weight, anticipation-aware support predicate is unchanged across all three methods and both control-group settings, independently of rank_deficient_action. Existing input and survey validation retain precedence; valid surveys needing period removal still raise NotImplementedError under 'drop', while 'error' raises the policy ValueError. Unidentified-cohort exclusion and completeness/finite-ATT guards remain active. Constructor and fit-time validation reject invalid values; BaseEstimator supplies transactional set_params. Results echo the fit-time policy in to_dict/summary and preserve it through aggregation and pickle restoration (legacy default 'drop'). introduced_in 4.0 follows the post-cut [M-144] convention; no default flip, deprecation, or removal." diff --git a/docs/v4-design.md b/docs/v4-design.md index 6dbe5c787..8e121488a 100644 --- a/docs/v4-design.md +++ b/docs/v4-design.md @@ -851,7 +851,7 @@ above; anything only one PR cares about stays in that PR's plan.** | 2: contract foundations | 3.9 | (a) results base + unified event-study representation [M-092] + to_dict completion + the Diagnostic marker base on the diagnostic result roster [M-091] (section 3.5); (b) `aggregate()` + fit(aggregate=) shims [M-020..M-027] [M-139] (M-020's shim already shipped; M-139 is the HAD workflow twin, a pre-cut amendment); (c) param renames [M-030..M-047] [M-084] [M-086..M-089] + their results-field mirrors [M-094] [M-095] (section 8 rule 9) + the public-function completeness sweep [M-097..M-113] (section 8 rule 10) + the dCDH results mirror [M-114] + the fourth `robust` site [M-115] + the 2(c)-ii missed-rename amendments [M-136..M-138] (LPDiD `level` value; the two post-dummy diagnostics params) + BaseEstimator mixin + ContinuousDiD covariates move; (d) alias introduction [M-062] (the Spillover introduction is cancelled [M-063]) + the alias-diet `__getattr__` warning shim [M-135] + wrapper deprecations [M-070..M-077] + the two inference-surface policies: `n_bootstrap` semantic unification [M-081] and the wild-cluster-bootstrap roster guard [M-096]; shipped insertions (all done): the aggregate contract [M-122], the ETWFE reference-period family [M-123] [M-124] [M-125], and the variance-consolidation program [M-126] [M-127] | | 3: merges | 3.9 | (a) TWFE event-study mode [M-010] + EventStudy warn [M-060] + the fit `time`->`post` rename [M-082] (gates: section 4.1's equivalence/divergence/pooled-parity test triple) (shipped: tests/test_v4_merge_mpd.py; consumer ports incl. HonestDiD/PreTrendsPower calendar routes); (b) TripleDifference facade [M-013] + the SDDD alias [M-064] (shipped: tests/test_v4_merge_ddd.py; the engine relocation into the private shared mixin, the keyword-only staggered fit params, and the pscore_trim tightening [M-142]. The fit-time aggregate=/balance_e= carve-out rows are scheduled REMOVALS and so are cited in the phase-5 cell, not here - their only lifecycle version is removed_in 4.0); (c) CiC method= [M-015] + its results-field mirror [M-143] (shipped: tests/test_v4_merge_cic.py; method= is keyword-only and lowercase-only, the QDiD CLASS is deprecated while the METHOD is not, and ChangesInChangesResults.estimator -> .method carries a dual-key to_dict() window through 3.9) | | 4: release + soak | 3.9 cut | Migration guide written (skeleton: section 10); maintainer cuts 3.9; maint/3.8 rule active | -| 5: enforcement | 4.0 | Removals [M-010..M-015, M-020..M-027, M-139, M-030, M-032..M-047 old names, M-060, M-061, M-064, M-070..M-077, M-084, M-086..M-089, M-001..M-003, M-117, M-118, M-119, M-120, M-140, M-141, M-143] + the alias diet [M-132]..[M-134] + the amendment's old names [M-094] [M-095] [M-097..M-115] [M-136..M-138] (incl. their consumer migrations and the `clean_control` serialized reporting key); M-031's old `time` name persists as the merged class's calendar column, so it is deliberately absent from the removal roster (its 4.0 enforcement is the M-085 behavior entry below); property window: [M-016] property-flips at 4.0 (removal at 5.0); storage flips [M-050..M-058]; default policies [M-004..M-006, M-128..M-131, M-080]; merged-class behavior enforcements [M-083] [M-085]; warning retirement [M-007]; fastpath go/no-go [M-008]; diagnostic-family docs/roster reorganization [M-090]; sentinel retirement [M-093]; the family-wide anticipation validation [M-144], the family-wide pscore_trim validation [M-145], and the family-wide summary-alpha guard [M-146] (staggered family at the initial cut; extended to the non-staggered summaries post-cut) (behavior tightenings landing at 4.0 - shipped post-3.9-cut, terminal `done`, no removal/deprecation fields); docs/llms.txt/README refresh | +| 5: enforcement | 4.0 | Removals [M-010..M-015, M-020..M-027, M-139, M-030, M-032..M-047 old names, M-060, M-061, M-064, M-070..M-077, M-084, M-086..M-089, M-001..M-003, M-117, M-118, M-119, M-120, M-140, M-141, M-143] + the alias diet [M-132]..[M-134] + the amendment's old names [M-094] [M-095] [M-097..M-115] [M-136..M-138] (incl. their consumer migrations and the `clean_control` serialized reporting key); M-031's old `time` name persists as the merged class's calendar column, so it is deliberately absent from the removal roster (its 4.0 enforcement is the M-085 behavior entry below); property window: [M-016] property-flips at 4.0 (removal at 5.0); storage flips [M-050..M-058]; default policies [M-004..M-006, M-128..M-131, M-080]; merged-class behavior enforcements [M-083] [M-085]; warning retirement [M-007]; fastpath go/no-go [M-008]; diagnostic-family docs/roster reorganization [M-090]; sentinel retirement [M-093]; the family-wide anticipation validation [M-144], the family-wide pscore_trim validation [M-145], and the family-wide summary-alpha guard [M-146] (staggered family at the initial cut; extended to the non-staggered summaries post-cut) (behavior tightenings landing at 4.0 - shipped post-3.9-cut, terminal `done`, no removal/deprecation fields); the additive Wooldridge comparison-support policy [M-147] (`unsupported_period_action="drop"` preserves filtering, `"error"` refuses before period removal; same post-cut lifecycle); docs/llms.txt/README refresh | | 6: front door | 4.1 | `event_study(data, outcome, unit, time, first_treat, estimator=...)` comparison entry point over the staggered family (sketch only; specified in its own plan) | Citation semantic for the table: a cell may cite a row whose current `phase` @@ -1113,8 +1113,8 @@ forever - a removed symbol resurrecting is a test failure. class/function rows and alias rows also assert `__all__` membership consistent with their status (stale `import *` entries fail). The shipped row ids are a - committed snapshot in the enforcement test (128 as of the DML - review-follow-ups pair; previously 126 as of the family-wide + committed snapshot in the enforcement test (129 as of the ETWFE + unsupported-period policy; previously 126 as of the family-wide anticipation validation row: Phase 1 + the diagnostic-family amendment + the M-092/M-093 results-contract rows + the M-094..M-096 amendment rows + @@ -1122,7 +1122,7 @@ forever - a removed symbol resurrecting is a test failure. reference-period pair M-123/M-124 + M-125 + M-126 + M-127..M-131 + the alias-diet family M-132..M-135 + the 2(c)-ii amendments M-136..M-138 + M-139 + the DDD-merge rows M-140..M-142 + the CiC - results-field mirror M-143 + the anticipation policy row M-144 + the DML review-follow-ups pair M-145/M-146; + results-field mirror M-143 + the anticipation policy row M-144 + the DML review-follow-ups pair M-145/M-146 + the unsupported-period policy M-147; the snapshot extends by a new id range in the same diff that appends rows): ids are never deleted or reused, and the test fails if any snapshot id disappears. diff --git a/tests/test_v4_matrix.py b/tests/test_v4_matrix.py index c56f046d7..15b3e1a4b 100644 --- a/tests/test_v4_matrix.py +++ b/tests/test_v4_matrix.py @@ -131,11 +131,12 @@ # pscore_trim tightening) = 124; + the CiC results-field rename (M-143) = 125; # + the family-wide anticipation validation row (M-144) = 126; + the DML # review-follow-ups pair (M-145 family-wide pscore_trim validation, M-146 -# family-wide summary-alpha guard) = 128. +# family-wide summary-alpha guard) = 128; + the ETWFE unsupported-period +# policy (M-147) = 129. # Ids are never reused and terminal rows are never deleted, so the ledger # only grows - raise the floor when rows are added; a lower parse count # means scanner/format drift or an illegal row deletion. -ROW_COUNT_FLOOR = 128 +ROW_COUNT_FLOOR = 129 # Committed snapshot of the shipped id set ("ids are never deleted or reused" # contract - a delete-one-add-one edit keeps the count above the floor but trips @@ -191,6 +192,7 @@ (143, 143), (144, 144), (145, 146), + (147, 147), ] EXPECTED_INITIAL_IDS = frozenset( f"M-{n:03d}" for lo, hi in _INITIAL_ID_RANGES for n in range(lo, hi + 1) @@ -589,14 +591,14 @@ def test_initial_ids_never_deleted(): """The shipped id set is immutable: ids are never deleted or reused (spec section 11). ROW_COUNT_FLOOR alone would let a delete-one-add-one edit pass; this snapshot cannot. - Extends as rows ship (128 as of the DML review-follow-ups pair: + Extends as rows ship (129 as of the ETWFE unsupported-period policy: Phase 1 + diagnostic-family + M-092/M-093 + M-094..M-096 + the M-097..M-115 public-function completeness sweep + M-117..M-120/M-122 + M-123/M-124 + M-125 + M-126 + M-127..M-131 + M-132..M-135 + - M-136..M-138 + M-139 + M-140..M-142 + M-143 + M-144 + M-145/M-146).""" + M-136..M-138 + M-139 + M-140..M-142 + M-143 + M-144 + M-145/M-146 + M-147).""" missing = sorted(EXPECTED_INITIAL_IDS - set(_ROW_IDS)) assert not missing, f"ledger rows deleted (ids are permanent): {missing}" - assert len(EXPECTED_INITIAL_IDS) == 128 + assert len(EXPECTED_INITIAL_IDS) == 129 def test_version_tuple_pads_to_three_components(): @@ -1060,17 +1062,18 @@ def _changes_at_4_0(row): Keyed on the two LIFECYCLE version fields only - a symbol removed at 4.0, or one whose deprecation warning starts firing at 4.0 (the ``field-flip`` family, removed - at 5.0). 108 of the 128 rows qualify. + at 5.0). 108 of the 129 rows qualify. - The 20 that do not, and why (this enumeration is the contract - a reader of the + The 21 that do not, and why (this enumeration is the contract - a reader of the guide must be able to trust that nothing 4.0-relevant was dropped): - 12 ``behavior`` rows with ``introduced_in: 3.9`` and no dep/rem: already shipped in 3.9, so there is no 4.0 action. They get their own guide section, not an appendix row. - - 3 ``behavior`` rows with ``introduced_in: 4.0`` and no deprecation/removal + - 4 ``behavior`` rows with ``introduced_in: 4.0`` and no deprecation/removal fields (``M-144`` anticipation validation, ``M-145`` pscore_trim validation, - ``M-146`` summary-alpha guard - the post-cut validation tightenings): they land + ``M-146`` summary-alpha guard, ``M-147`` ETWFE unsupported-period policy + - the post-cut behavior changes): they land AT 4.0 but remove/deprecate nothing, so they appear in the guide's "Remaining 4.0 changes" prose, not the ledger-derived appendix. - ``M-062``, ``M-063``: aliases, introduce-only / all lifecycle fields null. diff --git a/tests/test_wooldridge.py b/tests/test_wooldridge.py index 1a5c4680f..8b2ab2ade 100644 --- a/tests/test_wooldridge.py +++ b/tests/test_wooldridge.py @@ -1,5 +1,6 @@ """Tests for WooldridgeDiD estimator and WooldridgeDiDResults.""" +import pickle import warnings import numpy as np @@ -4341,6 +4342,233 @@ def test_bootstrap_runs_on_a_filtered_fit(self): assert np.isfinite(res.overall_se) +class TestUnsupportedPeriodAction: + """M-147: opt out of period deletion without weakening identification.""" + + @staticmethod + def _panel(supported=False): + rng = np.random.default_rng(147) + rows = [] + for j, g in enumerate((0, 3, 6)): + for u in range(20): + fe = rng.normal(0, 0.2) + stop = 7 if supported or g else 5 + for t in range(1, stop): + eta = fe + 0.05 * t + 0.4 * (g > 0 and t >= g) + rng.normal(0, 0.2) + rows.append((20 * j + u, t, g, 1 / (1 + np.exp(-eta)))) + return pd.DataFrame(rows, columns=["unit", "time", "cohort", "y"]) + + @staticmethod + def _fit(est, df, **kwargs): + return est.fit(df, outcome="y", unit="unit", time="time", first_treat="cohort", **kwargs) + + @staticmethod + def _assert_same_estimates(left, right): + assert left.n_obs == right.n_obs + assert left.groups == right.groups + assert left.time_periods == right.time_periods + pd.testing.assert_frame_equal(left.to_dataframe(level="gt"), right.to_dataframe(level="gt")) + np.testing.assert_equal(left._gt_vcov, right._gt_vcov) + for level in ("simple", "group", "calendar", "event_study"): + left.aggregate(level) + right.aggregate(level) + pd.testing.assert_frame_equal( + left.to_dataframe(level=level), right.to_dataframe(level=level) + ) + + def test_parameter_roundtrip_and_switch(self): + est = WooldridgeDiD() + assert est.get_params()["unsupported_period_action"] == "drop" + for action in ("error", "drop"): + assert est.set_params(unsupported_period_action=action) is est + assert est.unsupported_period_action == action + assert WooldridgeDiD(**est.get_params()).get_params() == est.get_params() + + @pytest.mark.parametrize( + "bad", + ["warn", "DROP", "", None, True, 0, 1.0, ["drop"], np.array(["drop"]), {"drop": True}], + ) + def test_invalid_policy_is_transactional_and_rechecked(self, bad): + with pytest.raises(ValueError, match="unsupported_period_action must be"): + WooldridgeDiD(unsupported_period_action=bad) + est = WooldridgeDiD(unsupported_period_action="error") + original = est.get_params() + with pytest.raises(ValueError, match="unsupported_period_action must be"): + est.set_params(alpha=0.1, unsupported_period_action=bad) + assert est.get_params() == original + est.unsupported_period_action = bad + with pytest.raises(ValueError, match="unsupported_period_action must be"): + self._fit(est, self._panel(supported=True)) + assert not est.is_fitted_ + + @pytest.mark.parametrize("method", ["ols", "logit", "poisson"]) + @pytest.mark.parametrize("control", ["never_treated", "not_yet_treated"]) + @pytest.mark.parametrize("rank", ["warn", "error", "silent"]) + def test_unsupported_matrix(self, method, control, rank, monkeypatch): + df = self._panel() + original = df.copy(deep=True) + options = dict(method=method, control_group=control, rank_deficient_action=rank) + periods = [5, 6] if method == "ols" and control == "never_treated" else [6] + n_dropped = int(df.time.isin(periods).sum()) + with warnings.catch_warnings(record=True) as implicit_warnings: + warnings.simplefilter("always") + implicit = self._fit(WooldridgeDiD(**options), df) + with warnings.catch_warnings(record=True) as explicit_warnings: + warnings.simplefilter("always") + explicit = self._fit(WooldridgeDiD(**options, unsupported_period_action="drop"), df) + assert [(w.category, str(w.message)) for w in implicit_warnings] == [ + (w.category, str(w.message)) for w in explicit_warnings + ] + assert any( + f"Dropped {n_dropped} of {len(df)} observations" in str(w.message) + for w in explicit_warnings + ) + assert explicit.n_obs == len(df) - n_dropped + assert explicit.time_periods == [t for t in range(1, 7) if t not in periods] + assert np.isfinite(explicit.att) and np.isfinite(explicit.se) + self._assert_same_estimates(implicit, explicit) + assert explicit.to_dict()["unsupported_period_action"] == "drop" + + def forbidden_build(*args, **kwargs): + pytest.fail("unsupported-period refusal must precede interaction construction") + + monkeypatch.setattr("diff_diff.wooldridge._build_interaction_matrix", forbidden_build) + strict = WooldridgeDiD(**options, unsupported_period_action="error") + with warnings.catch_warnings(record=True) as strict_warnings: + warnings.simplefilter("always") + with pytest.raises(ValueError) as exc: + self._fit(strict, df) + message = str(exc.value) + assert ( + f"Period(s) {', '.join(map(str, periods))} have no eligible comparison group" in message + ) + assert f"{n_dropped} of {len(df)} observations" in message + assert "unsupported_period_action='error'" in message + assert "unsupported_period_action='drop'" in message + assert not strict_warnings + assert not strict.is_fitted_ + with pytest.raises(RuntimeError, match="fit"): + _ = strict.results_ + pd.testing.assert_frame_equal(df, original) + + @pytest.mark.parametrize("method", ["ols", "logit", "poisson"]) + @pytest.mark.parametrize("control", ["never_treated", "not_yet_treated"]) + @pytest.mark.parametrize("rank", ["warn", "error", "silent"]) + def test_supported_matrix_and_result_provenance(self, method, control, rank): + df = self._panel(supported=True) + options = dict(method=method, control_group=control, rank_deficient_action=rank) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + dropped = self._fit(WooldridgeDiD(**options), df) + est = WooldridgeDiD(**options, unsupported_period_action="error") + strict = self._fit(est, df) + assert not caught + assert strict.n_obs == len(df) + assert np.isfinite(strict.att) and np.isfinite(strict.se) + self._assert_same_estimates(dropped, strict) + est.set_params(unsupported_period_action="drop") + for result in (strict, pickle.loads(pickle.dumps(strict))): + assert result.unsupported_period_action == "error" + assert result.to_dict()["unsupported_period_action"] == "error" + assert "Unsupported period action: error" in result.summary() + # Failed updates/refits must not overwrite the last completed fit's provenance. + with pytest.raises(ValueError, match="unsupported_period_action must be"): + est.set_params(unsupported_period_action=None) + est.set_params(unsupported_period_action="error") + with pytest.raises(ValueError, match="no eligible comparison group"): + self._fit(est, self._panel()) + assert est.results_ is strict + assert strict.to_dict()["unsupported_period_action"] == "error" + + def test_legacy_result_state_defaults_to_drop(self): + result = _make_minimal_results() + assert result.unsupported_period_action == "drop" + del result.__dict__["unsupported_period_action"] + restored = pickle.loads(pickle.dumps(result)) + assert restored.__dict__["unsupported_period_action"] == "drop" + assert restored.to_dict()["unsupported_period_action"] == "drop" + assert "Unsupported period action: drop" in restored.summary() + + @pytest.mark.parametrize("method", ["ols", "logit", "poisson"]) + @pytest.mark.parametrize("control", ["never_treated", "not_yet_treated"]) + @pytest.mark.parametrize("action", ["drop", "error"]) + def test_survey_positive_weight_support_and_validation(self, method, control, action): + from diff_diff.survey import SurveyDesign + + df = self._panel(supported=True) + df["w"] = 1.0 + df["psu"] = df.unit + design = SurveyDesign(weights="w", psu="psu") + est = WooldridgeDiD(method=method, control_group=control, unsupported_period_action=action) + result = self._fit(est, df, survey_design=design) + reference = self._fit( + WooldridgeDiD(method=method, control_group=control), df, survey_design=design + ) + self._assert_same_estimates(result, reference) + assert result.n_obs == len(df) + assert result.survey_metadata is not None + assert result.to_dict()["unsupported_period_action"] == action + # Observed but zero-weight controls cannot supply comparison support. + df.loc[(df.cohort == 0) & (df.time >= 5), "w"] = 0.0 + error_type = ValueError if action == "error" else NotImplementedError + with pytest.raises(error_type, match="no eligible comparison group") as exc: + self._fit(est, df, survey_design=design) + periods = "5, 6" if method == "ols" and control == "never_treated" else "6" + assert f"Period(s) {periods} have no eligible comparison group" in str(exc.value) + # Invalid design metadata in an unsupported period must not be hidden by the policy. + df.loc[(df.cohort == 3) & (df.time == 6), "psu"] = np.nan + with pytest.raises(ValueError) as exc: + self._fit(est, df, survey_design=design) + assert "no eligible comparison group" not in str(exc.value) + assert "psu" in str(exc.value).lower() + + @pytest.mark.parametrize("anticipation", [0, 1, 2]) + @pytest.mark.parametrize("method", ["ols", "logit", "poisson"]) + def test_all_treated_anticipation_threshold(self, anticipation, method): + df = self._panel() + df = df[df.cohort > 0] + periods = list(range(6 - anticipation, 7)) + n_unsupported = int(df.time.isin(periods).sum()) + with pytest.raises(ValueError) as exc: + self._fit( + WooldridgeDiD( + method=method, anticipation=anticipation, unsupported_period_action="error" + ), + df, + ) + assert f"Period(s) {', '.join(map(str, periods))} have no eligible comparison group" in str( + exc.value + ) + assert f"{n_unsupported} of {len(df)} observations" in str(exc.value) + + @pytest.mark.parametrize("action", ["drop", "error"]) + def test_policy_does_not_disable_unidentified_cohort_exclusion(self, action): + df = self._panel(supported=True) + df.loc[df.cohort == 6, "cohort"] = 1 + with pytest.warns(UserWarning, match="observations are EXCLUDED"): + result = self._fit(WooldridgeDiD(unsupported_period_action=action), df) + assert result.n_obs == len(df[df.cohort != 1]) + assert result.groups == [3] + assert np.isfinite(result.att) + + @pytest.mark.parametrize("options", [{"n_bootstrap": 49, "seed": 147}, {"cohort_trends": True}]) + def test_bootstrap_and_cohort_trends(self, options, ci_params): + if options.get("n_bootstrap"): + options = dict(options, n_bootstrap=ci_params.bootstrap(options["n_bootstrap"])) + df = self._panel(supported=True) + dropped = self._fit(WooldridgeDiD(**options), df) + strict = self._fit(WooldridgeDiD(**options, unsupported_period_action="error"), df) + self._assert_same_estimates(dropped, strict) + assert np.isfinite(strict.att) and np.isfinite(strict.se) + assert strict.unsupported_period_action == "error" + if options.get("n_bootstrap"): + assert strict._bootstrap_used + else: + assert set(strict.cohort_trend_coefs) == {3, 6} + with pytest.raises(ValueError, match="unsupported_period_action='error'"): + self._fit(WooldridgeDiD(**options, unsupported_period_action="error"), self._panel()) + + class TestWooldridgeDfConvention: """The three-value df_convention knob on WooldridgeDiD OLS arms (3.9 / M-127).""" From 4c4a178eeccdb140e6b6494b2cdb9120db382408 Mon Sep 17 00:00:00 2001 From: igerber Date: Sat, 5 Sep 2026 20:42:27 -0400 Subject: [PATCH 2/2] Add WooldridgeDiD opt-out for comparison-support filtering --- diff_diff/guides/llms-autonomous.txt | 6 ++- diff_diff/guides/llms-full.txt | 7 ++- diff_diff/wooldridge.py | 5 ++- docs/api/wooldridge_etwfe.rst | 5 ++- docs/methodology/REGISTRY.md | 2 +- docs/tutorials/16_wooldridge_etwfe.ipynb | 11 ++++- docs/v4-deprecations.yaml | 2 +- tests/test_wooldridge.py | 57 ++++++++++++++++++++++++ 8 files changed, 86 insertions(+), 9 deletions(-) diff --git a/diff_diff/guides/llms-autonomous.txt b/diff_diff/guides/llms-autonomous.txt index cdbbb663f..b416425e1 100644 --- a/diff_diff/guides/llms-autonomous.txt +++ b/diff_diff/guides/llms-autonomous.txt @@ -586,8 +586,10 @@ When `has_never_treated == False`: never-treated rows only on OLS + `never_treated`, and also rows before `g - anticipation` on other paths. The policy applies to all methods, independently of `rank_deficient_action`; other identification checks and - unidentified-cohort exclusion remain active. With `survey_design`, design - validation still runs first: `"error"` then raises `ValueError`, while + unidentified-cohort exclusion remain active. Only existing pre-filter checks + retain precedence; later covariate, nonlinear-outcome, or explicit-cluster + validation can be preempted by unsupported-period refusal. With `survey_design`, + its pre-filter design validation still runs first: `"error"` then raises `ValueError`, while `"drop"` retains the survey-domain `NotImplementedError` when periods would be removed. Fully supported survey fits are unaffected. SOME covariate specifications on such a panel are still rank-deficient diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index c9cf99291..540ed6b3e 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1617,6 +1617,10 @@ count before removal. Support requires positive-weight never-treated rows on OLS + `control_group="never_treated"`; elsewhere not-yet-treated rows before `g - anticipation` also qualify. This is independent of `rank_deficient_action` and does not control unidentified-cohort exclusion or bypass identification checks. +Only existing pre-filter configuration, cohort, and survey-design checks retain +precedence. Later checks (covariate columns, nonlinear outcomes, and non-Conley +explicit cluster columns) are not preflighted; unsupported-period refusal can +precede those input errors. The results record the fit-time policy in `unsupported_period_action`, `to_dict()` and `summary()`, including after aggregation or estimator reconfiguration. @@ -1642,7 +1646,8 @@ or unidentified-cohort exclusion raises `NotImplementedError` rather than deleti remove their PSUs and strata from the TSL variance. Restrict the frame yourself and re-fit only after confirming every PSU and stratum survives that restriction. With `unsupported_period_action="error"`, unsupported periods instead raise -`ValueError`, after existing input/design validation. The refusals are conditional: +`ValueError`, after the existing pre-filter checks, including survey-design +validation. The refusals are conditional: survey fits that drop nothing are unaffected. **Alias:** `ETWFE` diff --git a/diff_diff/wooldridge.py b/diff_diff/wooldridge.py index 289fa6a48..8a5d452ee 100644 --- a/diff_diff/wooldridge.py +++ b/diff_diff/wooldridge.py @@ -982,7 +982,10 @@ class WooldridgeDiD(BaseEstimator): exclusion. With ``survey_design``, ``"drop"`` still raises ``NotImplementedError`` if periods would be removed, because survey domain estimation is not supported; ``"error"`` raises ``ValueError`` - after the existing input and survey validation. + after the existing pre-filter configuration, cohort, and survey-design + checks. Later validation (including covariate columns, nonlinear + outcomes, and some explicit cluster columns) is not preflighted: an + unsupported-period refusal can precede those input errors. """ def __init__( diff --git a/docs/api/wooldridge_etwfe.rst b/docs/api/wooldridge_etwfe.rst index e4e2d28b5..af02f65df 100644 --- a/docs/api/wooldridge_etwfe.rst +++ b/docs/api/wooldridge_etwfe.rst @@ -55,7 +55,10 @@ Support requires a positive-weight never-treated observation for OLS with ``control_group="never_treated"``. All other paths also admit observations before their cohort's ``g - anticipation`` threshold. Unsupported periods can therefore occur even when never-treated units exist elsewhere in the panel. -With ``survey_design``, input/design validation runs first; ``"error"`` then +Existing pre-filter configuration, cohort, and survey-design checks retain +precedence. Later checks, including covariate columns, nonlinear outcomes, and +non-Conley explicit cluster columns, are not preflighted: an unsupported-period +refusal can precede those input errors. With ``survey_design``, ``"error"`` raises ``ValueError`` for unsupported periods, while ``"drop"`` retains the existing ``NotImplementedError`` because survey domain estimation is unsupported. diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 2b33baf9f..12e8e0702 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -2462,7 +2462,7 @@ The interaction coefficient `δ_{g,t}` identifies `ATT(g, t)` under parallel tre - **Note (reference-period sensitivity, W2025 Section 6.1):** the choice of reference is a NORMALIZATION, not an identifying assumption. The paper notes any set of pre-treatment periods may serve, and the pre-trend `t`-test is numerically identical whichever is used. `g − 1` is the paper's canonical choice and the one this library implements. **The point estimates are NOT invariant to it** — only the two-sided pre-trend test is (Section 6.1). Different references give different finite-sample 2×2 contrasts, all consistent for ATT(g, t) under parallel trends: on `mpdta`, ATT(2007, 2007) computed by hand against references 2006/2005/2004/2003 gives −0.0261 / −0.0571 / −0.0599 / −0.0294. This is exactly why issue #724 mattered — QR dropped `g2007_t2005`, silently making 2005 the reference, and the estimator returned −0.0571: a correct 2×2 DiD against the wrong baseline. It also means the unbalanced-panel fallback (the cohort's latest AVAILABLE pre-period) yields a different contrast from `g − 1`, which is legitimate but should be read as such. - **Note (unidentified cohorts — library identification limit):** a cohort with NO observed period before `g − anticipation` has no reference cell, so none of its ATTs are identified. The library warns naming the cohort and **excludes its observations** from the estimation sample. *(Conditional since comparison-support filtering landed: when the filter removes that cohort's rows first, the cohort never reaches this path and is instead named by the zero-cell / fully-dropped warning above. The observations are excluded either way; only which warning names them differs.)* Excluding only its columns would be worse than the original bug: `_filter_sample` retains every treated row, so the cohort would join the omitted baseline beside the controls and load its treatment effect onto the time fixed effects — measured on `mpdta` at `anticipation=1`, that moved `ATT(2006, 2006)` by 0.0077 and `ATT(2006, 2007)` by 0.0055, past the 5e-3 this library treats as material. Exclusion can cascade (the removed rows may have been another cohort's not-yet-treated comparison), so the surviving sample is re-checked and `fit()` raises when no comparison observations remain rather than returning an all-NaN fit. This is a **library limit, not the paper's last-cohort result** — W2025's "the last cohort `T` plays the role of the never-treated group" is conditioned on there being no never-treated group at all, a different case. - **Note (per-period comparison support, W2025 Section 5.4 — the cell half):** a period at which **no unit is untreated** carries no identified `ATT(g, t)`, because there is no untreated outcome to difference against. With the default `unsupported_period_action="drop"`, such periods are removed from the estimation sample **before the solve**. The eligible set is keyed on the regression baseline, not on the method: on the lead-and-lag branch (`never_treated` + OLS) only never-treated rows qualify, because every `(g, t)` except each cohort's reference is emitted, so a later cohort's rows sit in their own indicator; elsewhere not-yet-treated rows qualify too. **The omitted reference cells do NOT count as support** even though they sit in the baseline: cohort `h`'s indicator is absorbed by the unit FE (`1{h,t} = D_h − Σ_{t'≠t} h_{t'}`), so the period dummy stays reproducible from the emitted cells and the design remains collinear — measured on a panel with never-treated units through `t=4` and cohorts 3, 6 through `t=6`, retaining `t=5` on the strength of `ref(6)=5` raises on the lost `(6,1)`, while dropping it fits at `overall_att = 1.0171`. On an all-eventually-treated panel this yields exactly Eq. 5.15's cell set: cohort `G_max` is the reference and receives nothing. Anchored to Stata `jwdid` (`tests/test_etwfe_cs_stata_parity.py::TestAllEventuallyTreatedVsStataJwdid`): identical cell set, identical `N` (764 of 955), ATTs agreeing to ~1e-15. -- **Note:** Comparison-support filtering is configurable via `unsupported_period_action="drop" | "error"` (ledger M-147). The default `"drop"` preserves the reduction and warnings described here. `"error"` raises `ValueError` before period removal or design construction, naming unsupported periods and their observation count; it does not fit an unidentified full-sample design. The same positive-weight, anticipation-aware predicate applies on OLS, logit and Poisson, independently of `rank_deficient_action`. Existing input/survey validation runs first; for valid survey designs with unsupported periods, `"error"` raises `ValueError` and `"drop"` retains the survey-domain `NotImplementedError`. Other identification guards and unidentified-cohort exclusion are unchanged. Successful results retain the configured policy through aggregation and serialization and expose it via `summary()` and `to_dict()`. +- **Note:** Comparison-support filtering is configurable via `unsupported_period_action="drop" | "error"` (ledger M-147). The default `"drop"` preserves the reduction and warnings described here. `"error"` raises `ValueError` before period removal or design construction, naming unsupported periods and their observation count; it does not fit an unidentified full-sample design. The same positive-weight, anticipation-aware predicate applies on OLS, logit and Poisson, independently of `rank_deficient_action`. Only existing pre-filter checks (configuration, cohort, and survey design) retain precedence; later covariate, nonlinear-outcome, and explicit-cluster checks are not preflighted, so unsupported-period refusal may precede those errors. For valid survey designs with unsupported periods, `"error"` raises `ValueError` and `"drop"` retains the survey-domain `NotImplementedError`. Other identification guards and unidentified-cohort exclusion are unchanged. Successful results retain the configured policy through aggregation and serialization and expose it via `summary()` and `to_dict()`. - **Note (the reduction is always reported — deviation from `jwdid`):** Stata `jwdid` performs the same reduction **silently**, reporting only a smaller `N`. This library warns on every fit that drops rows, naming the periods, the observation count, and the branch-correct cause, and separately naming any cohort left with no estimated cells or stripped of every row. Neither warning is gated on `rank_deficient_action` — that setting governs how rank warnings surface, not whether the estimation sample changed. Being explicit is a deliberate improvement, not a numerical deviation: the estimates are identical. - **Note (reference movement under filtering):** on the `not_yet_treated` branch a reference period can never be filtered — reference eligibility (`t < g − anticipation`) IS the support predicate's own second disjunct evaluated at that cohort's rows — and the estimator asserts this at runtime, raising if it is ever violated (silent renormalization is the issue #724 defect class). On the `never_treated` + OLS branch it CAN move, because eligibility there does not imply never-treated presence: measured, dropping `t=5` moves `ref(6)` from 5 to 4. That is legitimate — the ATTs stay correctly labelled and validly identified, just normalized against a different baseline period — so it **warns** naming the cohort and both periods rather than raising. A reference that becomes `None` is not a move: that cohort is unidentified and the exclusion path already warns by name. - **Note (zero-cell cohorts are warned, not raised):** a cohort retained only as a control — cohort `G_max` under the Section 5.4 normalization — or one that loses every row to filtering produces no `ATT(g, t)`. `fit()` reports it by name and continues. `results.groups` and `_n_g_per_cohort` are derived from the **emitted cell set** so such a cohort is not advertised as estimated (`set(results.groups) == {g for (g, t) in group_time_effects}`). `cohort_trend_coefs` is keyed on PRESENT cohorts instead, because the trend baseline is exactly the zero-cell cohort; the two therefore differ by design, and the invariant on that dict is `set(cohort_trend_coefs) ⊆ present cohorts`. diff --git a/docs/tutorials/16_wooldridge_etwfe.ipynb b/docs/tutorials/16_wooldridge_etwfe.ipynb index d08e0e091..8ae559b41 100644 --- a/docs/tutorials/16_wooldridge_etwfe.ipynb +++ b/docs/tutorials/16_wooldridge_etwfe.ipynb @@ -250,6 +250,7 @@ "======================================================================\n", "Method: ols\n", "Control group: not_yet_treated\n", + "Unsupported period action: drop\n", "Observations: 3000\n", "Treated units: 210\n", "Control units: 300\n", @@ -523,6 +524,7 @@ "======================================================================\n", "Method: ols\n", "Control group: not_yet_treated\n", + "Unsupported period action: drop\n", "Observations: 3000\n", "Treated units: 210\n", "Control units: 300\n", @@ -629,6 +631,7 @@ "======================================================================\n", "Method: poisson\n", "Control group: not_yet_treated\n", + "Unsupported period action: drop\n", "Observations: 3000\n", "Treated units: 210\n", "Control units: 300\n", @@ -797,6 +800,7 @@ "======================================================================\n", "Method: logit\n", "Control group: not_yet_treated\n", + "Unsupported period action: drop\n", "Observations: 3000\n", "Treated units: 210\n", "Control units: 300\n", @@ -1025,6 +1029,7 @@ "======================================================================\n", "Method: poisson\n", "Control group: not_yet_treated\n", + "Unsupported period action: drop\n", "Observations: 2500\n", "Treated units: 191\n", "Control units: 500\n", @@ -1212,8 +1217,10 @@ "`control_group=\"never_treated\"`; other paths also admit not-yet-treated\n", "observations before `g - anticipation`. This applies to all three methods,\n", "independently of `rank_deficient_action`, and leaves other identification\n", - "checks and unidentified-cohort exclusion active. With `survey_design`, design\n", - "validation runs first: `\"error\"` then raises `ValueError` for unsupported\n", + "checks and unidentified-cohort exclusion active. Only existing pre-filter\n", + "checks retain precedence; later covariate, nonlinear-outcome, or explicit-cluster\n", + "validation can be preempted by unsupported-period refusal. With `survey_design`,\n", + "its pre-filter design validation runs first: `\"error\"` then raises `ValueError` for unsupported\n", "periods; `\"drop\"` still raises `NotImplementedError` because domain estimation\n", "is not supported.\n" ] diff --git a/docs/v4-deprecations.yaml b/docs/v4-deprecations.yaml index a49c6a84d..5ce72a9e1 100644 --- a/docs/v4-deprecations.yaml +++ b/docs/v4-deprecations.yaml @@ -1784,4 +1784,4 @@ rows: phase: 5 test_ref: tests/test_wooldridge.py code_refs: [diff_diff/wooldridge.py, diff_diff/wooldridge_results.py, docs/methodology/REGISTRY.md, diff_diff/guides/llms-full.txt, diff_diff/guides/llms-autonomous.txt] - notes: "Additive comparison-support policy: unsupported_period_action='drop' preserves [M-125] filtering and warnings; 'error' raises ValueError naming unsupported periods and their observation count before deleting periods or constructing the interaction matrix. The positive-weight, anticipation-aware support predicate is unchanged across all three methods and both control-group settings, independently of rank_deficient_action. Existing input and survey validation retain precedence; valid surveys needing period removal still raise NotImplementedError under 'drop', while 'error' raises the policy ValueError. Unidentified-cohort exclusion and completeness/finite-ATT guards remain active. Constructor and fit-time validation reject invalid values; BaseEstimator supplies transactional set_params. Results echo the fit-time policy in to_dict/summary and preserve it through aggregation and pickle restoration (legacy default 'drop'). introduced_in 4.0 follows the post-cut [M-144] convention; no default flip, deprecation, or removal." + notes: "Additive comparison-support policy: unsupported_period_action='drop' preserves [M-125] filtering and warnings; 'error' raises ValueError naming unsupported periods and their observation count before deleting periods or constructing the interaction matrix. The positive-weight, anticipation-aware support predicate is unchanged across all three methods and both control-group settings, independently of rank_deficient_action. Only existing pre-filter configuration, cohort and survey-design checks retain precedence; later covariate, nonlinear-outcome and explicit-cluster validation is not preflighted. Valid surveys needing period removal still raise NotImplementedError under 'drop', while 'error' raises the policy ValueError. Unidentified-cohort exclusion and completeness/finite-ATT guards remain active. Constructor and fit-time validation reject invalid values; BaseEstimator supplies transactional set_params. Results echo the fit-time policy in to_dict/summary and preserve it through aggregation and pickle restoration (legacy default 'drop'). introduced_in 4.0 follows the post-cut [M-144] convention; no default flip, deprecation, or removal." diff --git a/tests/test_wooldridge.py b/tests/test_wooldridge.py index 8b2ab2ade..e209f5fc9 100644 --- a/tests/test_wooldridge.py +++ b/tests/test_wooldridge.py @@ -4489,6 +4489,63 @@ def test_legacy_result_state_defaults_to_drop(self): assert restored.to_dict()["unsupported_period_action"] == "drop" assert "Unsupported period action: drop" in restored.summary() + @pytest.mark.parametrize("method", ["logit", "poisson"]) + @pytest.mark.parametrize( + "invalid", ["missing_outcome", "invalid_outcome", "exovar", "xtvar", "xgvar", "cluster"] + ) + def test_support_refusal_can_precede_later_input_validation(self, method, invalid): + """The policy preserves the existing pipeline, not a full-input preflight.""" + df = self._panel() + fit_options = {} + options = dict(method=method) + if invalid == "missing_outcome": + df = df.drop(columns="y") + expected_error, match = KeyError, "y" + elif invalid == "invalid_outcome": + # Invalid on a RETAINED period too, so the default fitter must reject it. + df.loc[df.time == 3, "y"] = -1.0 + expected_error, match = ValueError, f"method='{method}' requires" + elif invalid == "cluster": + options["cluster"] = "missing_cluster" + expected_error, match = KeyError, "missing_cluster" + else: + fit_options[invalid] = ["missing_covariate"] + expected_error, match = KeyError, "missing_covariate" + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with pytest.raises(ValueError, match="no eligible comparison group"): + self._fit( + WooldridgeDiD(**options, unsupported_period_action="error"), df, **fit_options + ) + assert not caught + # When filtering is allowed (or already performed), the existing input + # error surfaces at its normal point rather than being silently accepted. + with pytest.warns(UserWarning, match="Dropped"): + with pytest.raises(expected_error, match=match): + self._fit(WooldridgeDiD(**options), df, **fit_options) + with pytest.raises(expected_error, match=match): + self._fit( + WooldridgeDiD(**options, unsupported_period_action="error"), + df[df.time < 6], + **fit_options, + ) + + @pytest.mark.parametrize("method", ["logit", "poisson"]) + def test_outcomes_in_discarded_periods_are_not_preflighted(self, method): + """Full-frame outcome validation would break an existing successful fit.""" + df = self._panel() + df.loc[df.time == 6, "y"] = 2.0 if method == "logit" else -1.0 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = self._fit(WooldridgeDiD(method=method), df) + supported = self._fit( + WooldridgeDiD(method=method, unsupported_period_action="error"), df[df.time < 6] + ) + assert result.n_obs == len(df[df.time < 6]) + assert np.isfinite(result.att) and np.isfinite(result.se) + self._assert_same_estimates(result, supported) + @pytest.mark.parametrize("method", ["ols", "logit", "poisson"]) @pytest.mark.parametrize("control", ["never_treated", "not_yet_treated"]) @pytest.mark.parametrize("action", ["drop", "error"])