diff --git a/README.md b/README.md index c85a6da35..6a174dbef 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ Full guide: `diff_diff.get_llm_guide("practitioner")`. - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html) - Cattaneo, Jansson & Ma (2020) density-discontinuity test (`RDDensityTest`): rddensity 3.0 parity, robust bias-corrected inference, unrestricted/restricted models, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - placebo timing, group, permutation, leave-one-out +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html) - Baker et al. (2025) implicit weights a TWFE regression places on each ATT(g,t), against the ATT^O / ATT^simple targets, with the pre-trend contribution. Ports Callaway's `twfeweights` (MIT) - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html) - Rambachan & Roth (2023) sensitivity analysis: robust CI under PT violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html) - Roth (2022) minimum detectable violation and power curves - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html) - analytical and simulation-based MDE, sample size, power curves for study design diff --git a/benchmarks/R/generate_twfeweights_golden.R b/benchmarks/R/generate_twfeweights_golden.R new file mode 100644 index 000000000..5e3e619a5 --- /dev/null +++ b/benchmarks/R/generate_twfeweights_golden.R @@ -0,0 +1,507 @@ +#!/usr/bin/env Rscript +# Generate R `twfeweights` parity goldens for the diff-diff TWFE weight diagnostics. +# +# Requires: twfeweights (>= 0.9.0, MIT, Brantly Callaway), did, fixest, BMisc, +# DRDID, jsonlite +# Output: benchmarks/data/twfeweights_golden.json +# benchmarks/data/twfeweights_sim_panel.csv +# benchmarks/data/twfeweights_unbalanced_panel.csv +# +# The mpdta fixture reads the EXISTING benchmarks/data/mpdta_stata_panel.csv +# rather than writing a renamed copy of it; this script asserts the two agree +# bit-for-bit on every shared column before using it. +# +# Run from the repository root: +# Rscript benchmarks/R/generate_twfeweights_golden.R +# +# --------------------------------------------------------------------------- +# WHAT THIS PINS +# +# `diff_diff/twfe_weights.py` exposes two entry points, each folding several +# upstream R functions: +# +# attgt_weights(aggregation=) <- twfe_weights / attO_weights / +# att_simple_weights +# decompose_twfe_weights(method=) <- implicit_twfe_weights / +# implicit_aipw_weights +# +# plus covariate balance as a result-object method (<- twfe_cov_bal / +# aipw_cov_bal / mp_covariate_bal_summary_helper). +# +# RESERVED BLOCKS (pinned, but not yet read by any test): `decompose.aipw`, +# `balance.aipw` and `two_period.*` pin implicit_aipw_weights, aipw_cov_bal +# and the two_period_reg_weights / two_period_aipw_weights kernels. None of +# these has a Python surface yet - `method="aipw"` is a documented follow-up - +# so they are captured here so that follow-up needs no R re-run. Note the AIPW +# golden is COVARIATE-ADJUSTED: a time-invariant covariate is annihilated by +# double-demeaning but is NOT a no-op in a propensity score. +# +# --------------------------------------------------------------------------- +# NOTE (upstream bug — do NOT "simplify" the no-covariate calls below) +# +# twfeweights::implicit_twfe_weights() with xformula = ~1 (or NULL) builds +# model.matrix(BMisc::addCovToFormla("-1", ~1), data), an nT x 0 matrix, and +# fixest::demean() SEGFAULTS on a zero-column matrix. Reproduced in isolation +# on R 4.6.1 / fixest 0.14.2: +# +# fixest::demean(matrix(numeric(0), nrow = 10, ncol = 0), ids) +# *** caught segfault *** address ..., cause 'memory not mapped' +# +# This is a zero-column bug, NOT a conditioning problem with any particular +# fixture — every fixture hits it on the ~1 branch and no fixture hits it +# otherwise. +# +# The no-covariate decomposition is therefore generated by passing a +# TIME-INVARIANT covariate: double-demeaning annihilates such a regressor +# exactly, so the call is numerically identical to the ~1 branch. Verified on +# mpdta, where `lpop` is time-invariant within county: +# +# twfe_weights(att_gt(...)) aggregate = -0.03654894 +# implicit_twfe_weights(xformula = ~lpop) = -0.03654894 (exact) +# +# The Python parity test asserts BOTH covariates=None AND +# covariates=[] against this single golden, so the equivalence is +# proven by the test rather than assumed by the generator. +# --------------------------------------------------------------------------- + +suppressPackageStartupMessages({ + library(twfeweights) + library(did) + library(jsonlite) +}) + +stopifnot(packageVersion("twfeweights") == "0.9.0") + +# BMisc 1.4.9 emits .Deprecated warnings for makeBalancedPanel / addCovToFormla +# / getListElement on every internal call; they would otherwise flood the log. +quiet <- function(expr) suppressWarnings(suppressMessages(expr)) + +out_dir <- file.path("benchmarks", "data") +if (!dir.exists(out_dir)) { + stop("run this script from the repository root (", out_dir, " not found)") +} + +# --------------------------------------------------------------------------- +# Extraction helpers +# +# Field names below are read off the upstream S3 objects: +# mp_weights_obj $weights_df: group, time.period, weight, attgt, post +# decomposed_twfe $twfe_gt[[i]]: g, tp, weighted_outcome_diff (= ATT(g,t)), +# alpha_weight, ess, remainder, cov_bal_df +# decomposed_aipw $aipw_gt[[i]]: g, tp, est (= ATT(g,t)), att_weight, ess +# The scalar roll-ups mirror summary.decomposed_twfe / summary.decomposed_aipw +# exactly (twfeweights_mp.R:337 and :995). +# --------------------------------------------------------------------------- + +extract_mp_weights <- function(obj) { + df <- obj$weights_df + # mp_weights_obj stores `post` as a FACTOR (for ggplot colouring), so + # as.integer() would emit level codes 1/2 rather than the values 0/1. + # Round-trip through character. + post <- as.integer(as.character(df$post)) + stopifnot(all(post %in% c(0L, 1L))) + list( + group = as.numeric(df$group), + time = as.numeric(df$time.period), + post = post, + weight = as.numeric(df$weight), + att = as.numeric(df$attgt), + implied_att = sum(df$weight * df$attgt) + ) +} + +# implicit_* run in POSITIONAL time (BMisc::orig2t), so their $g / $tp are +# 1..T. attgt_weights' goldens carry RAW labels. Map back here so every block +# in the JSON shares one convention and the Python tests can assert labels. +to_orig <- function(pos, periods) { + out <- as.numeric(pos) + keep <- !is.na(out) & out >= 1 & out <= length(periods) + out[keep] <- as.numeric(periods[out[keep]]) + out +} + +extract_fwl <- function(obj, periods) { + cells <- obj$twfe_gt + g <- unlist(BMisc::getListElement(cells, "g")) + tp <- unlist(BMisc::getListElement(cells, "tp")) + att <- unlist(BMisc::getListElement(cells, "weighted_outcome_diff")) + wt <- unlist(BMisc::getListElement(cells, "alpha_weight")) + ess <- unlist(BMisc::getListElement(cells, "ess")) + rem <- unlist(BMisc::getListElement(cells, "remainder")) + post <- 1 * (tp >= g) + g <- to_orig(g, periods) + tp <- to_orig(tp, periods) + list( + cells = list( + group = as.numeric(g), time = as.numeric(tp), post = as.integer(post), + att = as.numeric(att), weight = as.numeric(wt), + ess = as.numeric(ess), remainder = as.numeric(rem) + ), + estimate = obj$est, + decomposition = obj$decomposition_est, + remainder = obj$decomposition_remainder, + pretrend_bias = obj$pt_violations_bias, + post_only = sum(wt[post == 1] * att[post == 1]), + # summary.decomposed_twfe:351 + effective_sample_size = sum(post) * sum(wt[post == 1] * ess[post == 1]) + ) +} + +extract_aipw <- function(obj, periods) { + cells <- obj$aipw_gt + g <- unlist(BMisc::getListElement(cells, "g")) + tp <- unlist(BMisc::getListElement(cells, "tp")) + att <- unlist(BMisc::getListElement(cells, "est")) + wt <- unlist(BMisc::getListElement(cells, "att_weight")) + ess <- unlist(BMisc::getListElement(cells, "ess")) + post <- 1 * (tp >= g) + g <- to_orig(g, periods) + tp <- to_orig(tp, periods) + list( + cells = list( + group = as.numeric(g), time = as.numeric(tp), post = as.integer(post), + att = as.numeric(att), weight = as.numeric(wt), ess = as.numeric(ess) + ), + estimate = obj$est, + decomposition = obj$decomposition_est, + remainder = obj$decomposition_remainder, + pretrend_bias = obj$pt_violations_bias, + post_only = sum(wt[post == 1] * att[post == 1]), + # summary.decomposed_aipw:1006 — note the inner sum is NOT post-filtered, + # unlike the twfe roll-up. Preserved verbatim. + effective_sample_size = sum(post) * sum(wt * ess) + ) +} + +# Per-cell balance tables, one row per (g, t) x covariate. +extract_balance_cells <- function(cells, periods) { + g <- unlist(BMisc::getListElement(cells, "g")) + tp <- unlist(BMisc::getListElement(cells, "tp")) + dfs <- BMisc::getListElement(cells, "cov_bal_df") + post_i <- 1 * (tp >= g) + g_o <- to_orig(g, periods) + tp_o <- to_orig(tp, periods) + rows <- do.call(rbind.data.frame, lapply(seq_along(dfs), function(i) { + d <- dfs[[i]] + cbind.data.frame( + group = g_o[i], time = tp_o[i], post = post_i[i], + covariate = rownames(d), d, row.names = NULL + ) + })) + as.list(lapply(rows, function(col) if (is.character(col)) col else as.numeric(col))) +} + +extract_balance_summary <- function(cells) { + s <- quiet(mp_covariate_bal_summary_helper(cells)) + out <- as.list(lapply(s, as.numeric)) + out$covariate <- rownames(s) + out +} + +extract_two_period <- function(obj) { + list( + estimate = as.numeric(obj$est), + ess = if (is.null(obj$ess)) NA_real_ else as.numeric(obj$ess), + weights = as.numeric(obj$weights), + dy = as.numeric(obj$dy), + treatment = as.numeric(obj$D) + ) +} + +# --------------------------------------------------------------------------- +# Per-fixture golden bundle +# +# `invariant_cov` must be TIME-INVARIANT within unit — it drives the +# no-covariate branch (see the segfault note at the top). +# `varying_cov` must be genuinely time-varying, so the covariate-adjusted +# weights differ from the unadjusted ones. +# --------------------------------------------------------------------------- + +build_fixture <- function(df, data_file, outcome, unit, time, first_treat, + invariant_cov, varying_cov, two_period_g, + data_file_out = NULL, columns_out = NULL, + derived_columns = NULL) { + stopifnot(all(tapply(df[[invariant_cov]], df[[unit]], + function(z) length(unique(z))) == 1)) + periods <- sort(unique(df[[time]])) + + # Slice the two-period sub-panel FIRST. Several upstream entry points + # (did::att_gt, and BMisc helpers reached from implicit_*) call + # data.table::setDT() on the frame they are handed, which converts it BY + # REFERENCE — after that, `df[cond, ]` silently takes data.table semantics + # and errors. Taking the subset up front sidesteps the whole problem. + tp_periods <- c(two_period_g - 1, two_period_g) + sub <- as.data.frame(df)[df[[time]] %in% tp_periods & + df[[first_treat]] %in% c(0, two_period_g), ] + + ag <- quiet(att_gt( + yname = outcome, tname = time, idname = unit, gname = first_treat, + xformla = ~1, data = df, control_group = "nevertreated", + base_period = "universal", bstrap = FALSE + )) + # did::att_gt calls data.table::setDT(data), which converts the caller's + # frame BY REFERENCE. Everything below assumes data.frame `[` semantics, so + # convert back explicitly rather than relying on what att_gt left behind. + df <- as.data.frame(df) + + inv_f <- as.formula(paste0("~", invariant_cov)) + var_f <- as.formula(paste0("~", varying_cov)) + bal_f <- as.formula(paste0("~", invariant_cov, "+", varying_cov)) + + common <- list(yname = outcome, tname = time, idname = unit, + gname = first_treat, data = df) + + fwl_nocov <- quiet(do.call(implicit_twfe_weights, + c(common, list(xformula = inv_f)))) + fwl_cov <- quiet(do.call(implicit_twfe_weights, + c(common, list(xformula = var_f)))) + fwl_gmin1 <- quiet(do.call(implicit_twfe_weights, + c(common, list(xformula = inv_f, + base_period = "gmin1")))) + aipw <- quiet(do.call(implicit_aipw_weights, + c(common, list(xformula = inv_f)))) + + # Balance is taken off the COVARIATE-ADJUSTED decomposition: on the + # no-covariate branch the implicit weights are constant within the treated + # and comparison groups, so weighted == unweighted and the table is + # degenerate (verified). fwl_cov gives a non-trivial reweighting. + bal_fwl <- quiet(twfe_cov_bal(fwl_cov, bal_f)) + bal_aipw <- quiet(aipw_cov_bal(aipw, bal_f)) + + # Two-period kernels: the (g = two_period_g) cohort against never-treated, + # over periods {g-1, g}. These are private in Python, so this is the only + # place they are pinned. (`sub` was sliced at the top of this function.) + sub_common <- list(yname = outcome, tname = time, idname = unit, + gname = first_treat, data = sub) + tp_reg <- quiet(do.call(two_period_reg_weights, + c(sub_common, list(xformula = var_f)))) + tp_aipw <- quiet(do.call(two_period_aipw_weights, + c(sub_common, list(xformula = var_f)))) + + out <- list( + data_file = if (is.null(data_file_out)) data_file else data_file_out, + columns = if (is.null(columns_out)) { + list(outcome = outcome, unit = unit, time = time, + first_treat = first_treat, + invariant_cov = invariant_cov, varying_cov = varying_cov) + } else { + columns_out + }, + two_period_group = two_period_g, + attgt_weights = list( + twfe = extract_mp_weights(quiet(twfe_weights(ag))), + overall = extract_mp_weights(quiet(attO_weights(ag))), + simple = extract_mp_weights(quiet(att_simple_weights(ag))) + ), + decompose = list( + fwl_nocov = extract_fwl(fwl_nocov, periods), + fwl_cov = extract_fwl(fwl_cov, periods), + fwl_gmin1 = extract_fwl(fwl_gmin1, periods), + aipw = extract_aipw(aipw, periods) + ), + balance = list( + fwl = list(cells = extract_balance_cells(bal_fwl$twfe_gt, periods), + summary = extract_balance_summary(bal_fwl$twfe_gt)), + aipw = list(cells = extract_balance_cells(bal_aipw$aipw_gt, periods), + summary = extract_balance_summary(bal_aipw$aipw_gt)) + ), + two_period = list(reg = extract_two_period(tp_reg), + aipw = extract_two_period(tp_aipw)) + ) + if (!is.null(derived_columns)) { + out$derived_columns <- derived_columns + } + out +} + +# --------------------------------------------------------------------------- +# Fixture 1 — mpdta (real data) +# +# did::mpdta: 500 US counties x 2003-2007, cohorts {2004, 2006, 2007} plus +# never-treated (first.treat == 0), time-invariant `lpop`. Also exercises +# non-1..T time labels, which the Python side handles by positional rescaling. +# --------------------------------------------------------------------------- + +data(mpdta, package = "did") +mpdta_df <- data.frame( + unit = as.numeric(mpdta$countyreal), + period = as.numeric(mpdta$year), + first_treat = as.numeric(mpdta$first.treat), + outcome = as.numeric(mpdta$lemp), + lpop = as.numeric(mpdta$lpop) +) +mpdta_df <- mpdta_df[order(mpdta_df$unit, mpdta_df$period), ] +# `lpop` is time-invariant; build a genuinely time-varying companion from it so +# the covariate-adjusted branch is non-degenerate on this fixture too. +mpdta_df$lpop_t <- mpdta_df$lpop * (mpdta_df$period - 2002) / 5 + +# The fixture READS benchmarks/data/mpdta_stata_panel.csv (already in the repo +# for the Stata parity suites) instead of writing a renamed copy. Assert the +# two sources agree bit-for-bit on every shared column, so they cannot drift. +stata_path <- file.path(out_dir, "mpdta_stata_panel.csv") +if (!file.exists(stata_path)) { + stop("expected ", stata_path, " (the mpdta fixture now reads it)") +} +stata_df <- read.csv(stata_path) +stata_df <- stata_df[order(stata_df$countyreal, stata_df$year), ] +# The identifiers must match exactly; the float columns are compared at CSV +# round-trip precision, NOT bit-for-bit. write.csv emits 15 significant digits, +# so a CSV column always sits within ~1e-15 relative of the in-memory double it +# came from. That gap is pre-existing and unchanged by this switch: the fixture +# previously read twfeweights_mpdta_panel.csv, itself a 15-digit round-trip of +# these same values, and the parity tolerances already absorb it. +rt_tol <- 1e-14 +stopifnot( + nrow(stata_df) == nrow(mpdta_df), + identical(as.numeric(stata_df$countyreal), mpdta_df$unit), + identical(as.numeric(stata_df$year), mpdta_df$period), + identical(as.numeric(stata_df$first.treat), mpdta_df$first_treat), + max(abs(as.numeric(stata_df$lemp) - mpdta_df$outcome)) <= + rt_tol * max(1, max(abs(mpdta_df$outcome))), + max(abs(as.numeric(stata_df$lpop) - mpdta_df$lpop)) <= + rt_tol * max(1, max(abs(mpdta_df$lpop))) +) + +# --------------------------------------------------------------------------- +# Fixture 2 — sim_staggered (simulated) +# +# 3 equal cohorts of 100, which keeps the AIPW propensity score bounded away +# from 0/1 (100 controls per cell). The equal cohorts are also exactly what +# makes the comparison-group normalizer VANISH at t = 3 (-1/3 + 1/3), so this +# fixture deliberately exercises the documented 0/0 cells - it is not a +# "no cell is degenerate" design. +# +# `0.3 * x1 * period` gives each unit a trend, so pretrend_bias is non-zero, +# but x1 is iid and cohorts are assigned by unit INDEX, so E[x1 | g] does not +# vary by cohort: the differential pre-trend is zero in expectation and the +# observed value (~0.093) is sampling noise, not a designed pre-trend. +# `xtv`'s two structured terms (0.2 * period and 0.5 * x1) are absorbed by the +# two-way fixed effects, so the covariate branch regresses on the residual +# noise - adequate for parity, but not a "well-conditioned" design. +# --------------------------------------------------------------------------- + +make_sim <- function(seed, cohort_sizes, cohort_times, n_periods) { + set.seed(seed) + n <- sum(cohort_sizes) + g <- rep(cohort_times, times = cohort_sizes) + x1 <- rnorm(n) + unit_fe <- rnorm(n) + df <- do.call(rbind, lapply(seq_len(n_periods), function(t) { + data.frame(unit = seq_len(n), period = t, first_treat = g, + x1 = x1, unit_fe = unit_fe) + })) + df <- df[order(df$unit, df$period), ] + df$xtv <- 0.2 * df$period + 0.5 * df$x1 + rnorm(nrow(df), 0, 0.5) + treated <- (df$first_treat != 0) & (df$period >= df$first_treat) + df$outcome <- df$unit_fe + 0.5 * df$period + 0.3 * df$x1 * df$period + + 1.0 * treated * (df$period - df$first_treat + 1) + rnorm(nrow(df)) + df$unit_fe <- NULL + rownames(df) <- NULL + df +} + +sim_df <- make_sim(20260831, c(100, 100, 100), c(0, 3, 4), 5) + +# --------------------------------------------------------------------------- +# Fixture 3 — unbalanced_cohorts +# +# Fixture 2 has equal thirds, so p_g == 1/3 and several of the weight formulas +# coincide — a bug in the cohort-share computation would pass silently there. +# Unequal cohort masses (120 / 70 / 60) break that degeneracy. Do not drop this +# fixture. +# --------------------------------------------------------------------------- + +unb_df <- make_sim(20260901, c(120, 70, 60), c(0, 3, 5), 6) + +# --------------------------------------------------------------------------- +# Build + write +# --------------------------------------------------------------------------- + +write.csv(sim_df, file.path(out_dir, "twfeweights_sim_panel.csv"), + row.names = FALSE) +write.csv(unb_df, file.path(out_dir, "twfeweights_unbalanced_panel.csv"), + row.names = FALSE) + +cat("building mpdta ...\n") +fx_mpdta <- build_fixture( + mpdta_df, "twfeweights_mpdta_panel.csv", + "outcome", "unit", "period", "first_treat", + "lpop", "lpop_t", two_period_g = 2004, + # Emitted names point at the SHARED stata panel; the R calls above keep + # using mpdta_df's own names, so nothing inside build_fixture changes. + data_file_out = "mpdta_stata_panel.csv", + columns_out = list(outcome = "lemp", unit = "countyreal", time = "year", + first_treat = "first.treat", + invariant_cov = "lpop", varying_cov = "lpop_t"), + derived_columns = list(lpop_t = "lpop * (year - 2002) / 5") +) +cat("building sim_staggered ...\n") +fx_sim <- build_fixture(sim_df, "twfeweights_sim_panel.csv", + "outcome", "unit", "period", "first_treat", + "x1", "xtv", two_period_g = 3) +cat("building unbalanced_cohorts ...\n") +fx_unb <- build_fixture(unb_df, "twfeweights_unbalanced_panel.csv", + "outcome", "unit", "period", "first_treat", + "x1", "xtv", two_period_g = 3) + +payload <- list( + meta = list( + description = paste( + "R twfeweights parity goldens for diff_diff attgt_weights /", + "decompose_twfe_weights. Regenerate with:", + "Rscript benchmarks/R/generate_twfeweights_golden.R" + ), + upstream = paste( + "twfeweights (Brantly Callaway), MIT License,", + "Copyright (c) 2023 Brantly Callaway" + ), + r_version = paste(R.version$major, R.version$minor, sep = "."), + twfeweights_version = as.character(packageVersion("twfeweights")), + did_version = as.character(packageVersion("did")), + fixest_version = as.character(packageVersion("fixest")), + BMisc_version = as.character(packageVersion("BMisc")), + DRDID_version = as.character(packageVersion("DRDID")), + seeds = list(sim_staggered = 20260831L, unbalanced_cohorts = 20260901L), + mpdta_provenance = paste( + "fixtures.mpdta is data(mpdta, package = \"did\") version", + as.character(packageVersion("did")), + "- read from the shared benchmarks/data/mpdta_stata_panel.csv, whose", + "columns this generator asserts are bit-identical to data(mpdta).", + "`lpop_t` is derived (see fixtures.mpdta.derived_columns)." + ), + reserved_blocks = paste( + "decompose.aipw, balance.aipw and two_period.* are PINNED BUT UNUSED:", + "they capture implicit_aipw_weights, aipw_cov_bal and the", + "two_period_reg_weights / two_period_aipw_weights kernels, none of which", + "has a Python surface yet (method=\"aipw\" is a documented follow-up).", + "They are kept so that follow-up needs no R re-run. NOTE the AIPW golden", + "is covariate-adjusted: a time-invariant covariate is annihilated by", + "double-demeaning but is NOT a no-op in a propensity score." + ), + label_convention = paste( + "Every cells block (attgt_weights.*, decompose.*, balance.*) carries", + "ORIGINAL period labels. implicit_* run in positional time internally;", + "the generator maps them back before emitting." + ), + no_covariate_note = paste( + "decompose.fwl_nocov is generated with xformula = ~,", + "which is numerically the ~1 branch (double-demeaning annihilates a", + "time-invariant regressor exactly). The ~1 branch itself cannot be", + "called: fixest::demean segfaults on the zero-column model matrix it", + "builds. See the comment block at the top of the generator." + ) + ), + fixtures = list( + mpdta = fx_mpdta, + sim_staggered = fx_sim, + unbalanced_cohorts = fx_unb + ) +) + +out_path <- file.path(out_dir, "twfeweights_golden.json") +write_json(payload, out_path, auto_unbox = TRUE, digits = NA, pretty = TRUE) +cat("wrote", out_path, "\n") +cat(" mpdta twfe implied_att =", fx_mpdta$attgt_weights$twfe$implied_att, "\n") +cat(" mpdta fwl_nocov estimate =", fx_mpdta$decompose$fwl_nocov$estimate, "\n") +cat(" sim fwl_nocov estimate =", fx_sim$decompose$fwl_nocov$estimate, "\n") +cat(" unb twfe implied_att =", fx_unb$attgt_weights$twfe$implied_att, "\n") diff --git a/benchmarks/R/requirements.R b/benchmarks/R/requirements.R index 7e3cb8695..45fcb0b2a 100644 --- a/benchmarks/R/requirements.R +++ b/benchmarks/R/requirements.R @@ -19,6 +19,8 @@ required_packages <- c( "nprobust", # Calonico-Cattaneo-Farrell local-linear (DIDHAD dependency) "Synth", # Abadie-Diamond-Hainmueller (2010) synthetic control (SyntheticControl R-parity; ships data(basque)) "qte", # Callaway qte package (Athey-Imbens CiC + QDiD R-parity; ships data(lalonde)) + "BMisc", # Callaway utility package (twfeweights dependency: weighted_ecdf, orig2t) + "DRDID", # Sant'Anna & Zhao (2020) doubly-robust DiD (twfeweights AIPW dependency) # Utilities "jsonlite", # JSON output for Python interop @@ -27,7 +29,9 @@ required_packages <- c( # synthdid must be installed from GitHub github_packages <- list( - synthdid = "synth-inference/synthdid" + synthdid = "synth-inference/synthdid", + # TWFE weight diagnostics parity goldens (not on CRAN) + twfeweights = "bcallaway11/twfeweights" ) install_if_missing <- function(pkg) { diff --git a/benchmarks/data/twfeweights_golden.json b/benchmarks/data/twfeweights_golden.json new file mode 100644 index 000000000..30cfbe030 --- /dev/null +++ b/benchmarks/data/twfeweights_golden.json @@ -0,0 +1,586 @@ +{ + "meta": { + "description": "R twfeweights parity goldens for diff_diff attgt_weights / decompose_twfe_weights. Regenerate with: Rscript benchmarks/R/generate_twfeweights_golden.R", + "upstream": "twfeweights (Brantly Callaway), MIT License, Copyright (c) 2023 Brantly Callaway", + "r_version": "4.6.1", + "twfeweights_version": "0.9.0", + "did_version": "2.5.1", + "fixest_version": "0.14.2", + "BMisc_version": "1.4.9", + "DRDID_version": "1.3.0", + "seeds": { + "sim_staggered": 20260831, + "unbalanced_cohorts": 20260901 + }, + "mpdta_provenance": "fixtures.mpdta is data(mpdta, package = \"did\") version 2.5.1 - read from the shared benchmarks/data/mpdta_stata_panel.csv, whose columns this generator asserts are bit-identical to data(mpdta). `lpop_t` is derived (see fixtures.mpdta.derived_columns).", + "reserved_blocks": "decompose.aipw, balance.aipw and two_period.* are PINNED BUT UNUSED: they capture implicit_aipw_weights, aipw_cov_bal and the two_period_reg_weights / two_period_aipw_weights kernels, none of which has a Python surface yet (method=\"aipw\" is a documented follow-up). They are kept so that follow-up needs no R re-run. NOTE the AIPW golden is covariate-adjusted: a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score.", + "label_convention": "Every cells block (attgt_weights.*, decompose.*, balance.*) carries ORIGINAL period labels. implicit_* run in positional time internally; the generator maps them back before emitting.", + "no_covariate_note": "decompose.fwl_nocov is generated with xformula = ~, which is numerically the ~1 branch (double-demeaning annihilates a time-invariant regressor exactly). The ~1 branch itself cannot be called: fixest::demean segfaults on the zero-column model matrix it builds. See the comment block at the top of the generator." + }, + "fixtures": { + "mpdta": { + "data_file": "mpdta_stata_panel.csv", + "columns": { + "outcome": "lemp", + "unit": "countyreal", + "time": "year", + "first_treat": "first.treat", + "invariant_cov": "lpop", + "varying_cov": "lpop_t" + }, + "two_period_group": 2004, + "attgt_weights": { + "twfe": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "weight": [-0.113075467453585, 0.0457198057404491, 0.0457198057404491, 0.0324868663076129, -0.0108510103349257, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110522, 0.578994031944316], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "implied_att": -0.0365489366740672 + }, + "overall": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "weight": [0, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0, 0, 0, 0.104712041884817, 0.104712041884817, 0, 0, 0, 0, 0.68586387434555], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "implied_att": -0.031018282228749 + }, + "simple": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "weight": [0, 0.0687285223367698, 0.0687285223367698, 0.0687285223367698, 0.0687285223367698, 0, 0, 0, 0.13745704467354, 0.13745704467354, 0, 0, 0, 0, 0.450171821305842], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "implied_att": -0.039951275155177 + } + }, + "decompose": { + "fwl_nocov": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, 0, 0.00652011242423291, 0.00376929367371423, -0.000825313279148546, -0.0374551778725037, 0, 0.0305066555832921, 0.0277807626971762, -0.00330635669251204, -0.0293607674117092], + "weight": [-0.113075467453585, 0.0457198057404492, 0.0457198057404492, 0.0324868663076129, -0.0108510103349258, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110521, 0.578994031944315], + "ess": [309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": -0.0365489366740672, + "decomposition": -0.0365489366740672, + "remainder": 0, + "pretrend_bias": -0.00817768207864544, + "post_only": -0.0283712545954217, + "effective_sample_size": 2163 + }, + "fwl_cov": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "att": [0, -0.0133665432176392, -0.0704231581031491, -0.267917985756793, -0.0863164461874393, 0, 0.00207806356666779, 0.00376929367371422, -0.126820852395217, -0.0470847199292006, 0, 0.0267475453560011, 0.0277807626971762, -0.12743173909835, -0.0323931143733938], + "weight": [-0.1135230713667, 0.0464885411048002, 0.0461635736399947, 0.0324772245138304, -0.0116062678919253, -0.0912244608682691, -0.10633971008501, -0.108093577640393, 0.197464333015477, 0.108193415578195, -0.0876606039551204, -0.133316485481803, -0.135213842067535, -0.224628248535168, 0.580819180039627], + "ess": [288.006463639308, 296.614163218515, 309, 5.4990531247817, 304.859399183937, 288.006463639308, 296.614163218515, 309, 5.4990531247817, 304.859399183937, 288.006463639308, 296.614163218515, 309, 5.4990531247817, 304.859399183937], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": -0.0398490715024001, + "decomposition": -0.0398490715024001, + "remainder": 0, + "pretrend_bias": 0.0206741188438872, + "post_only": -0.0605231903462873, + "effective_sample_size": 1650.8229562172 + }, + "fwl_gmin1": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286273, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "weight": [-0.113075467453585, 0.0457198057404492, 0.0457198057404492, 0.0324868663076129, -0.0108510103349258, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110521, 0.578994031944315], + "ess": [309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309], + "remainder": [-5.65463002249983, -5.65463002249983, -5.65463002249983, -5.65463002249983, -5.65463002249983, -5.60480843375399, -5.60480843375399, -5.60480843375399, -5.60480843375399, -5.60480843375399, -5.63889628205466, -5.63889628205466, -5.63889628205466, -5.63889628205467, -5.63889628205466] + }, + "estimate": -0.0365489366740662, + "decomposition": -0.0365489366740672, + "remainder": 9.57567358739198e-16, + "pretrend_bias": -0.00893136238322589, + "post_only": -0.0276175742908413, + "effective_sample_size": 2163 + }, + "aipw": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2006, 2006, 2007], + "time": [2004, 2005, 2006, 2007, 2006, 2007, 2007], + "post": [1, 1, 1, 1, 1, 1, 1], + "att": [-0.0145296683111164, -0.0764218817440481, -0.140448336820238, -0.106903898121729, 0.000960573746698802, -0.0412938655881805, -0.0287813610394872], + "weight": [0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.104712041884817, 0.104712041884817, 0.68586387434555], + "ess": [293.350247392128, 293.350247392128, 293.350247392128, 293.350247392128, 253.063348228582, 253.063348228582, 295.099929127014] + }, + "estimate": -0.0328195971622414, + "decomposition": -0.0328195971622414, + "remainder": 0, + "pretrend_bias": 0, + "post_only": -0.0328195971622414, + "effective_sample_size": 2002.7927075831 + } + }, + "balance": { + "fwl": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2003, 2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007, 2003, 2003, 2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007, 2003, 2003, 2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007], + "post": [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1], + "covariate": ["mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t"], + "unweighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877], + "unweighted_covs_comparison": [3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216], + "unweighted_diff": [0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174], + "weighted_covs_treated": [3.4193925119911, 2.05163550719466, 3.54599912772186, 2.12759947663312, 3.47639737864719, 2.08583842718831, 3.37676805929306, 2.02606083557584, 4.03397261350373, 2.42038356810224, 3.63829301967841, 2.18297581180704, 3.70441978730257, 2.22265187238154, 3.75408740697126, 2.25245244418276, 3.72734009420077, 2.23640405652046, 3.6564541037253, 2.19387246223518, 2.98671356447979, 1.79202813868788, 3.30348590139512, 1.98209154083708, 3.45864088333128, 2.07518452999877, 3.55072510262075, 2.13043506157245, 3.38741487149293, 2.03244892289576], + "weighted_covs_comparison": [3.53045958121647, 2.11827574872988, 3.4460894159446, 2.06765364956676, 3.18343355120266, 1.9100601307216, 12.7324015067384, 7.63944090404307, 3.33323025444049, 1.99993815266429, 3.53045958121647, 2.11827574872988, 3.4460894159446, 2.06765364956676, 3.18343355120266, 1.9100601307216, 12.7324015067384, 7.63944090404307, 3.33323025444049, 1.99993815266429, 3.53045958121647, 2.11827574872988, 3.4460894159446, 2.06765364956676, 3.18343355120266, 1.9100601307216, 12.7324015067384, 7.63944090404307, 3.33323025444049, 1.99993815266429], + "weighted_diff": [-0.111067069225372, -0.066640241535223, 0.099909711777264, 0.059945827066358, 0.292963827444527, 0.175778296466716, -9.35563344744539, -5.61338006846723, 0.700742359063241, 0.420445415437945, 0.107833438461934, 0.0647000630771606, 0.258330371357971, 0.154998222814783, 0.570653855768603, 0.342392313461162, -9.00506141253767, -5.40303684752261, 0.32322384928481, 0.193934309570886, -0.54374601673668, -0.326247610042007, -0.142603514549474, -0.0855621087296847, 0.275207332128623, 0.165124399277174, -9.1816764041177, -5.50900584247062, 0.0541846170524449, 0.0325107702314671], + "sd": [1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162], + "unweighted_log_ratio_sd_diff": [-1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315], + "weighted_log_ratio_sd_diff": [-2.16928559609177, -2.16928559609177, -1.69865356192303, -1.69865356192303, -1.40022970637138, -1.40022970637138, -4.92827389243409, -4.92827389243409, -0.738212307151284, -0.738212307151284, -2.24359876506552, -2.24359876506552, -1.8116712863662, -1.8116712863662, -1.14224374564749, -1.14224374564749, -4.44862281223905, -4.44862281223905, -1.84942446599253, -1.84942446599253, -1.54827828974358, -1.54827828974358, -1.33211844247205, -1.33211844247205, -0.450837075241316, -0.450837075241316, -3.45912678775289, -3.45912678775289, -0.922554990848926, -0.922554990848926], + "unweighted_frac_treated_extreme": [0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764], + "weighted_frac_treated_extreme": [0, 0, 0, 0, 0.05, 0.05, 0, 0, 0.2, 0.2, 0, 0, 0, 0, 0.075, 0.075, 0, 0, 0, 0, 0.0229007633587786, 0.0229007633587786, 0.0229007633587786, 0.0229007633587786, 0.0610687022900764, 0.0610687022900764, 0, 0, 0.0305343511450382, 0.0305343511450382] + }, + "summary": { + "unweighted_treat": [3.55096217446187, 2.13057730467712], + "unweighted_untreat": [3.18343355120266, 1.91006013072159], + "unweighted_diff": [0.367528623259216, 0.22051717395553], + "weighted_treat": [3.48727645229424, 2.09236587137654], + "weighted_untreat": [5.49282183830626, 3.29569310298376], + "weighted_diff": [-2.00554538601202, -1.20332723160721], + "sd": [1.27667557327224, 0.766005343963346], + "unweighted_log_ratio_sd_diff": [-0.769948848899112, -0.769948848899111], + "weighted_log_ratio_sd_diff": [-1.90947382058374, -1.90947382058374], + "unweighted_frac_treated_extreme": [0.0640703583030666, 0.0640703583030666], + "weighted_frac_treated_extreme": [0.0177218618987178, 0.0177218618987178], + "covariate": ["mean_lpop", "mean_lpop_t"] + } + }, + "aipw": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2007, 2007], + "time": [2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007, 2006, 2006, 2007, 2007, 2007, 2007], + "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "covariate": ["mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t"], + "unweighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], + "unweighted_covs_comparison": [3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216], + "unweighted_diff": [0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.275207332128624, 0.165124399277174], + "weighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], + "weighted_covs_comparison": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], + "weighted_diff": [8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 8.88178419700125e-16], + "sd": [1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.27801007041527, 0.766806042249162] + }, + "summary": { + "unweighted_treat": [3.52237381972876, 2.11342429183725], + "unweighted_untreat": [3.18343355120266, 1.9100601307216], + "unweighted_diff": [0.338940268526096, 0.203364161115658], + "weighted_treat": [3.52237381972876, 2.11342429183725], + "weighted_untreat": [3.52237381972875, 2.11342429183725], + "weighted_diff": [8.88178419700125e-16, 7.4867395587288e-16], + "sd": [1.27727853948457, 0.766367123690741], + "unweighted_log_ratio_sd_diff": [0, 0], + "weighted_log_ratio_sd_diff": [0, 0], + "unweighted_frac_treated_extreme": [0, 0], + "weighted_frac_treated_extreme": [0, 0], + "covariate": ["mean_lpop", "mean_lpop_t"] + } + } + }, + "two_period": { + "reg": { + "estimate": -0.0149216931463987, + "ess": 295.437073557126, + "weights": [0.914205378751222, 1.10807424931814, 0.933944555836912, 1.30885089779359, 0.77667566709387, 1.09886549432803, 0.857802290117748, 1.29533608776587, 1.21797362329684, 0.890296869666088, 0.876826211579155, 1.22341271500028, 0.861220240964523, 1.2217569875752, 1.23432024149811, 1.10053686950126, 1.53243455097736, 0.991986984922291, 0.852012353181866, 0.892799434382799, 1.15625878851813, 1.01070586338101, 0.797104804854826, 0.82150263374418, 0.925758568885049, 1.06901911216371, 1.15731335732235, 1.20326032975876, 0.991885473966669, 1.07626137455175, 0.958503061349015, 0.921310469219996, 0.781305672392859, 0.880672266280691, 0.868599070143875, 1.15374257218318, 0.978235389547607, 1.20698785760044, 0.864242279469781, 0.984348590136261, 0.677018937975777, 0.78566066241169, 1.07051467897138, 0.851002176458188, 0.800796993125892, 0.873966920756733, 0.922755161442552, 0.970155022871146, 0.906505733152204, 1.16334110821022, 1.00654509305044, 1.00714845007322, 1.00861119030535, 1.01139099432848, 1.01655775675883, 0.999377477471499, 1.00885020589396, 0.99839371957829, 0.972753520276552, 0.967719834724691, 0.983415466966356, 1.00756804487529, 1.00057938242599, 1.01802260153828, 0.983491689143243, 1.0162255160028, 1.00802635808135, 0.993215960705129, 0.993102181707199, 0.999004556093054, 1.18092367771598, 1.08808685196963, 1.23100981435726, 1.0159731214296, 1.05650610397299, 1.10794776362609, 1.08543904432851, 1.26567040981729, 1.18545393782311, 1.24368399209914, 1.26059743946738, 1.08112581656952, 1.25304541183838, 1.10727033756971, 1.10436293348918, 1.1694637092217, 1.10854681715011, 0.757196653021297, 0.98296939076698, 1.06678038285255, 1.09802731714199, 1.30342090571583, 1.02010825188693, 1.02240090966505, 1.0076789351107, 1.27816316617544, 0.870787994093949, 0.864507234249911, 0.874319684458889, 0.958503061349015, 0.971488264668903, 1.07593314326871, 1.34547694644207, 0.987118006851055, 0.867133874241953, 0.880346868666857, 0.92188543431963, 1.45715035175091, 0.958529642604906, 0.752567595243238, 0.97420384402187, 0.787153467781112, 1.08523202059903, 0.864818407544575, 0.989121047467235, 0.832273511291086, 0.833084041058782, 1.23693372233058, 0.675719067675065, 0.656205952440136, 0.89244421716068, 0.829434045397887, 1.17439425571453, 0.941590453075537, 0.766703290704483, 0.772482352776408, 0.799168669044801, 0.74157547066872, 0.721387168342452, 0.858610943520528, 0.994797353734057, 1.47335777302894, 1.18514252521156, 1.0803767751372, 1.50009361600496, 1.30150183654471, 0.846743269804263, 1.17080805501646, 0.861696060941864, 0.990574796380501, 1.12968407360812, 1.09171654630869, 0.804140956149301, 0.831231009547144, 0.772076141498604, 0.850308351294569, 0.880658131758654, 1.49267787652659, 0.784758519879128, 0.681653607455449, 0.596392417171735, 1.13283686776408, 0.8429053802999, 0.691259374141176, 0.833271972854951, 0.842390476649212, 1.38993252780947, 0.75168132143843, 0.806891915601875, 0.925099414970179, 0.65723510736642, 0.845159246065923, 0.90737500976245, 1.27095562512986, 0.967288076540284, 0.773426347208265, 0.958219263901373, 1.00982894404555, 1.32997931908278, 1.13888195173276, 0.74351757596311, 1.25854765822588, 0.628166041980897, 0.761122527803733, 0.606941516111651, 1.16800760300955, 0.64339210703374, 0.767652647794539, 0.784683120487471, 0.688103599457281, 0.764929768007258, 0.825623696972631, 0.888876693782396, 0.966707035387086, 0.770906152874736, 1.06914691812822, 1.03713164810658, 1.21511287945244, 0.660496935454807, 1.01335391953086, 0.91107858003953, 0.86150589590303, 0.856908216912444, 1.05679134782963, 1.17622845309429, 1.17317400146166, 1.0970151373967, 1.52515362690788, 1.1447241641711, 0.955589011684891, 1.32121999096743, 0.938232159752092, 1.2686858175866, 0.922799077792815, 1.42558003259381, 1.05749743636383, 1.09540926270813, 1.00318548722524, 1.34970798424194, 1.13005176807204, 1.17650664269292, 1.06420125597473, 1.16827467058984, 0.961741161438828, 1.03563770711794, 1.06417305269481, 0.565642036368166, 1.01178430075611, 0.80013382343569, 0.804163323876345, 0.93645145535438, 0.498752941366539, 0.873185246178276, 0.591658673083957, 0.671509856439639, 0.85343303820078, 0.804185688602419, 0.543035144130157, 0.836526939309612, 0.937016901846683, 0.888580417998956, 1.24643165992158, 1.0836459985709, 1.0550178960445, 1.02953012986436, 1.42446146773525, 1.13292714422746, 0.986819202405292, 1.01935952633555, 1.08394687794462, 1.28673644369391, 1.12770172727897, 1.60279560849242, 0.94803517304973, 0.94936028579014, 0.996230905422344, 0.947827464556498, 1.0489182301759, 0.824459438478706, 0.833102843775575, 1.04806606792118, 1.43863822710714, 1.10992438468713, 0.869145215288365, 1.75368577552688, 0.956263947765851, 1.48129133194083, 0.714177591294102, 0.526137284043039, 1.18516755063542, 1.08877777658533, 1.04685002787374, 1.39110059759066, 1.08142279773759, 0.672301055720514, 1.11637172339878, 1.17790730294754, 0.655607208827931, 0.782634337497015, 0.728931237851374, 1.1121882817121, 0.969716513585689, 0.921553965756354, 0.8357145095486, 1.01889756947789, 0.85343303820078, 0.669665695616983, 0.65506102391745, 1.09662092099898, 0.876391731353484, 1.17058226607269, 0.935233286711417, 0.668458481030845, 0.571884667288901, 0.673529835319247, 1.10335516966084, 1.03836004925978, 1.04917045987158, 0.950619215471048, 0.88606795324462, 0.87790748338468, 1.09531959221919, 1.38223356446573, 0.820856369845608, 0.914101320529017, 1.22008833913762, 1.19820046386571, 1.16675729242193, 0.739214676891295, 1.15287231341345, 1.0390337443305, 0.954983748994095, 1.3962143210842, 1.05844399017882, 1.00276474444757, 1.61570706617249, 1.13776110303408, 0.950377493029127, 0.92344550363475, 1.14599221194468, 1.01003722393432, 0.89876316093955, 1.20719528706948, 1.05242527002734, 1.10239361278195, 0.776120851588995, 0.94518023966827, 0.962590677452325, 1.30057892353915, 1.08619456613142, 1.16587595058709, 1.33511731721303, 1.37900024035138, 0.877418175728803, 1.22829047749297, 0.998079075301512, 1.16175961199361, 0.883530934105611, 0.996195569866188], + "dy": [0.0402064204780404, -0.00617921636595753, -0.00769234562315635, -0.0109482153338982, 0.271933715483642, 0.0670107102829602, -0.159064694629687, 0.0776028399652979, -0.0348005291494164, -0.361790044605503, -0.023530497410194, 0.0407981988772361, -0.039441732051297, 0.0183057960884394, 0.0578857923998042, -0.0689928714869517, -0.0489120485013519, -0.141078598259906, -0.231801614057324, -0.0979804083602032, -0.107245530353598, 0.0255333020051651, -0.100083458556983, -0.42744401482694, -0.00557104504945549, -0.0268472500361883, 0.0447260357486368, 0.0994408366375259, 0.115069329784787, -0.0392207131532816, 0.0330062964681703, -0.188348264660815, 0.37469344944141, 0.185403223331362, 0.105360515657826, -0.123805501162851, -0.127561225638019, 0.00391083799199876, -0.182321556793955, 0.0138891121606672, 0.352821374622742, 0.0202027073175195, 0.00183992692200707, -0.0615578929994332, -0.145851877012563, -0.101020402315209, -0.0900261516577192, 0.00645163528148807, 0.0823995317670665, -0.0217808084613926, -0.118481603620523, -0.00881062968215485, -0.0836500292869244, -0.332133835022615, 0.0377403279828474, -0.0805032188490706, -0.127833371509885, -0.103666774395267, -0.0391226547103702, -0.0522125710903065, -0.114421687953678, 0.00524935588614373, 0.024916351264535, -0.287682072451781, -0.052592450119171, 0.0108109161042158, -0.0619667237491992, -0.049576446606288, 0.0494724535390656, -0.0782007473074744, -0.0358461317731358, -0.0870113769896301, -0.0922165496035534, 0.264301623807717, -0.0495596909484606, -0.0583355770129632, -0.0856554445784941, -0.11963720201451, -0.0900336686743657, 0.0372967877532266, -0.0117011085851217, -0.081526607634208, -0.0620637099045016, -0.0557419830082351, -0.0589674760881156, -0.0501712931478435, -0.033953878930344, -0.0277795641070755, 0.0201011793210872, 0.0344321964214842, -0.0153377239837091, -0.0636388359666737, -0.0337381396318506, -0.136464423894526, -0.210379509589112, -0.0663859260598123, -0.24231346742193, -0.0682679711290026, -0.0795120629277335, -0.0880333494853227, -0.191268342783761, -0.0861116918187319, -0.0228979640932554, -0.0592556962090933, -0.0804723532225733, 0.0714589639821446, -0.0876299974856956, -0.0657078415693917, -0.249811798396371, -0.111791405988116, -0.168898536461814, -0.316669609325034, -0.0500409460710847, 0.0302398851897179, -0.234400705838844, 0.094778406327289, -0.0897699994287091, -0.155955083059526, 0.143100843640673, -0.0730251350148903, -0.0628009012390303, -0.0145987994211527, -0.00685403642779203, -0.189888919858863, -0.0266682470821618, 0.037740327982847, -0.0111733005981254, -0.117783035656383, -0.0384662808277962, 0, 0.0571584138399484, -0.0694119283885524, -0.0735133828982697, -0.104572803919577, -0.174540558863677, -0.159272329947313, -0.241162056816889, -0.151592882072704, 0.336472236621213, -0.364897923119164, 0.03964483134724, -0.229033345370273, -0.182321556793954, -0.251314428280907, 0.127833371509885, 0.0752830604205483, -0.0685626714372969, -0.0333465799413801, -0.197610249309045, -0.102654154060084, 0.0645385211375711, -0.071686734959413, 0.0268112574506567, -0.11247798342669, -0.116682320080198, 0.143100843640673, -0.0513428079171341, -0.0591888713903304, 0.0285733724440558, -0.2383103438577, -0.0444517625708336, 0.0909717782057271, -0.220542769614153, -0.03650636928728, -0.00735297430525872, -0.136859182717197, -0.0115943327809198, -0.306619273471531, -0.0180556878262053, -0.0133740595252565, -0.847297860387203, -0.0694513863237711, -0.0165293019512109, -0.391671785975829, 0, -0.0750083720980301, -0.246860077931526, -0.351397886837889, -0.23211222129697, 0.131028262406404, 0.0389154162496732, -0.0231224174208542, -0.0749976519277089, -0.0967787628858519, -0.264151575041587, 0.0642791514045653, -0.181470854913642, -0.035350988878931, -0.22314355131421, -0.0585533087515744, -0.170625517030763, -0.117783035656384, -0.223143551314211, -0.271496748030764, -0.231840227823137, -0.0830516576038791, -0.0348753293140129, -0.0905435929019962, -0.039441732051297, -0.091434205959632, -0.0468571586878825, -0.0959639882617438, -0.0117859422190874, -0.0923733201310153, -0.0130171058024899, 0.00475060275859818, -0.0717936920187423, -0.130053128248198, -0.0133312519554591, 0.124674861694305, -0.163954679923878, -0.0845573880280623, -0.132324877296182, 0.0560894666510432, -0.208466836898277, -0.0640702467202106, -0.0487901641694322, -0.0536210914390978, -0.0444517625708345, 0.0861776962410516, -0.00652885888246324, 0, -0.146379535102372, 0.310154928303839, -0.0176995770994006, 0.0115608224010764, 0.100470530363635, 0.162518929497775, 0.264692554227082, -0.113077144618971, -0.120144311842064, 0.059033531584193, 0.00392927813988919, -0.124850202879476, 0.00626961301359508, 0.0298084662978262, -0.118055998671591, -0.131576357788719, -0.00881062968215485, 0.0217399866364056, 0.0184337016888385, 0.079111001652536, -0.038374012881615, -0.305864391534971, -0.167054084663166, 0.126675144754063, -0.0807613566444587, 0.0213911899813182, 0.030459207484709, 0.0540672212702757, 0.145310091817136, -0.025075940768212, -0.274987152970533, -0.232931557680373, -0.103570370042048, -0.141746288743414, 0.00681006216133007, 0.0512932943875506, -0.606135803570316, -0.0969922659873097, -0.0986425361174703, -0.0369435151916848, -0.094288947978324, 0.141650517063027, -1.02961941718116, -0.0822893947565149, -0.0191944472561474, 0.328504066972036, -0.0706175672139531, -1.46633706879343, -0.334639055539552, -0.154681606306414, 0.133531392624523, -0.0668629452291265, 0.0144930073025664, 0.120627987788614, 0.176930708159078, -0.382992252256106, 0.0499410291375364, -0.113944259349218, 0.0173917427118688, -0.168059109873391, -0.0281708769666964, -0.367724780125317, -0.470003629245735, -0.188221278921143, 0, -0.0435712482936399, 0.0165293019512109, 0.255933374137201, 0.0327898228229908, -0.052471051737454, -0.0284494118748899, -0.0729489996461679, 0.112618878108149, -0.0134602190556974, 0.0395764757017059, -0.0756595628451411, -0.492476485097794, 0.0179109265665307, -0.0551320706947163, -0.101096116871369, 0.0107473405956728, 0.092721740050222, -0.034635496662756, -0.00818102901645723, 0.0590490288528001, 0.202236866494896, -0.402992918962775, -0.195516044873487, -0.0569081322718796, -0.0377403279828474, -0.0691272710570754, -0.112417707967825, -0.188742124596877, 0.0428199971829288, 0.0607562094210721, 0.101747438973162, 0.000974184197843009, -0.0126184639592113, -0.0388979479328331, 0.0638081142684905, -0.0234736356127776, 0.0601685214664531, -0.07417986546408, -0.0202436062766465, 0.0964137296680203, -0.0302330106601936, 0.0640072185482552], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "aipw": { + "estimate": -0.0145296683111164, + "ess": 293.350247392128, + "weights": [0.894002251049714, 1.08912887056764, 0.912139448798106, 1.33663687068034, 0.777353982575372, 1.07895492287744, 0.844159798828147, 1.3183278848748, 1.21827258329935, 0.872521288429006, 0.86064891071784, 1.22505040897046, 0.847098610659438, 1.22298314584019, 1.23875720803122, 1.08079433857419, 1.67945989357214, 0.967653605894451, 0.839205021306279, 0.874745094998973, 1.14396143122207, 0.986274905763588, 0.793655426945849, 0.813577909783472, 0.904573126031373, 1.04663392636343, 1.14519210123462, 1.20012641825204, 0.967553597396161, 1.05438584858977, 0.935224694727841, 0.900488358508805, 0.781018532914758, 0.864021879677238, 0.853478380599116, 1.14103041278715, 0.954200123207928, 1.2046977512144, 0.849705627797744, 0.960157332889347, 0.702565586174883, 0.784481381193752, 1.04823001919817, 0.838343574206719, 0.796638337684592, 0.858149958258436, 0.901812997158555, 0.946383153027563, 0.887026666261882, 1.15225205545184, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.17309689736406, 1.06716813845226, 1.23458096954703, 0.99157960228583, 1.03337549477273, 1.08898847481201, 1.06429258825978, 1.27901832490392, 1.17852895390884, 1.25064709755962, 1.27241523475185, 1.05962508654199, 1.26264882346518, 1.08823686024391, 1.0850169829417, 1.15946797717608, 1.08965356969476, 0.762126536933417, 0.95881005552149, 1.04424929648179, 1.07803366329225, 1.32925013731868, 0.995764287470606, 0.998092093051924, 0.983239427591536, 1.29542652220053, 0.855380262678252, 0.849934582420623, 0.858457868237986, 0.935224694727841, 0.9476684674978, 1.05403326965604, 1.3875512537957, 0.962868424186321, 0.852207710912963, 0.863735990959885, 0.901015308497714, 1.55514398580636, 0.935249998401912, 0.758552494022138, 0.95029188025848, 0.785671952662053, 1.06406808753133, 0.850203555377142, 0.964834093590168, 0.822533389701864, 0.823211329274651, 1.24206426011762, 0.701639876047414, 0.687891345473986, 0.874429096523289, 0.820162869388286, 1.16531204434823, 0.919264300892949, 0.769520096761839, 0.774050093411802, 0.795321421756718, 0.750133494599614, 0.734916916052496, 0.844854160183868, 0.970426525099808, 1.58110223940849, 1.17815474202094, 1.05881662670873, 1.6248775532266, 1.32664934577575, 0.834721535320327, 1.16105846772014, 0.847508548356641, 0.966263255565178, 1.11338493606689, 1.07112275936809, 0.799349711866075, 0.821662255653284, 0.773730798373221, 0.837752419995297, 0.864009459343324, 1.61261501076916, 0.78376277257012, 0.705876292742996, 0.647428686090855, 1.11696902174642, 0.831471116458873, 0.712788525836175, 0.823368599043228, 0.831036004982153, 1.45197650533514, 0.757870151528899, 0.801587265560697, 0.903966634067484, 0.688609587838628, 0.833378425203411, 0.88781143556202, 1.2859343470801, 0.943625225641593, 0.774792618745101, 0.934954580596145, 0.985394537886997, 1.36577400696854, 1.12387359379345, 0.751614058902251, 1.26975693234241, 0.668612420905503, 0.765171028921774, 0.654385132001179, 1.1577477383766, 0.679012060093675, 0.77026241566466, 0.783702742793706, 0.710510112907514, 0.768135289274691, 0.816992668722927, 0.871261846989218, 0.94306726889844, 0.772811895241902, 1.04677022713451, 1.01318017562438, 1.21472287797626, 0.690891037043235, 0.988938195082353, 0.891162830885818, 0.847344689605216, 0.84339276238123, 1.03367583677142, 1.16749363935029, 1.16386294782544, 1.07692221020635, 1.66701335998637, 1.13058730375163, 0.932454891509071, 1.35361774608789, 0.916127968293349, 1.28295957022046, 0.901853294887475, 1.50580256277375, 1.0344196777086, 1.07516120251468, 0.978750662101694, 1.39355713356033, 1.11380233235004, 1.16782487713884, 1.04150887454439, 1.15806305850471, 0.938312253912984, 1.01163956811404, 1.04147894758405, 0.627576820467188, 0.987358672362671, 0.796101729352255, 0.799367879632361, 0.914469343531904, 0.586506346702584, 0.85746807099136, 0.644331572783776, 0.698650747799085, 0.840418052265205, 0.799386045379082, 0.613378159731018, 0.826097329094868, 0.914995695834427, 0.87099933523313, 1.25415777108945, 1.06234975602779, 1.03180994165805, 1.00536574424389, 1.50408349546532, 1.11707181872119, 0.962575541260206, 0.99500527053113, 1.06267552030451, 1.306809166267, 1.11113734646778, 1.80465227361454, 0.925313475141831, 0.926562233085535, 0.971844061197384, 0.925117889238856, 1.02541821473023, 0.816026483828058, 0.823227062864221, 1.02452843411696, 1.52601767939589, 1.0911845575514, 0.853952506965403, 2.10561116586107, 0.93309568004779, 1.59396704068207, 0.729558976902732, 0.602979072986013, 1.17818480963529, 1.06791977822604, 1.02326006063466, 1.45370921722291, 1.05994579825869, 0.69921162182759, 1.09837880259324, 1.16949406859886, 0.687473835164664, 0.782073370173473, 0.740566165391499, 1.09370529557565, 0.945960791989393, 0.900711482725538, 0.82541539067686, 0.994537255170615, 0.840418052265205, 0.697345210807313, 0.687093199004881, 1.07648964211119, 0.860268711765294, 1.16079118422479, 0.913336436309487, 0.696491928549023, 0.631556234278697, 0.700083598765149, 1.08390315180698, 1.01444872130449, 1.02568172844303, 0.927750198039274, 0.868776406041439, 0.8615958327451, 1.07506295514825, 1.44060768659023, 0.813043719924969, 0.893907609746538, 1.22090330227284, 1.19394906972845, 1.15627267688281, 0.748337722818574, 1.14001845186309, 1.01514511301218, 0.931880629247007, 1.46131939576755, 1.03541768771894, 0.97833141796237, 1.82862252736225, 1.12259014409163, 0.927521982019721, 0.902446667017561, 1.13204983708822, 0.985603565025875, 0.880067648926837, 1.20495265096482, 1.02908828203376, 1.0828414651844, 0.776916027928803, 0.922628819453102, 0.939123978418286, 1.32540040382158, 1.06511229655857, 1.15523404587142, 1.37295557236873, 1.43585985120983, 0.861167191951046, 1.23116101723212, 0.973674667516388, 1.15039551027658, 0.866537571207786, 0.971809095301766], + "dy": [0.0402064204780404, -0.00617921636595753, -0.00769234562315635, -0.0109482153338982, 0.271933715483642, 0.0670107102829602, -0.159064694629687, 0.0776028399652979, -0.0348005291494164, -0.361790044605503, -0.023530497410194, 0.0407981988772361, -0.039441732051297, 0.0183057960884394, 0.0578857923998042, -0.0689928714869517, -0.0489120485013519, -0.141078598259906, -0.231801614057324, -0.0979804083602032, -0.107245530353598, 0.0255333020051651, -0.100083458556983, -0.42744401482694, -0.00557104504945549, -0.0268472500361883, 0.0447260357486368, 0.0994408366375259, 0.115069329784787, -0.0392207131532816, 0.0330062964681703, -0.188348264660815, 0.37469344944141, 0.185403223331362, 0.105360515657826, -0.123805501162851, -0.127561225638019, 0.00391083799199876, -0.182321556793955, 0.0138891121606672, 0.352821374622742, 0.0202027073175195, 0.00183992692200707, -0.0615578929994332, -0.145851877012563, -0.101020402315209, -0.0900261516577192, 0.00645163528148807, 0.0823995317670665, -0.0217808084613926, -0.118481603620523, -0.00881062968215485, -0.0836500292869244, -0.332133835022615, 0.0377403279828474, -0.0805032188490706, -0.127833371509885, -0.103666774395267, -0.0391226547103702, -0.0522125710903065, -0.114421687953678, 0.00524935588614373, 0.024916351264535, -0.287682072451781, -0.052592450119171, 0.0108109161042158, -0.0619667237491992, -0.049576446606288, 0.0494724535390656, -0.0782007473074744, -0.0358461317731358, -0.0870113769896301, -0.0922165496035534, 0.264301623807717, -0.0495596909484606, -0.0583355770129632, -0.0856554445784941, -0.11963720201451, -0.0900336686743657, 0.0372967877532266, -0.0117011085851217, -0.081526607634208, -0.0620637099045016, -0.0557419830082351, -0.0589674760881156, -0.0501712931478435, -0.033953878930344, -0.0277795641070755, 0.0201011793210872, 0.0344321964214842, -0.0153377239837091, -0.0636388359666737, -0.0337381396318506, -0.136464423894526, -0.210379509589112, -0.0663859260598123, -0.24231346742193, -0.0682679711290026, -0.0795120629277335, -0.0880333494853227, -0.191268342783761, -0.0861116918187319, -0.0228979640932554, -0.0592556962090933, -0.0804723532225733, 0.0714589639821446, -0.0876299974856956, -0.0657078415693917, -0.249811798396371, -0.111791405988116, -0.168898536461814, -0.316669609325034, -0.0500409460710847, 0.0302398851897179, -0.234400705838844, 0.094778406327289, -0.0897699994287091, -0.155955083059526, 0.143100843640673, -0.0730251350148903, -0.0628009012390303, -0.0145987994211527, -0.00685403642779203, -0.189888919858863, -0.0266682470821618, 0.037740327982847, -0.0111733005981254, -0.117783035656383, -0.0384662808277962, 0, 0.0571584138399484, -0.0694119283885524, -0.0735133828982697, -0.104572803919577, -0.174540558863677, -0.159272329947313, -0.241162056816889, -0.151592882072704, 0.336472236621213, -0.364897923119164, 0.03964483134724, -0.229033345370273, -0.182321556793954, -0.251314428280907, 0.127833371509885, 0.0752830604205483, -0.0685626714372969, -0.0333465799413801, -0.197610249309045, -0.102654154060084, 0.0645385211375711, -0.071686734959413, 0.0268112574506567, -0.11247798342669, -0.116682320080198, 0.143100843640673, -0.0513428079171341, -0.0591888713903304, 0.0285733724440558, -0.2383103438577, -0.0444517625708336, 0.0909717782057271, -0.220542769614153, -0.03650636928728, -0.00735297430525872, -0.136859182717197, -0.0115943327809198, -0.306619273471531, -0.0180556878262053, -0.0133740595252565, -0.847297860387203, -0.0694513863237711, -0.0165293019512109, -0.391671785975829, 0, -0.0750083720980301, -0.246860077931526, -0.351397886837889, -0.23211222129697, 0.131028262406404, 0.0389154162496732, -0.0231224174208542, -0.0749976519277089, -0.0967787628858519, -0.264151575041587, 0.0642791514045653, -0.181470854913642, -0.035350988878931, -0.22314355131421, -0.0585533087515744, -0.170625517030763, -0.117783035656384, -0.223143551314211, -0.271496748030764, -0.231840227823137, -0.0830516576038791, -0.0348753293140129, -0.0905435929019962, -0.039441732051297, -0.091434205959632, -0.0468571586878825, -0.0959639882617438, -0.0117859422190874, -0.0923733201310153, -0.0130171058024899, 0.00475060275859818, -0.0717936920187423, -0.130053128248198, -0.0133312519554591, 0.124674861694305, -0.163954679923878, -0.0845573880280623, -0.132324877296182, 0.0560894666510432, -0.208466836898277, -0.0640702467202106, -0.0487901641694322, -0.0536210914390978, -0.0444517625708345, 0.0861776962410516, -0.00652885888246324, 0, -0.146379535102372, 0.310154928303839, -0.0176995770994006, 0.0115608224010764, 0.100470530363635, 0.162518929497775, 0.264692554227082, -0.113077144618971, -0.120144311842064, 0.059033531584193, 0.00392927813988919, -0.124850202879476, 0.00626961301359508, 0.0298084662978262, -0.118055998671591, -0.131576357788719, -0.00881062968215485, 0.0217399866364056, 0.0184337016888385, 0.079111001652536, -0.038374012881615, -0.305864391534971, -0.167054084663166, 0.126675144754063, -0.0807613566444587, 0.0213911899813182, 0.030459207484709, 0.0540672212702757, 0.145310091817136, -0.025075940768212, -0.274987152970533, -0.232931557680373, -0.103570370042048, -0.141746288743414, 0.00681006216133007, 0.0512932943875506, -0.606135803570316, -0.0969922659873097, -0.0986425361174703, -0.0369435151916848, -0.094288947978324, 0.141650517063027, -1.02961941718116, -0.0822893947565149, -0.0191944472561474, 0.328504066972036, -0.0706175672139531, -1.46633706879343, -0.334639055539552, -0.154681606306414, 0.133531392624523, -0.0668629452291265, 0.0144930073025664, 0.120627987788614, 0.176930708159078, -0.382992252256106, 0.0499410291375364, -0.113944259349218, 0.0173917427118688, -0.168059109873391, -0.0281708769666964, -0.367724780125317, -0.470003629245735, -0.188221278921143, 0, -0.0435712482936399, 0.0165293019512109, 0.255933374137201, 0.0327898228229908, -0.052471051737454, -0.0284494118748899, -0.0729489996461679, 0.112618878108149, -0.0134602190556974, 0.0395764757017059, -0.0756595628451411, -0.492476485097794, 0.0179109265665307, -0.0551320706947163, -0.101096116871369, 0.0107473405956728, 0.092721740050222, -0.034635496662756, -0.00818102901645723, 0.0590490288528001, 0.202236866494896, -0.402992918962775, -0.195516044873487, -0.0569081322718796, -0.0377403279828474, -0.0691272710570754, -0.112417707967825, -0.188742124596877, 0.0428199971829288, 0.0607562094210721, 0.101747438973162, 0.000974184197843009, -0.0126184639592113, -0.0388979479328331, 0.0638081142684905, -0.0234736356127776, 0.0601685214664531, -0.07417986546408, -0.0202436062766465, 0.0964137296680203, -0.0302330106601936, 0.0640072185482552], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + }, + "derived_columns": { + "lpop_t": "lpop * (year - 2002) / 5" + } + }, + "sim_staggered": { + "data_file": "twfeweights_sim_panel.csv", + "columns": { + "outcome": "outcome", + "unit": "unit", + "time": "period", + "first_treat": "first_treat", + "invariant_cov": "x1", + "varying_cov": "xtv" + }, + "two_period_group": 3, + "attgt_weights": { + "twfe": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], + "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "implied_att": 1.38176729464315 + }, + "overall": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "weight": [0, 0, 0.166666666666667, 0.166666666666667, 0.166666666666667, 0, 0, 0, 0.25, 0.25], + "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "implied_att": 1.73623602393453 + }, + "simple": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "weight": [0, 0, 0.2, 0.2, 0.2, 0, 0, 0, 0.2, 0.2], + "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "implied_att": 1.76984085043211 + } + }, + "decompose": { + "fwl_nocov": { + "cells": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "att": [0, -0.0839722781780438, 0.848539485953839, 1.76991196956896, 2.84209802207204, 0, 0.0713186971413766, -0.203874498662049, 0.823397571607694, 1.90590552823814], + "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], + "ess": [100, 100, 99.4718023394506, 100, 100, 100, 100, 99.4718023394506, 100, 100], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.38176729464315, + "decomposition": 1.38176729464315, + "remainder": 0, + "pretrend_bias": 0.0929885879714431, + "post_only": 1.28877870667171, + "effective_sample_size": 499.00962938647 + }, + "fwl_cov": { + "cells": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "att": [0, -0.0821486884936649, 34.2972067369434, 1.76608384450794, 2.83842487370002, 0, 0.0730127750279008, 33.2436059932573, 0.823520882264303, 1.90613909549667], + "weight": [-0.250278200442825, -0.250043995598454, 0.375228054332711, 0.0628082321912258, 0.0622859095173414, -0.0622827984811969, -0.06218597232544, -0.375209033152085, 0.249640734110113, 0.250037069848608], + "ess": [99.9941013522363, 99.9944461946957, 0.00609297505217851, 99.9936664953336, 99.9941598816125, 99.9941013522363, 99.9944461946957, 0.00609297505217851, 99.9936664953336, 99.9941598816125], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.38188157642338, + "decomposition": 1.38188157642338, + "remainder": 0, + "pretrend_bias": -12.4573008473221, + "post_only": 13.8391824237455, + "effective_sample_size": 312.37838960871 + }, + "fwl_gmin1": { + "cells": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "att": [0.0839722781780438, 0, 0.944867949565343, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], + "ess": [100, 100, 99.4718023394506, 100, 100, 100, 100, 99.4718023394506, 100, 100], + "remainder": [-0.940505910590496, -0.940505910590496, -0.947616432141581, -0.940505910590496, -0.940505910590496, -1.62957956051008, -1.62957956051008, -1.62464805376604, -1.62957956051008, -1.62957956051008] + }, + "estimate": 1.38176729464315, + "decomposition": 1.38628305525382, + "remainder": -0.00451576061067098, + "pretrend_bias": -0.0508955308063108, + "post_only": 1.43717858606014, + "effective_sample_size": 499.00962938647 + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 4, 4], + "time": [3, 4, 5, 4, 5], + "post": [1, 1, 1, 1, 1], + "att": [0.950890455977506, 1.90549496406924, 3.01528915156681, 1.08514205063569, 2.23370942485274], + "weight": [0.166666666666667, 0.166666666666667, 0.166666666666667, 0.25, 0.25], + "ess": [99.1390613452281, 99.1390613452281, 99.1390613452281, 97.3682491639651, 97.3682491639651] + }, + "estimate": 1.80832529747437, + "decomposition": 1.80832529747437, + "remainder": 0, + "pretrend_bias": 0, + "post_only": 1.80832529747437, + "effective_sample_size": 491.268276272983 + } + }, + "balance": { + "fwl": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], + "time": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5], + "post": [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946], + "unweighted_covs_comparison": [0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121], + "weighted_covs_treated": [0.0322253630566276, 0.61354301308016, 0.0290807000435411, 0.611652284352772, 0.0314456717092465, 0.613125653540162, 0.0304194230673653, 0.611090717337329, 0.0261194212453405, 0.610939920369823, -0.0356287952936007, 0.583175368672645, -0.0455885285626065, 0.579867791877802, -0.0422942050601538, 0.580882621650816, -0.041129084294574, 0.581367403796091, -0.0426164850331386, 0.580717322776911], + "weighted_covs_comparison": [0.128912846595125, 0.667500533623696, 0.127496503049284, 0.666346647112834, -22.2803241752299, -13.6848306944835, 0.129245458534132, 0.667533121252504, 0.128528952295834, 0.667188479625278, 0.128912846595125, 0.667500533623696, 0.127496503049284, 0.666346647112834, -22.2803241752299, -13.6848306944835, 0.129245458534132, 0.667533121252504, 0.128528952295834, 0.667188479625278], + "weighted_diff": [-0.0966874835384971, -0.0539575205435364, -0.0984158030057429, -0.0546943627600621, 22.3117698469391, 14.2979563480237, -0.0988260354667672, -0.0564424039151755, -0.102409531050493, -0.0562485592554547, -0.164541641888725, -0.0843251649510506, -0.17308503161189, -0.0864788552350323, 22.2380299701697, 14.2657133161343, -0.170374542828706, -0.0861657174564133, -0.171145437328972, -0.0864711568483669], + "sd": [1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979], + "unweighted_log_ratio_sd_diff": [-0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373], + "weighted_log_ratio_sd_diff": [-0.078923039541646, -0.0981028673500817, -0.0787851460177666, -0.0992789443199351, -4.79800027862989, -5.26202924360934, -0.0788460372935362, -0.0965119215695593, -0.0755098089684916, -0.104025654316807, 0.027732869455797, 0.0160140941570539, 0.0107487270326319, -0.00359213115639778, -4.70526255947979, -5.16554315809729, 0.0155949542480163, 3.59691177780341e-05, 0.0154199797107926, 0.000125226527590172], + "unweighted_frac_treated_extreme": [0.05, 0.02, 0.05, 0.02, 0.05, 0.02, 0.05, 0.02, 0.05, 0.02, 0.12, 0.08, 0.12, 0.08, 0.12, 0.08, 0.12, 0.08, 0.12, 0.08], + "weighted_frac_treated_extreme": [0.05, 0.02, 0.04, 0.02, 0, 0, 0.05, 0.03, 0.04, 0.02, 0.11, 0.08, 0.1, 0.09, 0, 0, 0.11, 0.08, 0.11, 0.09] + }, + "summary": { + "unweighted_treat": [-0.00530837076689503, 0.597037457313671], + "unweighted_untreat": [0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.13350937598507, -0.0698986457475872], + "weighted_treat": [-0.00558643553339065, 0.596830265573273], + "weighted_untreat": [-8.27967753493962, -4.71798406790115], + "weighted_diff": [8.27409109940623, 5.31481433347443], + "sd": [1.02583786880053, 0.576529078265404], + "unweighted_log_ratio_sd_diff": [-0.0319022468238235, -0.0490791536243857], + "weighted_log_ratio_sd_diff": [-1.80225098421397, -1.98696177996179], + "unweighted_frac_treated_extreme": [0.0849774462771105, 0.0499806682375233], + "weighted_frac_treated_extreme": [0.0605964064257143, 0.0456045601712674], + "covariate": ["mean_x1", "mean_xtv"] + } + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 4, 4, 4, 4], + "time": [3, 3, 4, 4, 5, 5, 4, 4, 5, 5], + "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946], + "unweighted_covs_comparison": [0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121], + "weighted_covs_treated": [0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946], + "weighted_covs_comparison": [0.031409314043045, 0.615961633357426, 0.031409314043045, 0.615961633357426, 0.031409314043045, 0.615961633357426, -0.0420734072606479, 0.57732449107713, -0.0420734072606479, 0.57732449107713], + "weighted_diff": [0, -0.0028549349573127, 0, -0.0028549349573127, 0, -0.0028549349573127, 2.77555756156289e-17, 0.00362300201281618, 2.77555756156289e-17, 0.00362300201281618], + "sd": [1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979] + }, + "summary": { + "unweighted_treat": [-0.00533204660880145, 0.59702709574503], + "unweighted_untreat": [0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.133533051826977, -0.0699090073162288], + "weighted_treat": [-0.00533204660880145, 0.59702709574503], + "weighted_untreat": [-0.00533204660880146, 0.596643062217278], + "weighted_diff": [1.38777878078145e-17, 0.000384033527751737], + "sd": [1.0258527982419, 0.576537669697901], + "unweighted_log_ratio_sd_diff": [0, 0], + "weighted_log_ratio_sd_diff": [0, 0], + "unweighted_frac_treated_extreme": [0, 0], + "weighted_frac_treated_extreme": [0, 0], + "covariate": ["mean_x1", "mean_xtv"] + } + } + }, + "two_period": { + "reg": { + "estimate": 0.933749536861636, + "ess": 99.9926610233913, + "weights": [1.00722890988084, 0.992990992262401, 0.985696926864851, 1.00428546538914, 1.00255445781008, 0.99320823416596, 0.999615523785608, 1.00343083006454, 1.00137356889656, 0.986745204378563, 0.995094957332289, 0.986870840632856, 1.00528820819911, 0.998835332251396, 0.99700861697196, 0.997936725463451, 0.998846629895534, 0.99656967557782, 1.00045816004973, 0.990771166474457, 0.997338926459884, 0.998846092341373, 0.995772857502799, 1.01166869648943, 0.993003662361777, 1.00600474847494, 1.00405785167947, 1.00353809034622, 0.995126225338609, 1.00043085640342, 0.998206308876953, 0.996320942216609, 1.00237756422585, 0.985442690419153, 0.989720250034912, 0.995867683639298, 0.99111809890224, 1.00019084884885, 0.990912512219895, 1.00911662153634, 1.0192714498591, 1.01698591482046, 1.00242798608037, 0.991647428774127, 1.00134810289326, 1.00443878939999, 0.996778530830711, 1.00669024403109, 1.00001152211208, 1.01998818237323, 0.99264596683436, 0.9952948133176, 1.01360504654043, 1.009933572531, 1.01298877137443, 1.00106309713833, 0.99368962673189, 0.994645710006753, 1.00603342986714, 0.995879858693205, 1.01601455811524, 1.02010680280297, 0.989784340649968, 1.00825224525215, 1.00677285251964, 1.00871986746097, 1.00390290828464, 0.993629249244157, 1.00167286392168, 1.01886807947052, 1.01099154001983, 1.00201217904192, 0.99836526676243, 1.01002660972161, 0.996287759273861, 1.00250415790818, 0.983606011950821, 1.00384205736887, 0.991605834088229, 1.00537797220874, 1.01272476427711, 0.99235753997329, 0.987933977679291, 1.000650667459, 0.984180988085733, 0.996029922902563, 0.997939009434197, 0.990121709237807, 0.991953055467323, 1.00235884266265, 1.00125277134316, 0.992473987574461, 0.992917670060727, 0.998942314836784, 1.00485857501194, 1.00287939258367, 1.00418948560417, 0.985443467362189, 0.985970178418021, 1.01723852508477, 0.997466706557304, 0.992423956214259, 1.00991905980572, 0.992291387074752, 0.994820831943633, 0.989354780777259, 1.00319011306147, 1.00827468557464, 1.00431722998307, 0.996353080619044, 1.00439019192514, 0.999040966656546, 1.00231586960461, 0.978372337508447, 0.995019298342224, 0.992937322106751, 0.993590737901513, 1.00475685538413, 1.00449017738004, 1.01172939172658, 0.996227103707377, 0.998538789522835, 0.995737537791419, 0.99411674103665, 1.00138684721952, 1.00931125598526, 0.99549586869746, 1.00710500069773, 1.01347249983801, 0.996627686726185, 1.00244252373843, 1.00388526279484, 0.996225099533706, 1.01622034867275, 0.992126287936598, 0.998427079293577, 0.997025997515504, 0.994683358673275, 0.994260944805646, 1.01294005227182, 1.01510040197259, 1.00867105634545, 0.999364111810118, 0.988370930721448, 1.00754081330585, 1.01108908526376, 1.00082929965702, 0.999652563328744, 0.99115023644786, 0.993797952889325, 1.00062317593941, 0.996370067258762, 1.00862548801356, 0.992934433889504, 1.00625697060545, 1.01325255367648, 1.00669485794986, 0.994504199351021, 0.998979328323581, 0.989317777254636, 1.0079062968392, 1.00142985808414, 1.00927426598237, 0.988654526955168, 0.989280380585935, 1.00609361546911, 0.997618744628682, 1.00320239737719, 1.00752528142108, 1.00990809476446, 1.00300322465167, 0.991801531478728, 0.982098203337734, 0.993156779738883, 0.99999216518783, 1.01701536391709, 1.00655564623742, 0.987239057302495, 0.997613724292227, 0.996893085121885, 0.998388510818664, 1.00686703463208, 0.989947899004373, 0.99691959100035, 0.992555666528202, 1.02749634237056, 0.990975757502908, 0.998821219235584, 1.00181547622219, 0.982006002279984, 1.00349537276095, 1.01340976418696, 1.00161767714608, 0.998384149967381, 1.01077308854491, 0.993352591748675, 1.00138206112585, 1.00203727492767, 0.990541643636813, 0.988514062376352], + "dy": [-0.402033610301275, 2.18544275172545, 0.407877616958751, 1.8390018429467, 0.342089980783171, 5.17283289194947, 1.77541144069014, 0.177916426818689, 0.988193170415602, -1.73303800372523, -0.455268584403767, 0.627245807499127, 0.95163643365076, 0.729499027127282, -0.716548210368217, 0.989179554960285, 0.615290027522918, -0.928173862369419, -3.12066260941806, 0.254525318474001, 2.26949886412902, 0.829906104908926, 0.839722075215133, 0.402827764479532, 2.615721647119, 2.56884939538828, 0.40984832983609, 0.916597426324454, 3.56815993643643, 4.11857018395454, -0.583954358728381, 0.629543252397921, -2.59864890963595, -1.50575253730084, 0.10854369311674, 0.339066188455571, 0.840969127612396, 1.49640162269824, 2.15698629580388, 0.710710905149634, -0.780658854871792, 0.246639873748455, -2.99165681128261, 2.21630573154794, 1.36462660688514, 3.91230934269807, 0.750821589315842, 1.3797955671716, 1.29873425008199, -0.0142518982330498, 0.942704674378142, -0.00295039223517701, 1.26619389096853, 2.07005386994108, 1.30556335439981, 0.957684657053287, 2.07603562299318, 3.09870706931771, -1.07466886872445, 1.72619332970284, 0.185899746718614, 0.879844294714601, -0.110555708727767, 0.867499070153156, 1.97688576587969, 2.11054101983651, -0.213279861129251, 0.590859861669465, 0.286669554875457, -1.33313130715994, 0.166213410436903, -0.981498645950401, -1.08091042671938, -2.43836942530926, -0.476981977649933, 0.165773714712555, 1.34280270525293, 2.29324397361844, -2.48270705209481, -1.77399367825062, 0.528226838838877, 1.79264254406572, 1.15140499752099, 2.42338563275737, 0.813914692972622, 3.9331810215408, 0.638302157203735, 0.954304368656401, 0.44612105486203, 1.09255026112807, 1.66177432848481, 1.2244211095895, -0.307344980789505, 0.362311341134403, -0.707302108346803, -0.144104822654965, 2.80967986565376, 0.886724301741435, -0.437691146279499, 0.227891475847955, 4.03744118286034, 2.45748912778531, 0.144254311972021, 2.61650674781118, 4.31326476995183, 1.48977229748571, 2.34613355467357, 1.70052061441654, 3.22056170230932, 1.78075346385693, -0.711428102809615, 1.34730204516023, 1.03029253615151, 0.419549571357165, 1.58776493288537, 0.995197584563667, 0.642274699864077, 2.07636006524544, 2.38487012186388, 1.23008817197199, 1.38552304464906, -0.660629532295937, 2.90020370216734, 2.14880989578056, 0.0397416585750139, 3.38848652983915, 1.5391831069985, 3.45379719174161, 0.398418170016988, 2.92591755647753, 0.269001120074444, 3.49568149729268, 1.10053690527001, 0.534365065625511, 2.45681592071403, 1.59579092024735, 0.819893378575548, 2.81625032274877, -0.00324530506939347, 0.605258529293289, 3.85479849191654, 0.583541472525013, 2.80499048092448, 3.43673863609605, 2.53588037120317, 1.05364560288094, 1.74797954925627, 1.12861746747233, 2.84125481556006, 3.12911840005426, 0.302617858060385, 2.05438327553933, 0.414404978188778, 1.89216322378059, 1.13139413791383, 2.00133216270611, 0.643782885192826, 1.88591067421528, 0.252206326893992, 0.687547110267291, 1.95059516009538, 0.154987971181278, -1.84401576567585, 1.42142999538519, 1.45740057865859, 0.642256402067309, 2.85687382154418, 2.56544873039477, 1.55960416827183, 5.34308795625776, -0.487166716367998, 1.11969723983059, 1.31495808447936, 1.95142653439584, 3.50775782707169, 1.04468240755541, 2.15013315602819, 2.36947457467637, 1.91237343735871, 0.613733460400012, 4.06876671513254, 3.00698872518514, 2.43255060792113, 0.590220612096163, 0.790617682152215, 0.3088784929935, -0.390557110671398, 0.783542381656206, 2.54825966725677, 0.531623874159247, 0.754254870486599, 2.26979312680552, 0.882020570135368, 3.67733208815144, 1.18041126492999, 1.67479917330522, 1.13098167264533, 3.93443859164881, 0.545035532418525, -0.835817441782716], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + }, + "aipw": { + "estimate": 0.932908107897949, + "ess": 99.999180206262, + "weights": [0.999672729706382, 0.998021972909027, 1.00246030815386, 0.998513055504986, 1.00585412572353, 0.997051177858498, 0.994461321822849, 0.996136791680998, 0.99802322685533, 1.00130465103858, 1.0036020329346, 1.00209547702384, 1.00124115667369, 0.996393987795192, 0.998049799389463, 1.00262259770517, 1.00030309128241, 1.00064739548438, 1.00291569113999, 1.00435106542457, 1.0011434294141, 1.00188564912852, 1.00179620123262, 0.995509793569397, 1.00003833654497, 1.00001014848312, 1.00212284054395, 1.00181323882764, 1.00114490591003, 0.999410260939198, 1.00368717706179, 1.00000109453662, 0.998485491222804, 1.00698466777593, 0.996625230168786, 1.00352109899527, 0.995024150944156, 0.998998191251484, 1.00554621969829, 0.997495901024524, 0.999565102450949, 0.99382696946915, 0.993461661221226, 0.999828271614111, 1.00045584463728, 0.996827971795766, 1.00081229366678, 0.998565528198841, 1.00389512128434, 0.995518787588971, 1.00440712427909, 0.999875209026417, 1.0032200133693, 0.999900835948388, 0.997523979887541, 1.00025940775218, 1.00175413984952, 1.0043515707889, 0.999784272657803, 0.998278347015222, 1.00199659550982, 0.998870210394531, 1.00435304482845, 1.00013634872456, 0.996946252359397, 0.996194426606418, 0.995532736428361, 0.998699972048666, 0.994346574614814, 0.997342357218583, 1.00141195618414, 0.998554195649132, 0.995582390998808, 1.00104190020894, 0.996355646424078, 0.999910936252083, 1.00118919151784, 0.994777646401869, 1.00165288081142, 1.00136294435556, 1.00030533982388, 0.999923527203744, 0.999932083612262, 1.00055286791905, 1.00310726818895, 1.0008836585536, 1.00106078377252, 1.00232921480015, 1.00148651721726, 0.999986839444585, 1.00007595480953, 1.0026103987571, 0.999783262359765, 1.00559848648777, 0.998470814142084, 0.996007891476203, 1.00066584402688, 1.00217882663762, 1.00022113959277, 1.00148293775449, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "dy": [-0.402033610301275, 2.18544275172545, 0.407877616958751, 1.8390018429467, 0.342089980783171, 5.17283289194947, 1.77541144069014, 0.177916426818689, 0.988193170415602, -1.73303800372523, -0.455268584403767, 0.627245807499127, 0.95163643365076, 0.729499027127282, -0.716548210368217, 0.989179554960285, 0.615290027522918, -0.928173862369419, -3.12066260941806, 0.254525318474001, 2.26949886412902, 0.829906104908926, 0.839722075215133, 0.402827764479532, 2.615721647119, 2.56884939538828, 0.40984832983609, 0.916597426324454, 3.56815993643643, 4.11857018395454, -0.583954358728381, 0.629543252397921, -2.59864890963595, -1.50575253730084, 0.10854369311674, 0.339066188455571, 0.840969127612396, 1.49640162269824, 2.15698629580388, 0.710710905149634, -0.780658854871792, 0.246639873748455, -2.99165681128261, 2.21630573154794, 1.36462660688514, 3.91230934269807, 0.750821589315842, 1.3797955671716, 1.29873425008199, -0.0142518982330498, 0.942704674378142, -0.00295039223517701, 1.26619389096853, 2.07005386994108, 1.30556335439981, 0.957684657053287, 2.07603562299318, 3.09870706931771, -1.07466886872445, 1.72619332970284, 0.185899746718614, 0.879844294714601, -0.110555708727767, 0.867499070153156, 1.97688576587969, 2.11054101983651, -0.213279861129251, 0.590859861669465, 0.286669554875457, -1.33313130715994, 0.166213410436903, -0.981498645950401, -1.08091042671938, -2.43836942530926, -0.476981977649933, 0.165773714712555, 1.34280270525293, 2.29324397361844, -2.48270705209481, -1.77399367825062, 0.528226838838877, 1.79264254406572, 1.15140499752099, 2.42338563275737, 0.813914692972622, 3.9331810215408, 0.638302157203735, 0.954304368656401, 0.44612105486203, 1.09255026112807, 1.66177432848481, 1.2244211095895, -0.307344980789505, 0.362311341134403, -0.707302108346803, -0.144104822654965, 2.80967986565376, 0.886724301741435, -0.437691146279499, 0.227891475847955, 4.03744118286034, 2.45748912778531, 0.144254311972021, 2.61650674781118, 4.31326476995183, 1.48977229748571, 2.34613355467357, 1.70052061441654, 3.22056170230932, 1.78075346385693, -0.711428102809615, 1.34730204516023, 1.03029253615151, 0.419549571357165, 1.58776493288537, 0.995197584563667, 0.642274699864077, 2.07636006524544, 2.38487012186388, 1.23008817197199, 1.38552304464906, -0.660629532295937, 2.90020370216734, 2.14880989578056, 0.0397416585750139, 3.38848652983915, 1.5391831069985, 3.45379719174161, 0.398418170016988, 2.92591755647753, 0.269001120074444, 3.49568149729268, 1.10053690527001, 0.534365065625511, 2.45681592071403, 1.59579092024735, 0.819893378575548, 2.81625032274877, -0.00324530506939347, 0.605258529293289, 3.85479849191654, 0.583541472525013, 2.80499048092448, 3.43673863609605, 2.53588037120317, 1.05364560288094, 1.74797954925627, 1.12861746747233, 2.84125481556006, 3.12911840005426, 0.302617858060385, 2.05438327553933, 0.414404978188778, 1.89216322378059, 1.13139413791383, 2.00133216270611, 0.643782885192826, 1.88591067421528, 0.252206326893992, 0.687547110267291, 1.95059516009538, 0.154987971181278, -1.84401576567585, 1.42142999538519, 1.45740057865859, 0.642256402067309, 2.85687382154418, 2.56544873039477, 1.55960416827183, 5.34308795625776, -0.487166716367998, 1.11969723983059, 1.31495808447936, 1.95142653439584, 3.50775782707169, 1.04468240755541, 2.15013315602819, 2.36947457467637, 1.91237343735871, 0.613733460400012, 4.06876671513254, 3.00698872518514, 2.43255060792113, 0.590220612096163, 0.790617682152215, 0.3088784929935, -0.390557110671398, 0.783542381656206, 2.54825966725677, 0.531623874159247, 0.754254870486599, 2.26979312680552, 0.882020570135368, 3.67733208815144, 1.18041126492999, 1.67479917330522, 1.13098167264533, 3.93443859164881, 0.545035532418525, -0.835817441782716], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + } + } + }, + "unbalanced_cohorts": { + "data_file": "twfeweights_unbalanced_panel.csv", + "columns": { + "outcome": "outcome", + "unit": "unit", + "time": "period", + "first_treat": "first_treat", + "invariant_cov": "x1", + "varying_cov": "xtv" + }, + "two_period_group": 3, + "attgt_weights": { + "twfe": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121212, 0.212121212121212, 0.053030303030303, 0.053030303030303, -0.0378787878787879, -0.0378787878787879, -0.196969696969697, -0.196969696969697, 0.234848484848485, 0.234848484848485], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], + "implied_att": 1.88846922090824 + }, + "overall": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "weight": [0, 0, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0, 0, 0, 0, 0.230769230769231, 0.230769230769231], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], + "implied_att": 2.25856121537518 + }, + "simple": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "weight": [0, 0, 0.175, 0.175, 0.175, 0.175, 0, 0, 0, 0, 0.15, 0.15], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], + "implied_att": 2.43266114548473 + } + }, + "decompose": { + "fwl_nocov": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "att": [0, -0.14625813390341, 1.14471439240315, 2.1890605959757, 3.06376746663026, 4.04138335784424, 0, -0.0481726004625502, 0.331153365133827, 0.149892455219012, 1.19117629320198, 2.46506484725609], + "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121213, 0.212121212121213, 0.053030303030302, 0.053030303030302, -0.0378787878787875, -0.0378787878787875, -0.196969696969696, -0.196969696969696, 0.234848484848484, 0.234848484848484], + "ess": [120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.88846922090823, + "decomposition": 1.88846922090823, + "remainder": 0, + "pretrend_bias": -0.0541461639412249, + "post_only": 1.94261538484946, + "effective_sample_size": 719.999999999998 + }, + "fwl_cov": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "att": [0, -0.146200722768517, 1.17426164200888, 2.18125058212415, 3.08489273485064, 4.02603401811628, 0, -0.0607908732286701, 0.361790510905905, 0.134790399259838, 1.18902450338214, 2.46470121594505], + "weight": [-0.265232105650193, -0.265193408359637, 0.212690896337156, 0.211910908562023, 0.0525407450858316, 0.0532829640248196, -0.0383900377379283, -0.0374892499005717, -0.197107012378148, -0.196588185973519, 0.235333389228316, 0.234241096761852], + "ess": [119.857036307608, 119.848224124526, 83.99514247147, 85.9094063814382, 119.861183035227, 119.861142580033, 119.857036307608, 119.848224124526, 83.99514247147, 85.9094063814382, 119.861183035227, 119.861142580033], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.88897945007637, + "decomposition": 1.88897945007637, + "remainder": 0, + "pretrend_bias": -0.0567591745747959, + "post_only": 1.94573862465116, + "effective_sample_size": 630.228235498893 + }, + "fwl_gmin1": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914814, 0, 1.04128383798297, 2.31517239203707], + "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121213, 0.212121212121213, 0.053030303030302, 0.053030303030302, -0.0378787878787875, -0.0378787878787875, -0.196969696969696, -0.196969696969696, 0.234848484848484, 0.234848484848484], + "ess": [120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120], + "remainder": [-0.945688714374791, -0.945688714374791, -0.945688714374792, -0.945688714374792, -0.945688714374791, -0.945688714374791, -1.88599400101312, -1.88599400101312, -1.88599400101312, -1.88599400101312, -1.88599400101312, -1.88599400101312] + }, + "estimate": 1.88846922090823, + "decomposition": 1.88846922090823, + "remainder": -1.52655665885959e-16, + "pretrend_bias": -0.0613032635598606, + "post_only": 1.94977248446809, + "effective_sample_size": 719.999999999998 + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 3, 5, 5], + "time": [3, 4, 5, 6, 5, 6], + "post": [1, 1, 1, 1, 1, 1], + "att": [1.28404655298829, 2.32056347203679, 3.18519253119276, 4.15837159652383, 0.966820926227897, 2.20786930029771], + "weight": [0.134615384615385, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0.230769230769231, 0.230769230769231], + "ess": [119.903588727274, 119.903588727274, 119.903588727274, 119.903588727274, 115.072096311797, 115.072096311797] + }, + "estimate": 2.20641349591344, + "decomposition": 2.20641349591344, + "remainder": 0, + "pretrend_bias": 0, + "post_only": 2.20641349591344, + "effective_sample_size": 706.0420149054 + } + }, + "balance": { + "fwl": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], + "time": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6], + "post": [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [-0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615], + "unweighted_covs_comparison": [-0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237], + "unweighted_diff": [0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778], + "weighted_covs_treated": [-0.0552006329079015, 0.638635766785036, -0.0535601307184772, 0.640404316629236, -0.0544923109469719, 0.6401601593518, -0.0577693387390503, 0.637312816179378, -0.032929972143668, 0.648304910559683, -0.0616076261294278, 0.637080053522665, 0.0835621488155271, 0.737738482174229, 0.0945036039710708, 0.744157273744997, 0.108279525337989, 0.748815310739362, 0.103457592255963, 0.746870083930084, 0.104726058647783, 0.747708304746342, 0.101546993276873, 0.745734043929024], + "weighted_covs_comparison": [-0.0820211156139367, 0.693680674197995, -0.0796214274629394, 0.694352196990227, -0.0689935730028542, 0.71126680230441, -0.056084159820465, 0.688019746753786, -0.0799713013479677, 0.694299297567012, -0.0836355509702227, 0.693116732796288, -0.0820211156139367, 0.693680674197995, -0.0796214274629394, 0.694352196990227, -0.0689935730028542, 0.71126680230441, -0.056084159820465, 0.688019746753786, -0.0799713013479677, 0.694299297567012, -0.0836355509702227, 0.693116732796288], + "weighted_diff": [0.0268204827060351, -0.0550449074129595, 0.0260612967444623, -0.0539478803609912, 0.0145012620558822, -0.0711066429526097, -0.00168517891858527, -0.0507069305744083, 0.0470413292042998, -0.0459943870073296, 0.0220279248407949, -0.0560366792736232, 0.165583264429464, 0.0440578079762343, 0.17412503143401, 0.0498050767547699, 0.177273098340843, 0.0375485084349517, 0.159541752076428, 0.0588503371762982, 0.184697359995751, 0.0534090071793297, 0.185182544247096, 0.0526173111327358], + "sd": [0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283], + "unweighted_log_ratio_sd_diff": [-0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277], + "weighted_log_ratio_sd_diff": [-0.327379355480894, -0.388856622667763, -0.332728063219466, -0.393001836482863, -0.482246153471369, -0.760575572777059, -0.444382241712145, -0.729047181913841, -0.341186304526637, -0.365744546430006, -0.294845966896052, -0.373086629641763, -0.277042097435864, -0.27985283207957, -0.331236252179694, -0.345749079434454, -0.457886921184364, -0.693111339953135, -0.430872341074444, -0.671839969125508, -0.316957716605812, -0.335463807510608, -0.312906075652989, -0.333392440708849], + "unweighted_frac_treated_extreme": [0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333], + "weighted_frac_treated_extreme": [0.0285714285714285, 0.0142857142857143, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0142857142857143, 0.0142857142857142, 0.0142857142857143, 0.0428571428571429, 0.0285714285714285, 0.0428571428571428, 0.0285714285714285, 0.133333333333333, 0.05, 0.133333333333333, 0.0333333333333333, 0.05, 0, 0.05, 0, 0.0833333333333333, 0.0333333333333333, 0.1, 0.0333333333333333] + }, + "summary": { + "unweighted_treat": [0.0202808028752963, 0.690196397646397], + "unweighted_untreat": [-0.081027435542576, 0.693933252829235], + "unweighted_diff": [0.101308238417872, -0.00373685518283795], + "weighted_treat": [0.0195872436833426, 0.689860002475752], + "weighted_untreat": [-0.0736280525335835, 0.696237410214999], + "weighted_diff": [0.0932152962169262, -0.00637740773924623], + "sd": [0.904821714009258, 0.497941950253664], + "unweighted_log_ratio_sd_diff": [-0.323553909442524, -0.3621867923957], + "weighted_log_ratio_sd_diff": [-0.378261456962234, -0.512396249204556], + "unweighted_frac_treated_extreme": [0.0701874388083532, 0.030807497552334], + "weighted_frac_treated_extreme": [0.0536362673342276, 0.0247417574823458], + "covariate": ["mean_x1", "mean_xtv"] + } + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 3, 3, 5, 5, 5, 5], + "time": [3, 3, 4, 4, 5, 5, 6, 6, 5, 5, 6, 6], + "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [-0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615], + "unweighted_covs_comparison": [-0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237], + "unweighted_diff": [0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778], + "weighted_covs_treated": [-0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615], + "weighted_covs_comparison": [-0.0552433456376623, 0.707144787976769, -0.0552433456376623, 0.707144787976769, -0.0552433456376623, 0.707144787976769, -0.0552433456376623, 0.707144787976769, 0.105591944111321, 0.789549339556244, 0.105591944111321, 0.789549339556244], + "weighted_diff": [-1.38777878078145e-17, -0.0682714332823932, -1.38777878078145e-17, -0.0682714332823932, -1.38777878078145e-17, -0.0682714332823932, -1.38777878078145e-17, -0.0682714332823932, -8.32667268468867e-17, -0.041379069285629, -8.32667268468867e-17, -0.041379069285629], + "sd": [0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283] + }, + "summary": { + "unweighted_treat": [0.0189883265541762, 0.689318084960332], + "unweighted_untreat": [-0.0810274355425762, 0.693933252829237], + "unweighted_diff": [0.100015762096752, -0.00461516786890477], + "weighted_treat": [0.0189883265541762, 0.689318084960332], + "weighted_untreat": [0.0189883265541762, 0.745177657936526], + "weighted_diff": [-4.5903451979694e-17, -0.0558595729761943], + "sd": [0.904574085367837, 0.497745508663157], + "unweighted_log_ratio_sd_diff": [0, 0], + "weighted_log_ratio_sd_diff": [0, 0], + "unweighted_frac_treated_extreme": [0, 0], + "weighted_frac_treated_extreme": [0, 0], + "covariate": ["mean_x1", "mean_xtv"] + } + } + }, + "two_period": { + "reg": { + "estimate": 1.28957044213691, + "ess": 119.995700148386, + "weights": [1.00925477055982, 0.998723185675059, 0.996634725503275, 1.0062962595469, 1.0116846563981, 1.00452028093292, 1.00771486087647, 1.00204953103351, 1.0081898745094, 0.993138351267652, 1.00064950601217, 1.00958832842553, 1.00708633443042, 1.00143658864871, 0.999489519214752, 1.00911008327078, 1.00168297069056, 0.983050385189857, 1.00668374658622, 0.994367888846623, 1.00306411999543, 1.00581032890592, 0.995834544735198, 1.00135304942198, 0.997982911644572, 1.00549542164811, 0.992021816202564, 0.99946715234167, 0.994504899838761, 1.00336245204938, 0.99810636855178, 1.0042481533285, 1.00202880076963, 0.99377550716937, 1.00083681709583, 0.993840646573631, 0.995025443722193, 0.996441015885937, 1.00837717223639, 1.00287990494642, 1.00165231794312, 0.996029676232405, 0.990238585162026, 0.995938530509862, 0.998372849179898, 1.00217444576362, 1.0017515355913, 0.997885202550526, 1.00957796620665, 0.99454446944383, 1.00781508610486, 0.989676661130244, 1.00195850082524, 1.00228415784846, 1.0044654432561, 1.01551328546879, 0.993101453522634, 1.00042397201405, 1.00324727474383, 0.996107026201761, 1.00279636023904, 0.999681571328118, 1.00552645669481, 1.00510148957087, 0.991265296747816, 0.999928753044178, 0.994735345537259, 0.998610758079682, 0.999659826801045, 1.00695181771713, 1.00087111390321, 0.99239696826285, 0.997596950451456, 1.00167680363745, 0.998186813694359, 1.00314547233257, 0.989000472345918, 0.989810225580615, 1.00068860595832, 1.00045926654644, 1.00185508614782, 0.997098887679209, 0.995092885869272, 1.00882365715098, 0.995943321465413, 0.995345394281881, 1.00242170714711, 1.01131485449591, 1.01546992314392, 1.00473931229751, 1.00454884206903, 1.00120579562608, 0.991933253970624, 0.992552423967053, 1.00103232626908, 0.996073666367965, 1.00606469413823, 0.993935153780701, 0.993405482710502, 1.00576004097157, 1.00404133753852, 0.990720622466242, 0.9895886575647, 0.990789224140744, 0.998229047704058, 0.993249994137338, 0.997704430441061, 1.00139407975238, 0.997384073985491, 0.998716973738327, 0.996509691218583, 0.994121921942389, 1.00069492728128, 0.99361118386715, 1.01106122493825, 0.999614467420359, 0.99719516770885, 1.00761213299192, 1.00044650582276, 0.996040442935408, 0.995776410452928, 1.00014559518799, 1.00163036922118, 0.998107414758181, 0.998929265083554, 1.00394146389334, 0.994663562354314, 0.997264291076991, 0.999722078445559, 1.00127725705156, 0.999660594616231, 0.997495701681234, 0.999221017580553, 0.999202469015296, 1.00137763212731, 1.00356440217487, 0.998483204711311, 0.999264607253516, 1.00003520861784, 1.00231366955048, 0.996020219541034, 1.00107946453243, 1.00122012240469, 0.995993541722388, 0.9957999534808, 1.00358672328067, 1.00377596409354, 1.00283713463745, 0.998897288822817, 1.00140206334268, 0.994859786413371, 0.999617444016502, 1.00558977187743, 1.00151742123166, 0.999440356527955, 1.00348353086931, 1.00103531753353, 0.994836036333979, 0.996484335173441, 1.00690040871755, 1.00252882679405, 0.996057521027822, 1.00074181334411, 0.999624789407533, 0.997053878174417, 0.999466692467566, 0.998408862902966, 0.998885822158993, 0.996357520999377, 0.999797669251639, 0.99870100074395, 1.00039444814759, 1.00012122673967, 1.00333196253064, 0.997811178378975, 1.00012878664617, 1.00344301864085, 0.998071892714265, 1.00306218854884, 1.00194832910166, 0.998881145839971, 1.00128524589817, 0.999049662363083, 0.995286955499897, 1.00136819116663, 0.9990008180002, 1.00499457305456, 0.999427617457224, 1.00038272399958, 1.00793253859212], + "dy": [1.38026515156733, 0.408266791145164, -0.339431469645495, -0.682691992758838, -1.82871929455127, 1.14929021958467, 0.406093652733488, 1.39592916302462, 3.44235885109425, -0.820263245202869, -0.0318323352249712, 1.85077537805184, 0.662440201648147, -0.687203070639989, 1.81011852855793, -0.139481071156265, 2.4168721220432, 0.379022022235292, -0.914287504013753, 0.944043804231505, -0.215693960694542, 1.41072414986002, 1.36325294550578, -1.46123322504876, 0.715429531063086, 0.26309981815957, -0.358751783679667, 1.84249495658836, 1.73947344841319, -1.75329820036687, -0.379579864813726, -3.50097882222108, -1.05188660915389, 0.2536693555151, -0.971711160276842, -1.19521084045457, -2.01434628951932, 3.06082555979665, 0.838004826699251, 2.54024677095855, 1.18760409054941, 0.190422579698768, -2.79013189474126, 1.78440369318915, 1.76759766962424, -0.382078014138221, 0.337068442905171, 1.42244200085645, -0.382337659274782, 1.77717225422036, 1.5667240658887, -0.0648724566595706, 1.40835348803613, -1.07517031384426, 2.0448771635733, 0.20387928210247, -1.35210249661022, -0.586028724744774, -0.84201349457584, 1.90258131506353, 0.411606784318087, -0.439901415814671, 0.400413556728919, -0.522642332863673, 0.488397482651255, 1.51572329544617, 1.52262212337139, -0.205771811432765, -0.358852846817045, 2.97812490826596, 2.27471004314199, -1.73123416913395, -1.01084567421892, 1.110914474459, -0.56793074637455, 1.0457947568757, 0.189080638316264, 2.42007503599557, 0.591808809392759, 1.69135034688281, -1.40091447368174, 0.443539056450542, -1.29350918787999, 1.37109855555011, 0.910485433286087, 0.393159902321618, 0.52588872717339, -0.0699614188535208, -0.205053614911856, -0.121870541194142, 0.948639553493953, -2.55181124672349, -1.62116974064653, -0.927339136556927, -0.0293829057582144, -1.08857253360696, 0.436893948712147, -1.25929831720187, 1.66427394651085, -0.721911619253202, -0.187086780075112, -0.972907200749254, -2.49479634001194, 0.292516580055634, 0.631803837727804, 2.23814166613728, -1.83270431658743, -0.05769627797699, -0.666103978831666, -1.01907287013042, 0.479112620179818, -1.13589709492181, 2.18379229837681, 1.30267223459711, 3.10565034266606, 1.4545741281931, 1.49958177865965, 0.113309221745668, -1.75553538137332, 0.243298221611488, -0.599070157367475, 2.18711836254605, 2.61108014497317, 1.04904278673023, 2.57766985911455, 0.120403846438697, 2.91873222356864, 1.90846716393393, 1.35400218922492, 3.55370994373775, 3.48346062298992, 0.968735809911961, 2.14174807183156, 1.32808583874811, 2.16875563049545, 0.0929685385249678, 2.22990211084164, 1.47934760472248, -0.3262962186754, 0.757439304925393, 0.526780613595748, -1.390667914841, 1.59312449761787, 3.51618200657663, 2.47565428954465, 1.17747469721277, 0.996548478708077, 3.22839349697434, 1.548697329824, 3.64023772923273, 0.885084747149945, 1.48169501229723, -0.39715445631083, 2.15823078284085, 0.896920843058143, 3.9954179341778, 0.289878649966864, 3.46531030502758, 3.09985226481737, -0.975210984984278, 1.73748432962649, 0.0202998463626913, 2.77636804375209, 3.67621364383077, -2.51891159868698, 1.62921382344766, -0.30025454695664, 1.8676058001887, 0.645579086042666, 2.86346682269322, 0.972051047344538, 1.28776117593775, 3.75782604200966, 0.202780599744159, -0.738157146955598, 2.73954177588088, 3.76679370320148, 2.0531571498699, -0.581571362098442, 3.5528765383084, 1.09848339399543, 2.93359377645159, -0.0933376093283611, 1.29039352914662, 3.10380852101533, 1.23415773944705, 2.05415132958319, 1.86615510766519, 1.18501304815952, -0.0391895303226968], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + }, + "aipw": { + "estimate": 1.29320632882505, + "ess": 119.995175832593, + "weights": [0.988121343500857, 1.00319460592959, 0.999326474631137, 0.995659632023005, 0.998606433108666, 0.995983391028368, 0.997905454026944, 0.993067190848352, 0.987447676543771, 1.00722546202046, 0.996098999745148, 0.995760987841229, 0.992551110480417, 1.00946334325196, 1.00213904078528, 0.99656654815872, 0.997707286846142, 1.01031273245395, 0.992552032760831, 1.00246403828434, 0.995416775257639, 1.00074394014255, 0.993277350509223, 1.00046363134566, 0.999387638777348, 0.995436337298158, 1.01002540383716, 0.997641596157817, 1.00142964949572, 1.00306186221818, 0.987568975485235, 0.998901146783634, 0.994210429294235, 1.00267630862969, 0.999430296647091, 0.994417627787845, 1.0006388237355, 0.997913197441718, 1.00474394702358, 0.998439282442818, 0.99809325945128, 0.996577959626109, 1.00807177937616, 0.997758940856196, 0.996365194057743, 0.992843437496156, 0.992961566287008, 1.01184445051893, 0.991172499125968, 1.00607581691, 0.988090533687459, 1.01404534961072, 1.00099910277687, 0.99727551050779, 0.997430908996597, 0.995323694271336, 1.01079586728789, 1.00383782318859, 0.990728396959983, 0.999165942059289, 0.991442682725426, 1.00480350190228, 0.999686306593479, 0.996424768681225, 1.00526516565322, 0.994278461080079, 1.00117423547719, 0.999522222423479, 1.00388724440051, 0.994125269190137, 0.988914815060044, 1.00873014851802, 1.0008025087164, 0.993333496799252, 1.002362069121, 0.995094506867751, 1.01554227443789, 1.01149662395533, 1.00848620855313, 1.00208384713849, 1.0059243759256, 1.00602450217315, 1.00674544005413, 0.994182090996886, 0.998685827626876, 1.00574936778508, 0.994171531804436, 0.990220250456699, 0.994937460421399, 1.00221250585952, 1.00502295762636, 1.00314061240821, 1.01594165711587, 1.00263250272366, 1.00423439904399, 0.999478219836675, 0.9869427697955, 1.00847994595626, 0.999754343768729, 0.990426930175441, 1.00285881267497, 1.00224635046947, 0.999841222797544, 1.01201466214112, 0.996777185410859, 0.994290009671401, 1.00355719467655, 1.00143411749026, 1.00053710082179, 1.01403368944758, 1.00088018587467, 1.00333167103173, 0.998578495421041, 1.00604281551186, 0.995915049400813, 0.996641880507851, 1.00573881013259, 1.0049267688703, 0.998445151041532, 1.00210874012526, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "dy": [1.38026515156733, 0.408266791145164, -0.339431469645495, -0.682691992758838, -1.82871929455127, 1.14929021958467, 0.406093652733488, 1.39592916302462, 3.44235885109425, -0.820263245202869, -0.0318323352249712, 1.85077537805184, 0.662440201648147, -0.687203070639989, 1.81011852855793, -0.139481071156265, 2.4168721220432, 0.379022022235292, -0.914287504013753, 0.944043804231505, -0.215693960694542, 1.41072414986002, 1.36325294550578, -1.46123322504876, 0.715429531063086, 0.26309981815957, -0.358751783679667, 1.84249495658836, 1.73947344841319, -1.75329820036687, -0.379579864813726, -3.50097882222108, -1.05188660915389, 0.2536693555151, -0.971711160276842, -1.19521084045457, -2.01434628951932, 3.06082555979665, 0.838004826699251, 2.54024677095855, 1.18760409054941, 0.190422579698768, -2.79013189474126, 1.78440369318915, 1.76759766962424, -0.382078014138221, 0.337068442905171, 1.42244200085645, -0.382337659274782, 1.77717225422036, 1.5667240658887, -0.0648724566595706, 1.40835348803613, -1.07517031384426, 2.0448771635733, 0.20387928210247, -1.35210249661022, -0.586028724744774, -0.84201349457584, 1.90258131506353, 0.411606784318087, -0.439901415814671, 0.400413556728919, -0.522642332863673, 0.488397482651255, 1.51572329544617, 1.52262212337139, -0.205771811432765, -0.358852846817045, 2.97812490826596, 2.27471004314199, -1.73123416913395, -1.01084567421892, 1.110914474459, -0.56793074637455, 1.0457947568757, 0.189080638316264, 2.42007503599557, 0.591808809392759, 1.69135034688281, -1.40091447368174, 0.443539056450542, -1.29350918787999, 1.37109855555011, 0.910485433286087, 0.393159902321618, 0.52588872717339, -0.0699614188535208, -0.205053614911856, -0.121870541194142, 0.948639553493953, -2.55181124672349, -1.62116974064653, -0.927339136556927, -0.0293829057582144, -1.08857253360696, 0.436893948712147, -1.25929831720187, 1.66427394651085, -0.721911619253202, -0.187086780075112, -0.972907200749254, -2.49479634001194, 0.292516580055634, 0.631803837727804, 2.23814166613728, -1.83270431658743, -0.05769627797699, -0.666103978831666, -1.01907287013042, 0.479112620179818, -1.13589709492181, 2.18379229837681, 1.30267223459711, 3.10565034266606, 1.4545741281931, 1.49958177865965, 0.113309221745668, -1.75553538137332, 0.243298221611488, -0.599070157367475, 2.18711836254605, 2.61108014497317, 1.04904278673023, 2.57766985911455, 0.120403846438697, 2.91873222356864, 1.90846716393393, 1.35400218922492, 3.55370994373775, 3.48346062298992, 0.968735809911961, 2.14174807183156, 1.32808583874811, 2.16875563049545, 0.0929685385249678, 2.22990211084164, 1.47934760472248, -0.3262962186754, 0.757439304925393, 0.526780613595748, -1.390667914841, 1.59312449761787, 3.51618200657663, 2.47565428954465, 1.17747469721277, 0.996548478708077, 3.22839349697434, 1.548697329824, 3.64023772923273, 0.885084747149945, 1.48169501229723, -0.39715445631083, 2.15823078284085, 0.896920843058143, 3.9954179341778, 0.289878649966864, 3.46531030502758, 3.09985226481737, -0.975210984984278, 1.73748432962649, 0.0202998463626913, 2.77636804375209, 3.67621364383077, -2.51891159868698, 1.62921382344766, -0.30025454695664, 1.8676058001887, 0.645579086042666, 2.86346682269322, 0.972051047344538, 1.28776117593775, 3.75782604200966, 0.202780599744159, -0.738157146955598, 2.73954177588088, 3.76679370320148, 2.0531571498699, -0.581571362098442, 3.5528765383084, 1.09848339399543, 2.93359377645159, -0.0933376093283611, 1.29039352914662, 3.10380852101533, 1.23415773944705, 2.05415132958319, 1.86615510766519, 1.18501304815952, -0.0391895303226968], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + } + } + } + } +} diff --git a/benchmarks/data/twfeweights_sim_panel.csv b/benchmarks/data/twfeweights_sim_panel.csv new file mode 100644 index 000000000..48fbc4d96 --- /dev/null +++ b/benchmarks/data/twfeweights_sim_panel.csv @@ -0,0 +1,1501 @@ +"unit","period","first_treat","x1","xtv","outcome" +1,1,0,-0.44174194168612,0.962638158757321,0.250725238240098 +1,2,0,-0.44174194168612,0.480873514413711,0.302633430899334 +1,3,0,-0.44174194168612,0.114120072263393,-0.099400179401941 +1,4,0,-0.44174194168612,0.448153732387612,1.47267278381102 +1,5,0,-0.44174194168612,0.0450381423543704,-0.304624682447771 +2,1,0,1.00712662877707,0.809674316987755,1.40448682517955 +2,2,0,1.00712662877707,0.875226098004157,1.11498919693233 +2,3,0,1.00712662877707,1.73308918012076,3.30043194865778 +2,4,0,1.00712662877707,1.94159375407109,3.73624251678568 +2,5,0,1.00712662877707,1.49919266041645,5.00490117694144 +3,1,0,0.179611286037529,0.143495801148179,-0.619592378770517 +3,2,0,0.179611286037529,-0.183975411244885,1.59825802459073 +3,3,0,0.179611286037529,1.30125701337682,2.00613564154948 +3,4,0,0.179611286037529,2.14545926587906,3.02931552915487 +3,5,0,0.179611286037529,1.270347859372,0.626648784956356 +4,1,0,1.13465562548833,0.98082184944363,0.789054631222788 +4,2,0,1.13465562548833,0.757860539178009,0.885976737984897 +4,3,0,1.13465562548833,0.644275492814336,2.72497858093159 +4,4,0,1.13465562548833,0.804994143245251,2.7470035085833 +4,5,0,1.13465562548833,3.1402591500037,5.6594454100412 +5,1,0,-2.56473120432832,-1.42045200997001,-2.47773503860188 +5,2,0,-2.56473120432832,-0.99158787828575,-1.91595275185634 +5,3,0,-2.56473120432832,-0.956287354318694,-1.57386277107317 +5,4,0,-2.56473120432832,-0.173688498361684,-2.32481328982363 +5,5,0,-2.56473120432832,-0.220122926796195,-4.77426084425192 +6,1,0,2.03726526019619,0.906852984645204,3.09145390139369 +6,2,0,2.03726526019619,1.10736421765238,1.28301946741103 +6,3,0,2.03726526019619,1.94654212178223,6.4558523593605 +6,4,0,2.03726526019619,2.29731461571635,3.98797124370253 +6,5,0,2.03726526019619,1.90892019581178,8.63257346295759 +7,1,0,0.982893295690819,0.925972608667814,2.56477125464081 +7,2,0,0.982893295690819,1.72746462392985,1.37542121177932 +7,3,0,0.982893295690819,2.01554559420596,3.15083265246945 +7,4,0,0.982893295690819,1.17634719987377,4.35800530188081 +7,5,0,0.982893295690819,0.760671887747645,4.47518570225275 +8,1,0,1.80195765072669,1.92370507752248,2.46720402294067 +8,2,0,1.80195765072669,1.32616500548667,2.7227903293348 +8,3,0,1.80195765072669,1.28608793752108,2.90070675615349 +8,4,0,1.80195765072669,1.81009744339314,5.80136124967764 +8,5,0,1.80195765072669,1.15760990674598,6.55560198907432 +9,1,0,0.225424255156626,0.763669731908457,1.54288624086341 +9,2,0,0.225424255156626,0.874926359148595,0.836845360548911 +9,3,0,0.225424255156626,1.0117962358522,1.82503853096451 +9,4,0,0.225424255156626,0.603697174472571,-0.416276008333528 +9,5,0,0.225424255156626,1.18445658923149,2.87016682298805 +10,1,0,0.0609610990580548,-0.529661334550992,1.66925745533143 +10,2,0,0.0609610990580548,0.0914889871378843,4.08422527417878 +10,3,0,0.0609610990580548,1.48655808874342,2.35118727045354 +10,4,0,0.0609610990580548,0.00203530191935619,2.33692220382611 +10,5,0,0.0609610990580548,1.28112960064862,3.21562126389664 +11,1,0,0.211622353786371,0.677732585269737,1.01402669670136 +11,2,0,0.211622353786371,-0.455890381301695,1.09790190289759 +11,3,0,0.211622353786371,0.221008710144804,0.642633318493822 +11,4,0,0.211622353786371,0.855358439590161,1.02060294398976 +11,5,0,0.211622353786371,1.46959772659548,2.37526538283031 +12,1,0,-0.106312592206072,-9.23288872419215e-05,-0.658080268881745 +12,2,0,-0.106312592206072,-0.0970387920280995,0.736568103159182 +12,3,0,-0.106312592206072,1.28722421860427,1.36381391065831 +12,4,0,-0.106312592206072,0.493486811724755,0.441330823328568 +12,5,0,-0.106312592206072,0.954292388879426,0.728253826747307 +13,1,0,-0.775220530410582,-0.0485173410173907,0.428116160890274 +13,2,0,-0.775220530410582,0.10663036250203,-0.120754966975773 +13,3,0,-0.775220530410582,-0.0932015246981229,0.830881466674987 +13,4,0,-0.775220530410582,0.645640369635815,2.78067278573257 +13,5,0,-0.775220530410582,0.950987026453864,1.73479528567685 +14,1,0,2.04614622265237,0.509538313746868,-0.195487168960969 +14,2,0,2.04614622265237,1.26460645944908,2.49104556632605 +14,3,0,2.04614622265237,1.61979242853853,3.22054459345334 +14,4,0,2.04614622265237,1.96446721321234,2.94031366637701 +14,5,0,2.04614622265237,1.6819638784622,6.3045203371182 +15,1,0,1.91047856851638,1.41809400206497,0.352933179214503 +15,2,0,1.91047856851638,0.868574620082492,2.14983395101162 +15,3,0,1.91047856851638,1.38087806774412,1.4332857406434 +15,4,0,1.91047856851638,2.7376571119788,3.36504006381984 +15,5,0,1.91047856851638,2.50140408640708,5.0339321437845 +16,1,0,0.0904267516471942,-0.127925449618051,-0.629070962146693 +16,2,0,0.0904267516471942,-0.222640384632004,-0.60311595170131 +16,3,0,0.0904267516471942,0.209835589276016,0.386063603258975 +16,4,0,0.0904267516471942,0.518050979618486,1.34554957607334 +16,5,0,0.0904267516471942,1.35263633341545,1.76253705516832 +17,1,0,-0.978997126733862,-0.430729084042156,1.11006747787362 +17,2,0,-0.978997126733862,0.330410967028457,0.806951193076084 +17,3,0,-0.978997126733862,0.684625215247789,1.422241220599 +17,4,0,-0.978997126733862,0.436699992122361,1.05823352541785 +17,5,0,-0.978997126733862,-0.0965033949600655,-0.28861652804708 +18,1,0,-0.305160444674541,-0.916075051557808,-0.542992489238567 +18,2,0,-0.305160444674541,0.248257475817807,-0.860451526400478 +18,3,0,-0.305160444674541,0.798314680654779,-1.7886253887699 +18,4,0,-0.305160444674541,0.521228762410946,0.221495447624939 +18,5,0,-0.305160444674541,0.603880966224836,0.268782385230717 +19,1,0,-1.29347091631414,-0.440720995998017,-0.0917243454597562 +19,2,0,-1.29347091631414,-0.292457346868644,1.69390714797117 +19,3,0,-1.29347091631414,-0.076852304649137,-1.4267554614469 +19,4,0,-1.29347091631414,0.523168449121407,0.836559020778829 +19,5,0,-1.29347091631414,-0.126845315020416,-0.748902007493232 +20,1,0,-0.86946128396502,-0.731640889278664,0.897492140407103 +20,2,0,-0.86946128396502,-0.634157933584901,-0.356647943745405 +20,3,0,-0.86946128396502,0.414634427219905,-0.102122625271403 +20,4,0,-0.86946128396502,0.735617051894694,-0.493531099100508 +20,5,0,-0.86946128396502,0.0783122168879679,2.04744316684707 +21,1,0,0.193364867469568,0.527132325030444,2.83509885985534 +21,2,0,0.193364867469568,0.129936563195705,0.377414995366493 +21,3,0,0.193364867469568,0.613829784876638,2.64691385949551 +21,4,0,0.193364867469568,-0.120119586049108,2.99668502488452 +21,5,0,0.193364867469568,0.508258887874505,4.53816843932383 +22,1,0,-0.285006996929422,-0.544273797377208,0.61950479499134 +22,2,0,-0.285006996929422,-0.0470278141175047,3.0416910941977 +22,3,0,-0.285006996929422,0.307232669634666,3.87159719910662 +22,4,0,-0.285006996929422,1.01711272566063,2.77876159208761 +22,5,0,-0.285006996929422,1.31564072955683,3.62013111033041 +23,1,0,-0.200733998184601,0.406863652048468,0.0824709662570302 +23,2,0,-0.200733998184601,-0.0257062143537835,0.394833336515454 +23,3,0,-0.200733998184601,0.592886053520876,1.23455541173059 +23,4,0,-0.200733998184601,0.110099171857438,2.54019055351788 +23,5,0,-0.200733998184601,0.963691513380404,2.35389561030956 +24,1,0,1.24328569249421,0.92191360816656,1.14586532408122 +24,2,0,1.24328569249421,1.47628244052515,1.97289272773888 +24,3,0,1.24328569249421,0.727658826710546,2.37572049221841 +24,4,0,1.24328569249421,1.32540192428514,6.1798378307657 +24,5,0,1.24328569249421,1.99608430464066,7.76758515003412 +25,1,0,0.472298422553046,0.451130953214354,2.3849436399372 +25,2,0,0.472298422553046,0.393597463425591,-0.0898224276117208 +25,3,0,0.472298422553046,1.25037077852183,2.52589921950728 +25,4,0,0.472298422553046,0.523272852971633,2.37917100316457 +25,5,0,0.472298422553046,0.962139498902349,4.36517881139554 +26,1,0,0.416224904614732,-0.101584055903843,-0.579744769633871 +26,2,0,0.416224904614732,0.400325561772782,-0.972234260476899 +26,3,0,0.416224904614732,0.138863379638057,1.59661513491138 +26,4,0,0.416224904614732,0.397912352674014,0.742603700730964 +26,5,0,0.416224904614732,0.920278133420186,0.713813878237303 +27,1,0,-1.20810463188673,-0.656731791542808,0.112586405323568 +27,2,0,-1.20810463188673,-0.103560125599836,1.12355202824385 +27,3,0,-1.20810463188673,-0.197567905296184,1.53340035807994 +27,4,0,-1.20810463188673,0.974601656086776,-0.432564528542204 +27,5,0,-1.20810463188673,-0.0897357035583219,-0.828997632235555 +28,1,0,-1.43723667380504,0.0565108804189347,-1.84738581883577 +28,2,0,-1.43723667380504,-0.0297675553081633,-0.894498560080979 +28,3,0,-1.43723667380504,-0.0790701797677419,0.0220988662434747 +28,4,0,-1.43723667380504,-0.224757725463843,-0.847105774844605 +28,5,0,-1.43723667380504,0.166110286931856,-2.68630843144862 +29,1,0,1.08387441716567,0.401051829495605,2.54055593664914 +29,2,0,1.08387441716567,0.129584432966412,-0.290371476862269 +29,3,0,1.08387441716567,0.803794134135084,3.27778845957416 +29,4,0,1.08387441716567,0.930893075272324,4.61666902997821 +29,5,0,1.08387441716567,1.66326112137049,4.37994629191061 +30,1,0,1.14107632772678,-0.130216125775587,1.42153467508241 +30,2,0,1.14107632772678,0.543543313638396,1.82708108795342 +30,3,0,1.14107632772678,0.761496767847219,5.94565127190796 +30,4,0,1.14107632772678,1.11210754956847,3.8284928975795 +30,5,0,1.14107632772678,2.12456034096161,4.35152164443134 +31,1,0,-1.68363239672603,-0.394676649711449,1.23436313536159 +31,2,0,-1.68363239672603,-0.476159350468402,-1.2717016524923 +31,3,0,-1.68363239672603,-0.066870496453248,-1.85565601122068 +31,4,0,-1.68363239672603,0.69531782960049,0.800860195139755 +31,5,0,-1.68363239672603,0.252916979891308,0.5339463200228 +32,1,0,0.35862773442386,-0.254987375618001,-0.0878826690809986 +32,2,0,0.35862773442386,0.402486642358358,1.00755257360309 +32,3,0,0.35862773442386,0.973937634716594,1.63709582600101 +32,4,0,0.35862773442386,1.73454911285787,-1.18345432941171 +32,5,0,0.35862773442386,1.95248503861603,3.25719711630834 +33,1,0,1.03062394978694,1.1300648931605,0.251863182842662 +33,2,0,1.03062394978694,0.764447104885131,2.35818680820894 +33,3,0,1.03062394978694,0.81496241040219,-0.240462101427006 +33,4,0,1.03062394978694,1.23953989524317,2.39602149767805 +33,5,0,1.03062394978694,0.868796603908667,4.07416204116012 +34,1,0,-1.36021422621808,-0.037663969198275,0.281343353835558 +34,2,0,-1.36021422621808,-1.26017420369277,0.802690839736971 +34,3,0,-1.36021422621808,0.246925333860566,-0.703061697563873 +34,4,0,-1.36021422621808,-0.16975234477317,1.32797213567783 +34,5,0,-1.36021422621808,0.00740586453438835,0.44762238971308 +35,1,0,2.07462580052836,1.45675082321133,-0.57933493901855 +35,2,0,2.07462580052836,1.20926972518549,1.55463054174886 +35,3,0,2.07462580052836,2.34845238546939,1.6631742348656 +35,4,0,2.07462580052836,2.52239895186696,2.51391502849078 +35,5,0,2.07462580052836,1.52963756522155,4.43956443092784 +36,1,0,-0.624898820481009,0.322071656054509,-1.51999549840851 +36,2,0,-0.624898820481009,-0.436622498509552,-0.401234013390455 +36,3,0,-0.624898820481009,0.173813685252127,-0.0621678249348833 +36,4,0,-0.624898820481009,0.679651321432316,-0.737748502907965 +36,5,0,-0.624898820481009,0.716835039913072,0.736328659584627 +37,1,0,2.30895941653656,1.9742116653891,-2.10332094044947 +37,2,0,2.30895941653656,1.59260370469019,0.821201875196453 +37,3,0,2.30895941653656,2.61155608501098,1.66217100280885 +37,4,0,2.30895941653656,2.52595052377694,2.47477397356914 +37,5,0,2.30895941653656,2.54572354575135,2.57388225810638 +38,1,0,0.468838681888286,0.343993769445637,2.30521872621672 +38,2,0,0.468838681888286,0.641957713818,2.87394510695166 +38,3,0,0.468838681888286,0.880554440811486,4.3703467296499 +38,4,0,0.468838681888286,0.706193181423138,5.39737846119355 +38,5,0,0.468838681888286,1.30558403199082,5.23379128440331 +39,1,0,-0.427218215500197,0.566084390858194,2.13097968656818 +39,2,0,-0.427218215500197,-0.918399310828481,0.413215180305786 +39,3,0,-0.427218215500197,0.118235771072415,2.57020147610967 +39,4,0,-0.427218215500197,0.340995712937832,2.74807377011147 +39,5,0,-0.427218215500197,0.496211049768828,2.9555621794116 +40,1,0,0.777782496536268,0.300747632479431,-0.368684990628421 +40,2,0,0.777782496536268,1.0010007622223,0.595206586920402 +40,3,0,0.777782496536268,0.47188348651664,1.30591749207004 +40,4,0,0.777782496536268,1.86755346941996,3.31554734968523 +40,5,0,0.777782496536268,0.858892873747311,2.75457375011982 +41,1,0,-0.989113034685617,-0.134240789368439,-1.36994283379942 +41,2,0,-0.989113034685617,0.506570270785846,1.0534061137458 +41,3,0,-0.989113034685617,-0.895973224849036,0.27274725887401 +41,4,0,-0.989113034685617,1.31802792818807,0.26632977281089 +41,5,0,-0.989113034685617,0.454944454611298,2.40853446851008 +42,1,0,1.15840793985001,0.124868418660003,0.259801217583987 +42,2,0,1.15840793985001,1.87953027894752,2.17974265446549 +42,3,0,1.15840793985001,0.673567775709603,2.42638252821395 +42,4,0,1.15840793985001,0.63133400895871,2.89643454446268 +42,5,0,1.15840793985001,0.951863725906572,4.47572451802154 +43,1,0,2.67054324703151,2.31680906226424,1.05064037303325 +43,2,0,2.67054324703151,1.96713340733527,4.01260734361365 +43,3,0,2.67054324703151,2.01331188229246,1.02095053233103 +43,4,0,2.67054324703151,2.61485776901755,4.79120104206803 +43,5,0,2.67054324703151,2.06056399005803,7.36148928041125 +44,1,0,0.515574279047854,0.121900480753256,0.12827299244454 +44,2,0,0.515574279047854,0.443740380381531,0.669446613371002 +44,3,0,0.515574279047854,1.41716460656681,2.88575234491894 +44,4,0,0.515574279047854,1.06078400139549,-0.328837067278666 +44,5,0,0.515574279047854,0.846696081449088,1.42436157265577 +45,1,0,0.354081082960897,0.671167680765342,2.13948449234028 +45,2,0,0.354081082960897,0.293960347886397,1.20288799131482 +45,3,0,0.354081082960897,0.433020579196941,2.56751459819996 +45,4,0,0.354081082960897,0.845873361595047,3.54626002487861 +45,5,0,0.354081082960897,0.672652897873653,4.50017565808538 +46,1,0,1.52337503986612,1.32146148533203,3.34298891732902 +46,2,0,1.52337503986612,1.16076099809997,4.24667620804936 +46,3,0,1.52337503986612,1.03398841109356,8.15898555074743 +46,4,0,1.52337503986612,0.994797262587139,8.0779901676523 +46,5,0,1.52337503986612,1.1249651792548,5.68266893304037 +47,1,0,-0.483989426883416,0.574105356443331,-3.20613712871425 +47,2,0,-0.483989426883416,0.208918900884307,0.00649019809915752 +47,3,0,-0.483989426883416,0.741012271351296,0.757311787415 +47,4,0,-0.483989426883416,0.288505807018783,-0.62067301933059 +47,5,0,-0.483989426883416,0.278450260206523,0.549664430944102 +48,1,0,-0.241224470517284,0.538518235254474,0.562488211839541 +48,2,0,-0.241224470517284,0.745322403561392,-0.207881642487272 +48,3,0,-0.241224470517284,0.424900111677857,1.17191392468433 +48,4,0,-0.241224470517284,0.31091296192471,0.890812850085176 +48,5,0,-0.241224470517284,0.603363988372063,0.194681618635391 +49,1,0,-0.502923176484982,-0.845613766548061,-0.0280330536253708 +49,2,0,-0.502923176484982,-0.525656173107788,0.471650194793933 +49,3,0,-0.502923176484982,-0.27163538685412,1.77038444487592 +49,4,0,-0.502923176484982,0.748563048156958,1.06206817243644 +49,5,0,-0.502923176484982,0.862683732520299,1.87497630545043 +50,1,0,0.331707056486104,1.43310608470543,-1.03629025822675 +50,2,0,0.331707056486104,1.47412858157122,0.590271437555205 +50,3,0,0.331707056486104,0.00993825640601731,0.576019539322155 +50,4,0,0.331707056486104,1.18446092385485,1.06859243230635 +50,5,0,0.331707056486104,1.36296624104844,0.823833814241365 +51,1,0,-1.28376827728359,-0.054772706538596,-0.112182614259992 +51,2,0,-1.28376827728359,-0.647495854943373,-0.94712105801857 +51,3,0,-1.28376827728359,0.240043184837449,-0.00441638364042768 +51,4,0,-1.28376827728359,0.0606767301153908,0.875661792221071 +51,5,0,-1.28376827728359,0.379524142127041,1.22049460268803 +52,1,0,0.0268264038406972,0.0813711067876824,0.796399841450881 +52,2,0,0.0268264038406972,0.43253565739872,1.347067415838 +52,3,0,0.0268264038406972,1.09224494984207,1.34411702360282 +52,4,0,0.0268264038406972,0.775188384647136,2.98262612542836 +52,5,0,0.0268264038406972,0.413154429761364,2.67115624971039 +53,1,0,-1.88861627689539,-0.98294737042299,0.975402024308385 +53,2,0,-1.88861627689539,-0.364933260882153,-1.36155509419646 +53,3,0,-1.88861627689539,-1.28010414184404,-0.0953612032279289 +53,4,0,-1.88861627689539,-0.26285694011793,0.645459537255967 +53,5,0,-1.88861627689539,0.320342118934647,-1.34617214380709 +54,1,0,-0.758160785176121,-0.904859165411716,-1.92964585465319 +54,2,0,-0.758160785176121,0.426418257280298,-1.49702195088084 +54,3,0,-0.758160785176121,-0.172965732589018,0.573031919060233 +54,4,0,-0.758160785176121,0.501441761447204,0.60555176976647 +54,5,0,-0.758160785176121,0.748841788556021,0.386565229532384 +55,1,0,0.772657673186417,1.26108129853558,0.848326823659983 +55,2,0,0.772657673186417,0.994286367108801,0.533785646432795 +55,3,0,0.772657673186417,0.132121885819294,1.8393490008326 +55,4,0,0.772657673186417,1.4440198335426,4.31073338025935 +55,5,0,0.772657673186417,1.0421026037683,3.91593091869044 +56,1,0,0.0711352172241156,1.10908756660978,-1.62655633290739 +56,2,0,0.0711352172241156,0.34083565612227,-0.4916460672834 +56,3,0,0.0711352172241156,0.504409497242438,0.466038589769887 +56,4,0,0.0711352172241156,0.204265125308877,-0.613721113111909 +56,5,0,0.0711352172241156,1.43138380470924,0.955380118911139 +57,1,0,-0.794980135199814,-1.0945752615964,-0.419352393006358 +57,2,0,-0.794980135199814,-0.015679604949227,-2.37572615892288 +57,3,0,-0.794980135199814,0.782093277273994,-0.2996905359297 +57,4,0,-0.794980135199814,0.491006375103466,2.1447829543285 +57,5,0,-0.794980135199814,1.00082663497421,-0.477142427035938 +58,1,0,-1.02181782285825,-0.489529600000614,1.87489097842735 +58,2,0,-1.02181782285825,-0.634278175900665,-0.00496069455539061 +58,3,0,-1.02181782285825,0.0812610954568617,3.09374637476232 +58,4,0,-1.02181782285825,1.76062570141884,0.47159533276173 +58,5,0,-1.02181782285825,-0.118990402253873,2.51810462575589 +59,1,0,-0.798010454873102,0.0609546958435624,-2.36229581308972 +59,2,0,-0.798010454873102,0.454243995946293,0.0438796181667978 +59,3,0,-0.798010454873102,0.190314900617629,-1.03078925055765 +59,4,0,-0.798010454873102,0.449234077715798,-0.817729703997447 +59,5,0,-0.798010454873102,0.84368503725751,0.705517798639742 +60,1,0,-0.0306085669312452,0.452616374413926,-0.102673703149613 +60,2,0,-0.0306085669312452,0.813949076299101,-0.434575720306834 +60,3,0,-0.0306085669312452,1.4233380723612,1.29161760939601 +60,4,0,-0.0306085669312452,0.355689837861843,1.56195409402315 +60,5,0,-0.0306085669312452,0.965951055191722,0.909229841469014 +61,1,0,0.067078462093333,-1.09774855868318,0.0382140984881273 +61,2,0,0.067078462093333,-0.0734720467411606,0.173142949330493 +61,3,0,0.067078462093333,-1.19588725717607,0.359042696049106 +61,4,0,0.067078462093333,1.3856053140362,0.427910828219115 +61,5,0,0.067078462093333,1.44251835963329,3.56206647898682 +62,1,0,-1.52731617743477,-1.47707966252442,0.52878696784946 +62,2,0,-1.52731617743477,0.6725293667928,-0.713013683979517 +62,3,0,-1.52731617743477,-0.801863611756564,0.166830610735084 +62,4,0,-1.52731617743477,0.104227361969105,0.458581632509362 +62,5,0,-1.52731617743477,-0.0939253723866919,2.61021076307 +63,1,0,-0.576873399474489,0.568469415283613,-0.0885467512739461 +63,2,0,-0.576873399474489,-0.63462889673578,0.841368442972864 +63,3,0,-0.576873399474489,0.499041270194032,0.730812734245097 +63,4,0,-0.576873399474489,0.46564016095323,0.379790557893517 +63,5,0,-0.576873399474489,0.727842048483113,0.879392177634746 +64,1,0,-0.46221481649607,0.128333789084384,-0.810771268931039 +64,2,0,-0.46221481649607,0.370204400190163,-0.442277924307933 +64,3,0,-0.46221481649607,-0.0845670681314973,0.425221145845222 +64,4,0,-0.46221481649607,0.0289259292010756,1.92641758452772 +64,5,0,-0.46221481649607,0.832360426764675,1.76116093310025 +65,1,0,0.315703776453431,0.185119261555453,2.44859995246444 +65,2,0,0.315703776453431,1.132464076799,2.36264388338644 +65,3,0,0.315703776453431,0.804936552055527,4.33952964926613 +65,4,0,0.315703776453431,1.63738405965253,3.45633927678726 +65,5,0,0.315703776453431,0.654315798187792,2.29243512074624 +66,1,0,1.37658808505767,0.937250197886316,2.81324292738336 +66,2,0,1.37658808505767,1.31236937691446,1.43950882543872 +66,3,0,1.37658808505767,0.817377287895898,3.55004984527523 +66,4,0,1.37658808505767,0.697002957276908,5.44502481471952 +66,5,0,1.37658808505767,1.36246318909377,6.13221143906437 +67,1,0,1.42681697200005,0.929878026545164,1.29641812279912 +67,2,0,1.42681697200005,1.47078818640231,2.49949074805398 +67,3,0,1.42681697200005,1.39010723206942,2.28621088692472 +67,4,0,1.42681697200005,1.4994344602743,3.12588315590834 +67,5,0,1.42681697200005,2.07835117603224,5.93525294343137 +68,1,0,1.35308419041603,0.513814763134145,2.6808262083581 +68,2,0,1.35308419041603,0.71319979591278,3.11853804665789 +68,3,0,1.35308419041603,1.51616580195878,3.70939790832736 +68,4,0,1.35308419041603,0.303962610691678,5.32893930278764 +68,5,0,1.35308419041603,0.61382915360482,4.76798985640524 +69,1,0,1.79859786884555,0.714140771413365,-1.9749036071355 +69,2,0,1.79859786884555,1.75496633936581,1.15033972954931 +69,3,0,1.79859786884555,1.86609357285324,1.43700928442477 +69,4,0,1.79859786884555,1.03796845123607,4.69802297627656 +69,5,0,1.79859786884555,2.80748043992152,4.19758682907294 +70,1,0,-0.333836240206274,-0.271022425491758,1.63473212396469 +70,2,0,-0.333836240206274,1.03771957276743,2.02484454029549 +70,3,0,-0.333836240206274,-0.330129660933743,0.691713233135543 +70,4,0,-0.333836240206274,0.920392687785187,4.37086187320388 +70,5,0,-0.333836240206274,0.555092886752288,2.30949232147289 +71,1,0,-1.63895398153686,-1.18730170194834,2.09459510693227 +71,2,0,-1.63895398153686,0.0659017407261382,0.30490883430922 +71,3,0,-1.63895398153686,-0.62447901592315,0.471122244746123 +71,4,0,-1.63895398153686,-0.915053299758479,-0.865034452672139 +71,5,0,-1.63895398153686,1.14578444726516,0.986950199835889 +72,1,0,0.642543716487307,1.36176968092239,-1.81422208681334 +72,2,0,0.642543716487307,0.748030229144181,1.01388214037359 +72,3,0,0.642543716487307,0.82997265384777,0.0323834944231848 +72,4,0,0.642543716487307,0.880074784091964,1.82932376746001 +72,5,0,0.642543716487307,2.12456495392237,1.75625293742539 +73,1,0,1.8104991643694,0.841870657072021,2.72974425521786 +73,2,0,1.8104991643694,1.45889744701685,3.88174699419143 +73,3,0,1.8104991643694,1.85451418559813,2.80083656747205 +73,4,0,1.8104991643694,1.71168475350052,5.04857246953356 +73,5,0,1.8104991643694,2.09495458053712,6.84980828621333 +74,1,0,-0.932126399374389,-0.568022636104557,2.76205407969921 +74,2,0,-0.932126399374389,0.154151225212912,2.56941894156864 +74,3,0,-0.932126399374389,-0.45323497987067,0.131049516259385 +74,4,0,-0.932126399374389,0.707366959033985,2.23782040811992 +74,5,0,-0.932126399374389,1.2179979556303,3.53842169670308 +75,1,0,0.666231891524512,1.08694657400816,1.87974063555088 +75,2,0,0.666231891524512,1.27378252924364,2.53844953486072 +75,3,0,0.666231891524512,1.84808761734133,2.06146755721079 +75,4,0,0.666231891524512,1.2250029565835,4.65112423855464 +75,5,0,0.666231891524512,2.16385717182326,5.81266126562459 +76,1,0,0.341678631484337,1.06347467961005,0.883641144348533 +76,2,0,0.341678631484337,0.424007246284149,0.927200899437488 +76,3,0,0.341678631484337,0.463634111552505,1.09297461415004 +76,4,0,0.341678631484337,0.883946377784091,1.35138421699445 +76,5,0,0.341678631484337,0.846805066898023,4.35549048057971 +77,1,0,0.637620706206215,1.13509768737945,1.43068721208062 +77,2,0,0.637620706206215,0.119022913744544,4.02884996915181 +77,3,0,0.637620706206215,1.78409687300482,5.37165267440474 +77,4,0,0.637620706206215,0.572587375028069,4.54884339641776 +77,5,0,0.637620706206215,1.30157744988052,4.66081396519702 +78,1,0,0.916740129944944,0.527388555742362,1.67885626210288 +78,2,0,0.916740129944944,1.6516624227088,2.88404956264771 +78,3,0,0.916740129944944,1.57621531218402,5.17729353626615 +78,4,0,0.916740129944944,1.12169573790953,6.39677493288079 +78,5,0,0.916740129944944,1.56442333890756,3.51684585995395 +79,1,0,0.678393345349387,0.981084168977522,-0.303125990400053 +79,2,0,0.678393345349387,0.00845983067858702,3.77063304832269 +79,3,0,0.678393345349387,0.985461654447826,1.28792599622788 +79,4,0,0.678393345349387,1.37335705414109,3.93901911601086 +79,5,0,0.678393345349387,1.59918286286575,4.20477704101249 +80,1,0,-0.614505923587853,-0.074124901025198,0.557946043511742 +80,2,0,-0.614505923587853,0.0775885147622684,3.20589794298296 +80,3,0,-0.614505923587853,-0.129964058314692,1.43190426473233 +80,4,0,-0.614505923587853,1.12112075034697,3.91574997314387 +80,5,0,-0.614505923587853,0.349774794978119,2.92434398626071 +81,1,0,-0.827776110905095,0.0779920496627954,2.20724033209597 +81,2,0,-0.827776110905095,0.329874381465543,-0.343896532607536 +81,3,0,-0.827776110905095,-0.509582604067608,0.184330306231341 +81,4,0,-0.827776110905095,0.425631993327435,-0.677124424118725 +81,5,0,-0.827776110905095,0.576793045090816,5.25417091189951 +82,1,0,0.861142171302151,0.534905942272975,-0.559829415854437 +82,2,0,0.861142171302151,0.421001725699289,-0.640432437832771 +82,3,0,0.861142171302151,1.33334862780767,1.15221010623295 +82,4,0,0.861142171302151,1.51902787603164,1.84609545714379 +82,5,0,0.861142171302151,2.27137340338279,1.20605568714721 +83,1,0,0.616000967810865,1.56647356845175,0.856810537009867 +83,2,0,0.616000967810865,0.418959285740418,3.0293435997056 +83,3,0,0.616000967810865,1.71178089113956,4.1807485972266 +83,4,0,0.616000967810865,1.75291581545166,3.61295843048961 +83,5,0,0.616000967810865,1.95198829511699,5.40410230770897 +84,1,0,-0.364563738233122,0.729912191174741,-1.54643580105865 +84,2,0,-0.364563738233122,0.270810379458042,0.662855640387282 +84,3,0,-0.364563738233122,0.46985768052808,3.08624127314465 +84,4,0,-0.364563738233122,0.471373212800752,0.515624799993429 +84,5,0,-0.364563738233122,0.781470542382374,1.70977184890727 +85,1,0,0.143653481491035,0.940535155473577,-0.0595363250531367 +85,2,0,0.143653481491035,-0.338084292631134,-0.805588718694135 +85,3,0,0.143653481491035,1.27753543499624,0.00832597427848673 +85,4,0,0.143653481491035,0.89192092464685,2.03135738913338 +85,5,0,0.143653481491035,0.703431750258417,0.599027040421859 +86,1,0,-0.131220650539475,-0.0817446182852218,-0.396935575601526 +86,2,0,-0.131220650539475,0.191895364526733,-0.194650738734955 +86,3,0,-0.131220650539475,0.788377198501869,3.73853028280585 +86,4,0,-0.131220650539475,1.02415048291942,1.91396930977068 +86,5,0,-0.131220650539475,1.03878595672618,2.1859552336763 +87,1,0,-0.333066256749381,1.06674969434541,-0.0657657799846833 +87,2,0,-0.333066256749381,0.149647369032651,0.498161299881925 +87,3,0,-0.333066256749381,0.581926896494485,1.13646345708566 +87,4,0,-0.333066256749381,-0.247133860668094,-0.175924284009217 +87,5,0,-0.333066256749381,1.46732523994061,1.91561016686449 +88,1,0,-0.434370693446027,-0.437377299668023,-0.772574889112886 +88,2,0,-0.434370693446027,-0.152739483160924,-0.568834744275269 +88,3,0,-0.434370693446027,0.951913298045205,0.385469624381133 +88,4,0,-0.434370693446027,1.1189528375249,-0.057927566466312 +88,5,0,-0.434370693446027,0.568042594475733,0.443220743369422 +89,1,0,-0.141474869850029,0.0801484231476475,0.0270269788568515 +89,2,0,-0.141474869850029,0.0481236129066036,0.205532603073162 +89,3,0,-0.141474869850029,0.995260603221087,0.651653657935192 +89,4,0,-0.141474869850029,1.32639060956188,1.59469437463446 +89,5,0,-0.141474869850029,0.645906816771665,2.15442315953645 +90,1,0,1.03227036316416,0.63763451526355,-1.25422979344628 +90,2,0,1.03227036316416,0.405889210017968,-0.110219289419371 +90,3,0,1.03227036316416,0.458014774582056,0.982330971708697 +90,4,0,1.03227036316416,0.852538382878488,3.76935892335802 +90,5,0,1.03227036316416,0.962265711370071,2.56238146466164 +91,1,0,-0.486491594070801,-0.172045064925041,1.83038665225263 +91,2,0,-0.486491594070801,0.384618723183606,0.0474591275790941 +91,3,0,-0.486491594070801,0.531878509700311,1.7092334560639 +91,4,0,-0.486491594070801,0.793638805747236,2.5679967220646 +91,5,0,-0.486491594070801,0.722297893770697,3.12470764883272 +92,1,0,0.219980494850729,0.0734058917248482,0.384234866874619 +92,2,0,0.219980494850729,-0.219734183232657,0.903091028630454 +92,3,0,0.219980494850729,0.682596952495566,2.12751213821996 +92,4,0,0.219980494850729,1.23869541320448,3.5612634595244 +92,5,0,0.219980494850729,0.578532061197268,3.27222425245696 +93,1,0,0.288608812761447,0.421231189913603,-1.40212126529636 +93,2,0,0.288608812761447,0.454485182510659,0.422698666792128 +93,3,0,0.288608812761447,1.31865477532447,0.115353686002623 +93,4,0,0.288608812761447,1.08220992063066,2.67936764198457 +93,5,0,0.288608812761447,0.959458903140813,2.13821853667061 +94,1,0,-1.61902954772177,-1.20480995091084,1.82409496936011 +94,2,0,-1.61902954772177,-0.930824167709811,1.42101239978627 +94,3,0,-1.61902954772177,-0.584839870198029,1.78332374092068 +94,4,0,-1.61902954772177,-0.346404263307608,-0.227553617442285 +94,5,0,-1.61902954772177,1.1729349153929,1.9498809084647 +95,1,0,0.222271128189547,-0.0387473417629938,0.858725090646077 +95,2,0,0.222271128189547,0.767954290471442,2.69923913505218 +95,3,0,0.222271128189547,0.605075552915976,1.99193702670537 +95,4,0,0.222271128189547,1.3558238746813,2.73108721331929 +95,5,0,0.222271128189547,1.63139262575401,0.783913778717177 +96,1,0,1.54600822113029,2.02001061311009,2.62936331979604 +96,2,0,1.54600822113029,1.35702096802835,1.96438966667089 +96,3,0,1.54600822113029,1.36437355018916,1.82028484401593 +96,4,0,1.54600822113029,1.54806112741881,2.22526558385425 +96,5,0,1.54600822113029,2.88653595933768,5.29452516245512 +97,1,0,0.133697246677124,-0.0718599103232635,1.93759954133085 +97,2,0,0.133697246677124,0.243856103638007,1.83522434020185 +97,3,0,0.133697246677124,0.138526367764456,4.6449042058556 +97,4,0,0.133697246677124,1.8769167656904,3.54647214833668 +97,5,0,0.133697246677124,0.962401521139283,2.9247955599588 +98,1,0,0.513461762623951,0.550111450173374,1.13638082120044 +98,2,0,0.513461762623951,-0.116902446416303,2.0047933773136 +98,3,0,0.513461762623951,1.39013026554457,2.89151767905503 +98,4,0,0.513461762623951,1.5481840147936,2.39962626326609 +98,5,0,0.513461762623951,0.988855636857315,2.77565780113198 +99,1,0,0.0284391113388029,0.689085116598258,0.254359114140023 +99,2,0,0.0284391113388029,0.349968288749272,0.878606062697145 +99,3,0,0.0284391113388029,1.81169809337691,0.440914916417645 +99,4,0,0.0284391113388029,0.983652616962293,2.17316792107343 +99,5,0,0.0284391113388029,1.20691015927357,3.06483425373133 +100,1,0,-1.7202950012414,-0.115879309040799,-1.39743104846185 +100,2,0,-1.7202950012414,0.0489770680107625,-1.04291605039209 +100,3,0,-1.7202950012414,-1.17871267878631,-0.81502457454413 +100,4,0,-1.7202950012414,-0.105520702288695,0.334070265764476 +100,5,0,-1.7202950012414,-0.467384608422146,-0.890554303485205 +101,1,3,-0.657717766276882,-0.94206308645325,0.232488558205459 +101,2,3,-0.657717766276882,0.0705885988157602,0.953684678547186 +101,3,3,-0.657717766276882,0.0953022991421917,4.99112586140753 +101,4,3,-0.657717766276882,0.271992657738819,3.73290731784579 +101,5,3,-0.657717766276882,0.170440318698168,5.85181562906916 +102,1,3,0.606628948683484,-0.13131505600894,2.91452541834592 +102,2,3,0.606628948683484,1.4022497729769,0.261042740854546 +102,3,3,0.606628948683484,0.993231830563238,2.71853186863986 +102,4,3,0.606628948683484,0.884332783542463,4.56588455342279 +102,5,3,0.606628948683484,1.56713473862668,7.09247867741218 +103,1,3,-0.332641963407837,0.154534274202655,0.602279850487755 +103,2,3,-0.332641963407837,0.072432790726706,-0.309540706057066 +103,3,3,-0.332641963407837,1.16818496470408,-0.165286394085044 +103,4,3,-0.332641963407837,0.702511034762439,1.34374356615043 +103,5,3,-0.332641963407837,1.01447715390258,5.3160097148392 +104,1,3,0.469706705466534,0.279153249372299,-0.383474155857604 +104,2,3,0.469706705466534,1.26056420037322,0.450636566145736 +104,3,3,0.469706705466534,0.840143863086957,3.06714331395692 +104,4,3,0.469706705466534,1.5062897407947,5.65655038983017 +104,5,3,0.469706705466534,1.1964368519802,5.11830997961023 +105,1,3,1.66748637673741,1.12973770628659,-1.99633809999194 +105,2,3,1.66748637673741,1.71587553353966,-1.18150760955215 +105,3,3,1.66748637673741,1.51301509979169,3.13175716039968 +105,4,3,1.66748637673741,1.52614573553539,5.51260657016865 +105,5,3,1.66748637673741,1.80360251228863,6.86533385915753 +106,1,3,1.27268901841241,0.410841768659794,0.678968405380034 +106,2,3,1.27268901841241,2.42608384357659,0.996016318495919 +106,3,3,1.27268901841241,1.75308326994184,2.48578861598163 +106,4,3,1.27268901841241,1.50527027515069,6.60521083807514 +106,5,3,1.27268901841241,0.851577609230435,9.15879471695537 +107,1,3,-0.113521083630058,-0.617931847269183,0.389795760489174 +107,2,3,-0.113521083630058,0.00578001996492189,1.24529091081873 +107,3,3,-0.113521083630058,0.522769231917015,3.59142446549231 +107,4,3,-0.113521083630058,0.55896664892714,4.08609378496072 +107,5,3,-0.113521083630058,0.624181425631188,6.65868685406829 +108,1,3,1.42915725643045,1.03946831567814,-1.49277112713376 +108,2,3,1.42915725643045,0.355936899218436,-0.993582726915977 +108,3,3,1.42915725643045,1.3102549176241,0.706937887500567 +108,4,3,1.42915725643045,1.91255925354156,2.4057349385667 +108,5,3,1.42915725643045,1.11706892387658,5.42943529075469 +109,1,3,-0.559679157783233,0.240146108567419,1.85721232954093 +109,2,3,-0.559679157783233,-1.18164851474772,0.598496577056571 +109,3,3,-0.559679157783233,-0.567714929049523,3.8190582793659 +109,4,3,-0.559679157783233,1.58499831755575,3.61974345956697 +109,5,3,-0.559679157783233,0.281509011231452,5.64271736926589 +110,1,3,0.187728838960483,-0.0990282930604425,0.636491412067755 +110,2,3,0.187728838960483,0.522934521191726,-1.27512878217836 +110,3,3,0.187728838960483,0.451864219784705,0.505624681678577 +110,4,3,0.187728838960483,0.778209878446357,2.64560002072363 +110,5,3,0.187728838960483,1.85942863847523,5.79668804951045 +111,1,3,1.68213095275821,0.251934065681341,3.91605951374998 +111,2,3,1.68213095275821,1.35011115331486,3.79188977773365 +111,3,3,1.68213095275821,1.97032026344658,3.08046167492404 +111,4,3,1.68213095275821,2.2589275661647,8.27001943564112 +111,5,3,1.68213095275821,1.81228175371279,7.85436781063261 +112,1,3,-0.0788326245762765,-0.0886876014313262,0.741167712744016 +112,2,3,-0.0788326245762765,0.057345760642664,-0.289974470099434 +112,3,3,-0.0788326245762765,0.217463034910929,1.05732757506079 +112,4,3,-0.0788326245762765,1.61994832884064,5.16748675022915 +112,5,3,-0.0788326245762765,1.10512689283478,6.32704551373198 +113,1,3,-1.11820261456398,-0.326553621508285,4.59619133661404 +113,2,3,-1.11820261456398,0.138424198638174,3.08762952083826 +113,3,3,-1.11820261456398,0.580218918516405,4.11792205698977 +113,4,3,-1.11820261456398,0.180859352511177,5.12177393132146 +113,5,3,-1.11820261456398,0.466131001972245,6.08972404155704 +114,1,3,0.00617399378076931,0.655178842061438,-2.62808216419414 +114,2,3,0.00617399378076931,1.54562520211198,-0.333313638255983 +114,3,3,0.00617399378076931,-0.0719855219840838,0.0862359331011819 +114,4,3,0.00617399378076931,0.444363543616229,2.84552412987694 +114,5,3,0.00617399378076931,0.531577221298498,3.34103605859625 +115,1,3,-0.466490512919715,-0.70628149463559,-1.45817530327387 +115,2,3,-0.466490512919715,0.729198541085584,-1.93903353127595 +115,3,3,-0.466490512919715,0.543408386695873,-0.351268598390575 +115,4,3,-0.466490512919715,1.02986330956025,1.1754453578064 +115,5,3,-0.466490512919715,0.412796845409179,3.51229890894027 +116,1,3,-2.26408332836368,-1.30773888985853,-2.10077614733638 +116,2,3,-2.26408332836368,-1.37763784721948,0.236161311201754 +116,3,3,-2.26408332836368,-1.7425007121992,1.23135889576542 +116,4,3,-2.26408332836368,-0.708621021967853,1.87035942321069 +116,5,3,-2.26408332836368,-1.03635888476242,-0.645611331548352 +117,1,3,0.552125571623184,1.32365285117336,0.779376482838142 +117,2,3,0.552125571623184,0.930668266663576,3.04427235955234 +117,3,3,0.552125571623184,0.622006301346775,3.68654705941641 +117,4,3,0.552125571623184,1.77537177199262,5.84936566524921 +117,5,3,0.552125571623184,1.4463562521855,7.26081776390063 +118,1,3,-0.294573500984481,-0.226854558548949,0.624381599018543 +118,2,3,-0.294573500984481,0.451975050319793,-0.50960478842452 +118,3,3,-0.294573500984481,1.10372122526393,1.56675527682092 +118,4,3,-0.294573500984481,0.820290560391889,3.70060353900024 +118,5,3,-0.294573500984481,0.0496382963428972,4.85596894365002 +119,1,3,-0.723593527799914,-0.505152373633315,2.0676374826663 +119,2,3,-0.723593527799914,0.101750802662789,1.03136110282366 +119,3,3,-0.723593527799914,0.730559754677908,3.41623122468754 +119,4,3,-0.723593527799914,0.00221535064169642,3.78216320855324 +119,5,3,-0.723593527799914,0.636998000740745,4.93833031585068 +120,1,3,0.858310574796443,1.08316871624732,5.23943573915105 +120,2,3,0.858310574796443,0.863887874687908,5.35534446922412 +120,3,3,0.858310574796443,2.11534837934243,6.58543264119611 +120,4,3,0.858310574796443,1.88024199878906,9.8528988127391 +120,5,3,0.858310574796443,1.27582322704727,11.2611305249796 +121,1,3,0.220462861432781,-0.309440648082669,-0.755306621580593 +121,2,3,0.220462861432781,0.627055418074356,-0.023684948685408 +121,3,3,0.220462861432781,0.545149725436819,1.36183809596365 +121,4,3,0.220462861432781,0.792007240646273,3.18699848795418 +121,5,3,0.220462861432781,1.69726445585479,3.95436649310404 +122,1,3,-0.140752859229319,0.177062299482488,-1.76683361804301 +122,2,3,-0.140752859229319,0.304636966807162,0.993079818624433 +122,3,3,-0.140752859229319,0.421561519180845,0.332450286328496 +122,4,3,-0.140752859229319,1.36949693276948,3.85954344361671 +122,5,3,-0.140752859229319,0.568440407673914,3.85447057933448 +123,1,3,1.08154108301988,0.535266128351368,0.779969396251111 +123,2,3,1.08154108301988,0.880289324345213,1.59969760053008 +123,3,3,1.08154108301988,0.756275612370415,4.49990130269742 +123,4,3,1.08154108301988,1.05475695661148,5.66132166010569 +123,5,3,1.08154108301988,0.991482360834123,7.76732143736291 +124,1,3,0.000773525396488833,0.462844594613568,0.841288741599271 +124,2,3,0.000773525396488833,0.602602082712852,0.312361825429711 +124,3,3,0.000773525396488833,0.339182135812421,2.46117172121027 +124,4,3,0.000773525396488833,0.560962815672359,3.19918261277187 +124,5,3,0.000773525396488833,0.445130224054067,6.56037285845182 +125,1,3,-0.212450189809644,0.948086214476399,3.10996825066564 +125,2,3,-0.212450189809644,0.318713140332635,3.16644442473342 +125,3,3,-0.212450189809644,0.680601781620023,3.20618608330843 +125,4,3,-0.212450189809644,1.62789839447148,4.76744325623378 +125,5,3,-0.212450189809644,1.3134135459942,5.12920413147696 +126,1,3,-0.0749218641308017,-0.250989637603912,-0.34160343652017 +126,2,3,-0.0749218641308017,-0.107579672268752,0.194139114567561 +126,3,3,-0.0749218641308017,0.93589473033804,3.58262564440672 +126,4,3,-0.0749218641308017,0.683688042722152,1.72966565599836 +126,5,3,-0.0749218641308017,1.71491065530947,4.19726792441863 +127,1,3,0.627289566110117,1.05729793945288,1.62128596144571 +127,2,3,0.627289566110117,0.973758617212842,2.34129685541675 +127,3,3,0.627289566110117,0.828958721907215,3.88047996241525 +127,4,3,0.627289566110117,1.54879032336665,5.42297645981372 +127,5,3,0.627289566110117,1.48328874793323,8.63035389347065 +128,1,3,0.410545949407018,0.625141759106629,1.94363260911011 +128,2,3,0.410545949407018,0.840869756849677,2.63277909520217 +128,3,3,0.410545949407018,1.69458209211863,6.08657628694377 +128,4,3,0.410545949407018,0.638430284893267,6.45369861408724 +128,5,3,0.410545949407018,2.5733341846153,7.72778285671548 +129,1,3,1.33028430702108,2.13073984612338,1.69376984087629 +129,2,3,1.33028430702108,0.646345800359394,3.21444010789856 +129,3,3,1.33028430702108,2.04773265337806,3.61285827791554 +129,4,3,1.33028430702108,1.67761633423117,5.69764116082381 +129,5,3,1.33028430702108,0.912577561277432,9.44538478230292 +130,1,3,-1.30395806956874,0.140532026865759,-0.905148818564164 +130,2,3,-1.30395806956874,0.1334871419863,0.394705286548389 +130,3,3,-1.30395806956874,0.0860359670190863,3.32062284302592 +130,4,3,-1.30395806956874,0.30340861859709,3.28255261756849 +130,5,3,-1.30395806956874,0.658912821802418,2.56331948055225 +131,1,3,-0.357820487255056,0.604743403188498,-0.951886500117099 +131,2,3,-0.357820487255056,0.647902518744151,-0.167819315846137 +131,3,3,-0.357820487255056,1.10059087835971,0.101181804228307 +131,4,3,-0.357820487255056,0.308431637907341,3.75349531284482 +131,5,3,-0.357820487255056,0.718013535203741,2.89114217148319 +132,1,3,0.48766605796259,0.0140713891864764,-2.21162281394707 +132,2,3,0.48766605796259,0.156165054524314,-0.679269124727664 +132,3,3,0.48766605796259,0.732944740978031,2.81641237256502 +132,4,3,0.48766605796259,1.28017422519403,2.46764691181743 +132,5,3,0.48766605796259,0.938788379132021,4.56142902922779 +133,1,3,0.0965095252581635,-0.424257311714683,-0.906138381171024 +133,2,3,0.0965095252581635,0.682038652664297,0.467549662669626 +133,3,3,0.0965095252581635,0.599960579187045,1.56808656793963 +133,4,3,0.0965095252581635,0.670053117047675,5.99640467483164 +133,5,3,0.0965095252581635,0.00290729768220532,5.61187286084779 +134,1,3,0.470813281504669,-0.0237977219345792,-1.17861719509669 +134,2,3,0.470813281504669,0.0210231463695877,2.26535855456937 +134,3,3,0.470813281504669,1.65875503097192,2.79972362019488 +134,4,3,0.470813281504669,1.4522899241199,6.96850396868235 +134,5,3,0.470813281504669,1.19697537953027,5.53220254058808 +135,1,3,1.67155479620156,1.63003289483879,1.13465670732127 +135,2,3,1.67155479620156,1.66566175054474,2.16928327099793 +135,3,3,1.67155479620156,1.23104108297534,4.62609919171196 +135,4,3,1.67155479620156,2.51547061628803,6.42366307314904 +135,5,3,1.67155479620156,0.773175668804269,6.06302343062228 +136,1,3,-0.0332394453434365,0.0760360749606722,0.0208825919783583 +136,2,3,-0.0332394453434365,0.758090599491503,0.568066216720766 +136,3,3,-0.0332394453434365,0.865406851245269,2.16385713696812 +136,4,3,-0.0332394453434365,0.287846254413097,4.10855697907143 +136,5,3,-0.0332394453434365,0.735172436002937,4.53398714072422 +137,1,3,-0.688225331751058,0.0016693549665899,-0.605796558059025 +137,2,3,-0.688225331751058,0.31564905198807,2.09364296940547 +137,3,3,-0.688225331751058,0.302456958121517,2.91353634798101 +137,4,3,-0.688225331751058,1.4601427357091,3.55511507145684 +137,5,3,-0.688225331751058,0.52377242178836,6.28678207632846 +138,1,3,-0.500122627493091,0.396683358364411,1.06354027076539 +138,2,3,-0.500122627493091,0.147266283149743,-0.416532302563743 +138,3,3,-0.500122627493091,-0.0674183543222071,2.39971802018502 +138,4,3,-0.500122627493091,0.444002742154018,4.72359154487664 +138,5,3,-0.500122627493091,-0.00639513954455206,6.2797345135803 +139,1,3,0.18107731763868,0.358754646027512,2.56815917737501 +139,2,3,0.18107731763868,0.473843315161791,5.09625792076298 +139,3,3,0.18107731763868,0.222826468427173,5.09301261569359 +139,4,3,0.18107731763868,0.998960955021969,5.19457217305691 +139,5,3,0.18107731763868,0.94396796417614,7.6238515902996 +140,1,3,-1.37211800780397,-0.397986200765386,-0.486212418062664 +140,2,3,-1.37211800780397,-1.14777684605006,-0.587349940190411 +140,3,3,-1.37211800780397,0.207813697049092,0.0179085891028778 +140,4,3,-1.37211800780397,-0.0157145368531961,0.718055885197066 +140,5,3,-1.37211800780397,0.141532328048656,3.84922898075712 +141,1,3,-1.19897389691436,0.360461248314332,-1.8152035620548 +141,2,3,-1.19897389691436,-0.542266936350596,-2.95909668834627 +141,3,3,-1.19897389691436,0.99913729191338,0.895701803570273 +141,4,3,-1.19897389691436,0.397516705219247,1.30946775962812 +141,5,3,-1.19897389691436,0.557238702745939,2.26160192220899 +142,1,3,-0.998319924353911,-0.617579580251617,-1.74650801638042 +142,2,3,-0.998319924353911,-0.949218896378564,-0.530887740124697 +142,3,3,-0.998319924353911,0.039191340332905,0.0526537324003162 +142,4,3,-0.998319924353911,1.06166107117482,1.17556748272577 +142,5,3,-0.998319924353911,0.388788578221689,3.55709306215575 +143,1,3,1.20119683314219,0.774166948615702,1.59415308736676 +143,2,3,1.20119683314219,1.60847913451569,1.16604946065365 +143,3,3,1.20119683314219,1.79639042370985,3.97103994157813 +143,4,3,1.20119683314219,0.749734928571424,4.99089995233893 +143,5,3,1.20119683314219,1.4255599566404,9.34890063805875 +144,1,3,1.00628694430582,1.08167301522945,-0.555804553785998 +144,2,3,1.00628694430582,1.28955773107959,-0.794982544165143 +144,3,3,1.00628694430582,0.531935299948853,2.6417560919309 +144,4,3,1.00628694430582,1.96009703859714,3.43572503800884 +144,5,3,1.00628694430582,1.20637425198866,4.34655255306734 +145,1,3,1.67091120380766,1.25583677189642,1.88484533573143 +145,2,3,1.67091120380766,0.697055921239634,0.705532024610885 +145,3,3,1.67091120380766,1.58825290389066,3.24141239581406 +145,4,3,1.67091120380766,1.83616400511528,6.05043141490311 +145,5,3,1.67091120380766,1.28185088085329,5.02523739500611 +146,1,3,-0.379316649007007,-0.0834809234825081,-0.604888670214619 +146,2,3,-0.379316649007007,-0.795085926968904,-0.439042960407943 +146,3,3,-0.379316649007007,0.401301223844121,0.614602642472997 +146,4,3,-0.379316649007007,0.476915753436164,2.40152270032433 +146,5,3,-0.379316649007007,0.130152214243919,3.90321730111349 +147,1,3,0.420754458915134,0.466207147814328,-1.67292060140418 +147,2,3,0.420754458915134,0.569134175687715,-0.828083532691714 +147,3,3,0.420754458915134,0.883067633049473,0.919896016564556 +147,4,3,0.420754458915134,0.531632319632921,3.08518200522015 +147,5,3,0.420754458915134,0.635630282770971,4.59162567793378 +148,1,3,0.0311732664506885,-0.0978878456004359,0.293726126730062 +148,2,3,0.0311732664506885,0.565589374149626,0.569298807450127 +148,3,3,0.0311732664506885,0.778310646490644,1.69791627492246 +148,4,3,0.0311732664506885,0.962303021614927,2.33824377902591 +148,5,3,0.0311732664506885,1.19136094560224,4.05174848811233 +149,1,3,0.512760193053312,0.0915936923166048,0.00633837792554687 +149,2,3,0.512760193053312,0.581983898348367,-0.436480189053884 +149,3,3,0.512760193053312,0.0634121352834593,2.40477462650618 +149,4,3,0.512760193053312,0.569371944296203,4.32343142524583 +149,5,3,0.512760193053312,1.21441640506989,4.73370495644068 +150,1,3,1.87868646637167,0.69125924639008,3.38649600786785 +150,2,3,1.87868646637167,1.31653504546201,3.27059369523601 +150,3,3,1.87868646637167,1.02569583379149,6.39971209529027 +150,4,3,1.87868646637167,1.44661621609673,6.51984488748056 +150,5,3,1.87868646637167,2.02528698187357,9.61899939982145 +151,1,3,-1.84278530939641,-0.644029715762125,-2.22769544227137 +151,2,3,-1.84278530939641,-0.441329046218621,-1.45924645134561 +151,3,3,-1.84278530939641,-0.145124481339132,-1.15662859328522 +151,4,3,-1.84278530939641,-0.168275634333566,-1.17671356983013 +151,5,3,-1.84278530939641,1.1584080631215,-0.828004130089602 +152,1,3,1.72965756391273,0.255517813411765,1.53812815985889 +152,2,3,1.72965756391273,2.03646206475824,3.02640293459076 +152,3,3,1.72965756391273,1.96685280001739,5.0807862101301 +152,4,3,1.72965756391273,1.78778264470661,7.07323562820844 +152,5,3,1.72965756391273,2.63307633311382,9.20377058468654 +153,1,3,-0.29666031962568,-0.286682880554559,-0.905739111548258 +153,2,3,-0.29666031962568,0.0730518169668248,1.83539397771319 +153,3,3,-0.29666031962568,1.05754267911064,2.24979895590196 +153,4,3,-0.29666031962568,0.543774543760072,4.15543805256443 +153,5,3,-0.29666031962568,1.52983625197896,4.26308726987872 +154,1,3,-0.629413689190318,0.727509671424609,3.71401857261263 +154,2,3,-0.629413689190318,1.29760725996956,1.12434826453582 +154,3,3,-0.629413689190318,0.932495976740686,3.01651148831642 +154,4,3,-0.629413689190318,0.864584440262066,3.13699893765196 +154,5,3,-0.629413689190318,0.712000503501498,5.4012578917997 +155,1,3,1.26483805731984,0.101373228065778,0.373539829402359 +155,2,3,1.26483805731984,0.212809068946509,1.58504815689564 +155,3,3,1.26483805731984,0.99358154798726,2.71644229480947 +155,4,3,1.26483805731984,2.25517833481268,4.34058665417517 +155,5,3,1.26483805731984,1.97118903746078,7.74877923012855 +156,1,3,-0.883753614415126,-0.399535191884741,0.0960186166312004 +156,2,3,-0.883753614415126,-0.320901569906869,0.244002366570801 +156,3,3,-0.883753614415126,1.06156750938429,2.24533452927691 +156,4,3,-0.883753614415126,0.461016513134735,1.85726192341166 +156,5,3,-0.883753614415126,0.510702258754348,4.97094755371067 +157,1,3,-0.920994725090723,-0.892479349459743,-0.998800959364927 +157,2,3,-0.920994725090723,-1.15880706003958,0.367783265190597 +157,3,3,-0.920994725090723,-0.34037148362012,1.01156615038342 +157,4,3,-0.920994725090723,0.835538390428648,2.31233387560194 +157,5,3,-0.920994725090723,-0.25552330880026,3.01427008481776 +158,1,3,-0.34893368979522,-0.267555864749471,0.391980485124665 +158,2,3,-0.34893368979522,0.631473827249395,0.118968890864104 +158,3,3,-0.34893368979522,0.401379529995111,2.00487956507939 +158,4,3,-0.34893368979522,0.253640013407772,2.70578554698554 +158,5,3,-0.34893368979522,0.824772396694025,5.63766780336331 +159,1,3,-0.304255532851446,-0.675035468671625,0.975756093026101 +159,2,3,-0.304255532851446,0.0759771386422023,1.40147646679613 +159,3,3,-0.304255532851446,0.230792842617687,1.65368279369012 +159,4,3,-0.304255532851446,0.811272643636225,2.10771822600086 +159,5,3,-0.304255532851446,1.28756246587767,6.54277097690431 +160,1,3,-1.01219093638919,0.127507399755032,-2.35564452433788 +160,2,3,-1.01219093638919,0.268959034351851,-1.51029850831283 +160,3,3,-1.01219093638919,-0.407224246647464,-0.822751398045535 +160,4,3,-1.01219093638919,-0.0518131717045414,2.25240396176116 +160,5,3,-1.01219093638919,0.605259863963078,1.95588826400922 +161,1,3,-0.757717772717969,0.605568531426284,1.07474760513235 +161,2,3,-0.757717772717969,0.0731360455976896,-0.615467370613254 +161,3,3,-0.757717772717969,0.995768606560662,1.33512778948212 +161,4,3,-0.757717772717969,-0.542214426349538,3.73414229763899 +161,5,3,-0.757717772717969,0.376089632978491,4.32273833332243 +162,1,3,-0.633058606184867,0.444474326199027,-0.157864196813987 +162,2,3,-0.633058606184867,-0.358409061943328,1.36588036876563 +162,3,3,-0.633058606184867,0.00717898377558673,1.5208683399469 +162,4,3,-0.633058606184867,0.928888472555571,3.91583595246959 +162,5,3,-0.633058606184867,0.573737524170066,3.11203053101271 +163,1,3,-1.18786324515284,-0.340443408233712,1.14317643363866 +163,2,3,-1.18786324515284,-0.768032906626293,1.95833915002506 +163,3,3,-1.18786324515284,0.272259951460627,0.114323384349211 +163,4,3,-1.18786324515284,-0.298978178385548,0.61384379102693 +163,5,3,-1.18786324515284,0.637413178921166,5.50132822693627 +164,1,3,0.899797869140011,1.05596549859405,-0.21122143226474 +164,2,3,0.899797869140011,2.17406704550966,1.24046961464129 +164,3,3,0.899797869140011,1.44083698995944,2.66189961002648 +164,4,3,0.899797869140011,0.915916560712705,2.36672985820162 +164,5,3,0.899797869140011,1.38588314168802,5.02702750512009 +165,1,3,-0.301129927044137,0.752158340030402,-1.50907794920551 +165,2,3,-0.301129927044137,1.06436019651387,-0.62448442577378 +165,3,3,-0.301129927044137,0.384960393290526,0.832916152884812 +165,4,3,-0.301129927044137,0.949586844959261,1.88928233188054 +165,5,3,-0.301129927044137,0.943744899483462,2.01036225355556 +166,1,3,-0.549765513879748,0.651497636134973,0.935561103491787 +166,2,3,-0.549765513879748,-0.129357145730795,2.34756202467047 +166,3,3,-0.549765513879748,0.637365006242933,2.98981842673778 +166,4,3,-0.549765513879748,0.966660731480892,5.43125892856486 +166,5,3,-0.549765513879748,0.133694598822538,5.07759223972584 +167,1,3,-0.78547554309208,-0.134836724965572,-1.36799223380912 +167,2,3,-0.78547554309208,0.184023568082734,-0.0674588789252142 +167,3,3,-0.78547554309208,0.221814204202511,2.78941494261896 +167,4,3,-0.78547554309208,-0.0907598902618565,1.28630104303229 +167,5,3,-0.78547554309208,-0.490928636693926,3.28879673375415 +168,1,3,-1.16481614736776,-1.05549899084636,2.30664287448066 +168,2,3,-1.16481614736776,-0.109049141845168,0.403074535883857 +168,3,3,-1.16481614736776,0.408996655515945,2.96852326627862 +168,4,3,-1.16481614736776,1.13286892015195,3.26151734891858 +168,5,3,-1.16481614736776,0.25309773918563,4.60609542882273 +169,1,3,-0.100473245130853,-0.286277055576258,0.816176439613865 +169,2,3,-0.100473245130853,0.148238163475768,1.89100786414498 +169,3,3,-0.100473245130853,1.03809923428597,3.45061203241681 +169,4,3,-0.100473245130853,0.439631515666702,4.97362624547598 +169,5,3,-0.100473245130853,0.676649197592102,6.785372217849 +170,1,3,1.52370285962906,0.928353518230501,-1.74430422590474 +170,2,3,1.52370285962906,0.857441727058135,-1.77069380443786 +170,3,3,1.52370285962906,1.95225078764784,3.5723941518199 +170,4,3,1.52370285962906,0.174810994139092,3.22889090694693 +170,5,3,1.52370285962906,1.45240210098787,5.61292746676696 +171,1,3,1.70021766980079,0.845187574622548,2.88850505078726 +171,2,3,1.70021766980079,1.60306723055171,3.48529952975244 +171,3,3,1.70021766980079,2.10398199671952,2.99813281338445 +171,4,3,1.70021766980079,1.73711534632248,5.2631280122806 +171,5,3,1.70021766980079,0.948911760532366,8.4889015355624 +172,1,3,-0.382449184151253,0.601160041563296,-0.218836631756927 +172,2,3,-0.382449184151253,0.594478731981541,0.107666949516889 +172,3,3,-0.382449184151253,0.131925459706348,1.22736418934748 +172,4,3,-0.382449184151253,0.27205811111869,4.22513044903411 +172,5,3,-0.382449184151253,0.481932581280621,4.48764564425334 +173,1,3,-0.321864791232853,-0.526916540913429,-0.919112507738525 +173,2,3,-0.321864791232853,0.82545643954509,0.575624239613491 +173,3,3,-0.321864791232853,-0.471689102491135,1.89058232409286 +173,4,3,-0.321864791232853,1.13588994941228,3.41459357670557 +173,5,3,-0.321864791232853,0.36386567246704,5.03955541787637 +174,1,3,-0.976473495882479,0.711165565116071,0.282395370482445 +174,2,3,-0.976473495882479,-0.173011423957099,-0.569514663383517 +174,3,3,-0.976473495882479,-0.518998534076677,1.38191187101233 +174,4,3,-0.976473495882479,-0.107407599484349,2.48935378518072 +174,5,3,-0.976473495882479,-0.222395331439423,4.14571722953055 +175,1,3,0.206596723904414,-0.207059321698249,-0.0363719683260104 +175,2,3,0.206596723904414,0.481350779890947,-0.334257864561706 +175,3,3,0.206596723904414,0.723281523690698,3.17349996250998 +175,4,3,0.206596723904414,1.43357721571009,2.39430459844192 +175,5,3,0.206596723904414,0.591094458526448,4.36295883976816 +176,1,3,0.0577034421995391,0.709794129416285,-0.28756468129535 +176,2,3,0.0577034421995391,-0.143545223443117,2.92356539775918 +176,3,3,0.0577034421995391,1.56256666105461,3.9682478053146 +176,4,3,0.0577034421995391,0.606339899387379,4.5254491122782 +176,5,3,0.0577034421995391,0.76336680665267,8.17539594185283 +177,1,3,0.97012801697565,0.804576615031986,0.53560539768589 +177,2,3,0.97012801697565,0.196553211150512,1.36387006814793 +177,3,3,0.97012801697565,1.00301505882641,3.51400322417612 +177,4,3,0.97012801697565,0.271445191644487,5.04448566201533 +177,5,3,0.97012801697565,1.55312861154355,6.70569729949341 +178,1,3,-0.357064593741126,-0.0607403759752695,1.01233438166498 +178,2,3,-0.357064593741126,0.490630427379729,-0.312090709939029 +178,3,3,-0.357064593741126,-0.364345488254412,2.05738386473734 +178,4,3,-0.357064593741126,0.764552669846505,1.8568801615876 +178,5,3,-0.357064593741126,1.49628986724636,4.22922510038847 +179,1,3,1.83082274441214,1.10323167318757,1.2584912282798 +179,2,3,1.83082274441214,0.995153151371863,0.976468061563311 +179,3,3,1.83082274441214,1.03251198368829,2.88884149892202 +179,4,3,1.83082274441214,1.26989041677338,4.53978057520108 +179,5,3,1.83082274441214,2.27557816245241,9.01599947501831 +180,1,3,0.155573560993865,-0.192908369096136,1.27981156464175 +180,2,3,0.155573560993865,-0.157534180273984,2.81031818521695 +180,3,3,0.155573560993865,-0.182158192618231,3.42405164561696 +180,4,3,0.155573560993865,1.35094942132599,3.96796596950746 +180,5,3,0.155573560993865,1.90194981967979,8.03537055637734 +181,1,3,-0.561729182385448,-0.716364698575066,0.0258495576976853 +181,2,3,-0.561729182385448,0.0517589100032271,-1.9223798367341 +181,3,3,-0.561729182385448,0.155757851391865,2.14638687839844 +181,4,3,-0.561729182385448,1.47783449114265,2.58274092963544 +181,5,3,-0.561729182385448,0.870450644156802,3.24356808948639 +182,1,3,0.26725208660956,-0.0491357653714034,2.49956058511792 +182,2,3,0.26725208660956,-0.210909802338806,0.886149232902366 +182,3,3,0.26725208660956,0.622334850505215,3.8931379580875 +182,4,3,0.26725208660956,1.46665374250886,5.198204176303 +182,5,3,0.26725208660956,0.627533310561888,6.03883577577174 +183,1,3,-1.58651614698834,0.0078323858732432,2.93167490126052 +183,2,3,-1.58651614698834,-0.309478667711119,0.850162920976363 +183,3,3,-1.58651614698834,-0.931464591505026,3.28271352889749 +183,4,3,-1.58651614698834,0.331128687743081,2.80293524183476 +183,5,3,-1.58651614698834,0.235317940303293,1.89763913483971 +184,1,3,0.964008293737022,0.862304156601825,1.07052054416651 +184,2,3,0.964008293737022,0.78332774517736,2.54544572781402 +184,3,3,0.964008293737022,0.760983528069229,3.13566633991018 +184,4,3,0.964008293737022,1.86056654613075,5.340943408593 +184,5,3,0.964008293737022,1.53754550750828,7.19655087170372 +185,1,3,-0.136501349380018,-0.714009827475327,0.784769499960354 +185,2,3,-0.136501349380018,0.133897812852903,0.171777702203639 +185,3,3,-0.136501349380018,-0.263791603073048,0.962395384355854 +185,4,3,-0.136501349380018,1.19789754598816,2.59010782342184 +185,5,3,-0.136501349380018,0.268244919773508,3.92974278467078 +186,1,3,-1.28961941968184,-0.918430047518358,-1.9686296021114 +186,2,3,-1.28961941968184,-1.4415577155884,-0.148603099732199 +186,3,3,-1.28961941968184,1.16603286464104,0.160275393261301 +186,4,3,-1.28961941968184,-0.420341395456616,2.6351542975303 +186,5,3,-1.28961941968184,0.219569883299303,5.01919880111458 +187,1,3,-0.50487449632609,-0.397170453608418,1.91968990836625 +187,2,3,-0.50487449632609,0.310110929892259,1.93900182773006 +187,3,3,-0.50487449632609,-0.223467929354194,1.54844471705867 +187,4,3,-0.50487449632609,0.632636963181234,2.64265112445697 +187,5,3,-0.50487449632609,1.36338592016433,5.02622184660742 +188,1,3,0.346591684139401,0.147175470379114,-0.466845097919033 +188,2,3,0.346591684139401,1.03169374286638,0.659536188793482 +188,3,3,0.346591684139401,1.17291033726791,1.44307857044969 +188,4,3,0.346591684139401,0.9003299952002,2.15240892070531 +188,5,3,0.346591684139401,0.903593984244808,5.02830703671822 +189,1,3,0.576308110605711,-0.122536463758614,-1.57705029269863 +189,2,3,0.576308110605711,0.291477322026575,-0.306041213307675 +189,3,3,0.576308110605711,0.690232742116615,2.2422184539491 +189,4,3,0.576308110605711,1.01847578790693,1.08223612517947 +189,5,3,0.576308110605711,0.711335171990724,4.07221565330838 +190,1,3,2.08095198548021,1.33707232095089,1.3473200599755 +190,2,3,2.08095198548021,1.83156228593153,2.73102211717648 +190,3,3,2.08095198548021,0.526486445244764,3.26264599133572 +190,4,3,2.08095198548021,1.87485237391011,7.60163197735244 +190,5,3,2.08095198548021,1.4436691449854,8.69741127785079 +191,1,3,0.541741765731616,0.75677092222344,-0.17741464798554 +191,2,3,0.541741765731616,0.580456407073188,-0.352770440207899 +191,3,3,0.541741765731616,1.1237012894253,0.401484430278701 +191,4,3,0.541741765731616,0.771952065397894,5.02746098105613 +191,5,3,0.541741765731616,1.86655557642009,6.30572793910101 +192,1,3,0.374552810372194,0.606532770511559,2.05554214836543 +192,2,3,0.374552810372194,-0.514822382316804,2.18945756616643 +192,3,3,0.374552810372194,0.881168519061104,4.45925069297196 +192,4,3,0.374552810372194,0.812796538462421,6.0401274280228 +192,5,3,0.374552810372194,0.790236464746359,8.97534411362399 +193,1,3,-0.547755672644997,0.195225650839263,2.65504759756614 +193,2,3,-0.547755672644997,0.0367267682650421,-0.280690334987242 +193,3,3,-0.547755672644997,0.418469306023656,0.601330235148127 +193,4,3,-0.547755672644997,0.488693531216109,5.44972918275153 +193,5,3,-0.547755672644997,0.921971026069666,5.69678613783221 +194,1,3,0.823375237959988,0.794874722936144,-0.778832178556864 +194,2,3,0.823375237959988,1.12542250133637,-0.934186219579191 +194,3,3,0.823375237959988,1.22904636185405,2.74314586857224 +194,4,3,0.823375237959988,0.528522318252197,5.13677706748988 +194,5,3,0.823375237959988,1.4961081216668,6.19996985429195 +195,1,3,-2.28077468001876,-0.895318594048709,-1.24407433196296 +195,2,3,-2.28077468001876,-1.63435720128489,-0.86850217910119 +195,3,3,-2.28077468001876,-0.465149221884744,0.311909085828801 +195,4,3,-2.28077468001876,-0.0153283011291445,-0.250171464401362 +195,5,3,-2.28077468001876,0.402100989150701,0.679870171029218 +196,1,3,-0.877192455164409,0.0199183920553034,1.12042339769811 +196,2,3,-0.877192455164409,0.309966868712776,-0.870717661006155 +196,3,3,-0.877192455164409,-0.0191782684956241,0.804081512299063 +196,4,3,-0.877192455164409,1.184332013726,2.94402306952057 +196,5,3,-0.877192455164409,1.22883637027276,4.02831372207414 +197,1,3,-1.79625296210919,-0.361051529214706,-0.343292683318492 +197,2,3,-1.79625296210919,-0.384109190615947,1.42603749350341 +197,3,3,-1.79625296210919,-0.0226322056923884,2.55701916614874 +197,4,3,-1.79625296210919,-0.279036736874818,2.26910569930322 +197,5,3,-1.79625296210919,0.909628022790928,4.30698920325096 +198,1,3,0.294599695261137,0.89036589334075,1.69884316076478 +198,2,3,0.294599695261137,0.0796835929948345,2.69317132702638 +198,3,3,0.294599695261137,0.497516125839013,6.6276099186752 +198,4,3,0.294599695261137,0.354299212915527,7.44367073594493 +198,5,3,0.294599695261137,0.387027339411681,6.31279079228627 +199,1,3,1.03393476511463,1.05710791200614,-0.973952817405197 +199,2,3,1.03393476511463,1.5307145467393,0.234826291447174 +199,3,3,1.03393476511463,0.95979715049037,0.779861823865699 +199,4,3,1.03393476511463,1.57566721611891,0.355580473898519 +199,5,3,1.03393476511463,1.95098305906965,4.97771532813293 +200,1,3,0.67406226635374,0.275277717694543,0.843334384136715 +200,2,3,0.67406226635374,1.30090857740661,1.93264815873112 +200,3,3,0.67406226635374,0.555597032912415,1.0968307169484 +200,4,3,0.67406226635374,2.22816439869584,5.46923755530594 +200,5,3,0.67406226635374,1.61344654035566,7.23989637649208 +201,1,4,-0.145619005764009,0.425062464970386,0.261616790409442 +201,2,4,-0.145619005764009,0.868817131430003,-0.445132841776166 +201,3,4,-0.145619005764009,0.492502278198704,1.83029648571645 +201,4,4,-0.145619005764009,0.630758469484849,3.30405105563127 +201,5,4,-0.145619005764009,0.776041722729689,5.63701109236231 +202,1,4,0.20848748143,-0.194711403480027,0.179745580798258 +202,2,4,0.20848748143,0.492994024084207,1.13854396901761 +202,3,4,0.20848748143,0.783119039986885,2.92293105342569 +202,4,4,0.20848748143,0.299329049970995,4.60042133367869 +202,5,4,0.20848748143,1.26220961181419,5.89206406267787 +203,1,4,-0.0189341751210595,0.882938170134449,-2.05139873048146 +203,2,4,-0.0189341751210595,1.22759454414411,-0.0627311545998119 +203,3,4,-0.0189341751210595,0.408892095900875,-0.568041233080671 +203,4,4,-0.0189341751210595,0.436565159144488,0.272986052753453 +203,5,4,-0.0189341751210595,1.0238662931341,4.38776490089903 +204,1,4,0.527436038711728,0.697663366868311,1.93523689027915 +204,2,4,0.527436038711728,0.427325742523677,0.473254757727205 +204,3,4,0.527436038711728,2.00923508548696,0.622493693881969 +204,4,4,0.527436038711728,0.971970604102201,2.53441331906684 +204,5,4,0.527436038711728,0.778763771105946,5.68427368611851 +205,1,4,1.27844865328638,0.687467032299747,-0.380514602850353 +205,2,4,1.27844865328638,0.849429647247574,-0.681233299781682 +205,3,4,1.27844865328638,1.70010462355952,1.11108413627184 +205,4,4,1.27844865328638,0.655787819387356,3.39764474726244 +205,5,4,1.27844865328638,1.88135604224227,5.64744922059397 +206,1,4,-1.18246074137794,0.170036724255962,1.13871929843077 +206,2,4,-1.18246074137794,-0.12748806549754,-0.0532917062726999 +206,3,4,-1.18246074137794,-0.370786134857948,-0.896112659770101 +206,4,4,-1.18246074137794,-0.00441465296565557,3.02106930082714 +206,5,4,-1.18246074137794,1.42894983162112,4.63504222287801 +207,1,4,-0.157505532820991,0.716516078170618,0.509385682877153 +207,2,4,-0.157505532820991,0.000137832348927613,1.72928345426769 +207,3,4,-0.157505532820991,0.544829211905254,3.3758879541191 +207,4,4,-0.157505532820991,1.06084126726245,2.44803531226303 +207,5,4,-0.157505532820991,1.39705579795745,4.77175746613998 +208,1,4,0.397900614221911,1.00807136568749,-0.252075928949645 +208,2,4,0.397900614221911,0.838451756495337,-0.268883833866314 +208,3,4,0.397900614221911,0.826190093558104,3.20343190402967 +208,4,4,0.397900614221911,0.766221557988312,3.90633552973664 +208,5,4,0.397900614221911,1.24218805277057,6.43545801727386 +209,1,4,3.08818771202226,1.18057364412943,2.18396135142987 +209,2,4,3.08818771202226,2.27702919591625,3.55491931574783 +209,3,4,3.08818771202226,1.85263702901336,5.08918598891455 +209,4,4,3.08818771202226,2.78921862657327,6.17571524563542 +209,5,4,3.08818771202226,3.2225773233352,8.73776270096561 +210,1,4,1.68702207569785,1.7993545395904,0.532838840080434 +210,2,4,1.68702207569785,0.869440033595608,2.2410028229324 +210,3,4,1.68702207569785,1.01883836380461,2.13221062747574 +210,4,4,1.68702207569785,2.00583631958716,5.87886551140786 +210,5,4,1.68702207569785,1.70231546818835,8.79615064905242 +211,1,4,-0.875883372538378,-0.978958973674517,0.581621421483876 +211,2,4,-0.875883372538378,-0.786653411189003,1.41121195690622 +211,3,4,-0.875883372538378,-0.190563082665021,2.20404293055772 +211,4,4,-0.875883372538378,0.260808890938771,2.21556675699285 +211,5,4,-0.875883372538378,0.834906311643272,3.43606019304585 +212,1,4,1.30400386151833,-0.0739311314895021,-0.27259878117807 +212,2,4,1.30400386151833,1.81479648564854,2.13416997391053 +212,3,4,1.30400386151833,0.995640877361736,1.45154720287005 +212,4,4,1.30400386151833,1.30443724412722,3.63731216528277 +212,5,4,1.30400386151833,1.13207952550597,5.59816451674992 +213,1,4,-0.23259357883221,0.543204811836342,-1.01449534225791 +213,2,4,-0.23259357883221,0.12693829181683,-1.00590686596478 +213,3,4,-0.23259357883221,-0.550437546679332,-0.857902657994927 +213,4,4,-0.23259357883221,1.08009092657631,1.48411443232596 +213,5,4,-0.23259357883221,1.28748975003943,2.58372978563663 +214,1,4,0.934509368328794,0.283756358537217,3.21736850838587 +214,2,4,0.934509368328794,0.997296492121042,3.83994823577147 +214,3,4,0.934509368328794,0.404522074604913,3.87328670453522 +214,4,4,0.934509368328794,1.75594042974006,4.3940685244154 +214,5,4,0.934509368328794,1.15095624853172,5.93523237524799 +215,1,4,-2.02633774528942,-0.627275698118266,-1.29360704342256 +215,2,4,-2.02633774528942,-0.984718100457219,-0.77486879653039 +215,3,4,-2.02633774528942,-0.797652009536068,-3.57862337525148 +215,4,4,-2.02633774528942,-0.696679896135425,-0.940107496082131 +215,5,4,-2.02633774528942,-0.600475556059633,-0.057816328034735 +216,1,4,-1.1345723514721,-0.999801486681439,-1.99926954853096 +216,2,4,-1.1345723514721,0.051632320309278,-1.30478035939822 +216,3,4,-1.1345723514721,-0.218461416657432,-0.639274587840224 +216,4,4,-1.1345723514721,0.477947371104333,0.953246898586203 +216,5,4,-1.1345723514721,0.248014439178224,2.70084584246067 +217,1,4,0.29136391143389,0.183245908076356,1.68485761265153 +217,2,4,0.29136391143389,0.395426152640086,1.93259002639756 +217,3,4,0.29136391143389,0.872056192556017,0.908055760001004 +217,4,4,0.29136391143389,0.48010831096284,2.72864700809539 +217,5,4,0.29136391143389,0.766822417416036,4.90222656954324 +218,1,4,1.02025953266783,1.18979863184446,4.13767206136758 +218,2,4,1.02025953266783,1.31261695412382,3.3885702756416 +218,3,4,1.02025953266783,1.62271221719155,3.51247762596368 +218,4,4,1.02025953266783,2.30279001427239,6.44275765130315 +218,5,4,1.02025953266783,2.15392921059779,6.075212576793 +219,1,4,0.0658993928206432,0.790306891360803,-0.482614087946492 +219,2,4,0.0658993928206432,-0.349717346583336,1.52429755080546 +219,3,4,0.0658993928206432,0.56426858103447,3.24615220616319 +219,4,4,0.0658993928206432,1.20758231685881,2.95719739567898 +219,5,4,0.0658993928206432,0.327607937927172,7.82263208692572 +220,1,4,-1.02912673591929,-1.16919501498074,-3.13433236697156 +220,2,4,-1.02912673591929,0.434877627773687,-1.23305344836268 +220,3,4,-1.02912673591929,-0.797303788823903,-0.284062193305323 +220,4,4,-1.02912673591929,0.116308946916564,1.35017095093358 +220,5,4,-1.02912673591929,0.94393107694499,1.91875502569586 +221,1,4,-0.382443187726708,0.389597842296713,0.382879340799262 +221,2,4,-0.382443187726708,0.374049508941051,0.114914620326604 +221,3,4,-0.382443187726708,-0.127663034111751,1.20459217867908 +221,4,4,-0.382443187726708,0.293992048559745,2.99629254268107 +221,5,4,-0.382443187726708,1.36640484160858,5.19669505614587 +222,1,4,-2.07378709910601,-1.2515950543621,0.958666353461258 +222,2,4,-2.07378709910601,-0.827366544533211,0.931687279406236 +222,3,4,-2.07378709910601,0.2149092035418,1.61719600463625 +222,4,4,-2.07378709910601,-0.182078896920811,1.44021079569763 +222,5,4,-2.07378709910601,-1.05195888378581,1.57467042273008 +223,1,4,0.0571434861440279,0.947769560761134,1.0358412759774 +223,2,4,0.0571434861440279,0.41621436563672,0.292865445061844 +223,3,4,0.0571434861440279,1.48618542216699,3.40291389631824 +223,4,4,0.0571434861440279,0.762338360571163,1.96188222901659 +223,5,4,0.0571434861440279,0.177984948090528,4.52859209908139 +224,1,4,-0.190737942505499,-0.738255736922229,0.127569868577206 +224,2,4,-0.190737942505499,0.372240080607862,-0.12501447773974 +224,3,4,-0.190737942505499,0.619002253283965,0.887761699211636 +224,4,4,-0.190737942505499,-0.221048847835207,2.18896719978384 +224,5,4,-0.190737942505499,1.03986148135431,5.75670165761339 +225,1,4,0.504750370322883,-0.0984297837436904,1.67458725198859 +225,2,4,0.504750370322883,1.27231230071316,2.08663858390143 +225,3,4,0.504750370322883,0.322652963868896,2.95092850709882 +225,4,4,0.504750370322883,0.478947178775107,5.43233008713638 +225,5,4,0.504750370322883,0.825625553003256,8.74656660654164 +226,1,4,0.566646966396415,0.385509168954166,0.789949604386638 +226,2,4,0.566646966396415,0.761102906211001,0.77574412438031 +226,3,4,0.566646966396415,0.864967381215401,1.58861176824269 +226,4,4,0.566646966396415,1.51851886429129,3.61972237667015 +226,5,4,0.566646966396415,1.76621774098752,4.61367138253264 +227,1,4,0.416298605248852,0.241969846324854,0.461314108917958 +227,2,4,0.416298605248852,0.307258447403572,1.02689317889856 +227,3,4,0.416298605248852,0.389077651776476,0.410468139553837 +227,4,4,0.416298605248852,1.17249774667565,3.58318359838724 +227,5,4,0.416298605248852,1.09203011758049,3.99468544614685 +228,1,4,-1.8216741221751,-0.49545657763507,-0.918010485280072 +228,2,4,-1.8216741221751,0.027308409221532,1.68188149042672 +228,3,4,-1.8216741221751,-0.594474408950622,1.80605678566885 +228,4,4,-1.8216741221751,-0.541947413609883,1.61611304815113 +228,5,4,-1.8216741221751,0.627253636608009,2.48045304876612 +229,1,4,1.95837805386369,1.21634091921974,-0.462827312292844 +229,2,4,1.95837805386369,1.74988569899565,0.945110855282057 +229,3,4,1.95837805386369,1.52757147585732,2.50011393068116 +229,4,4,1.95837805386369,1.99132635448695,4.01187377755918 +229,5,4,1.95837805386369,1.31915164171884,5.35579248615526 +230,1,4,-0.854064145631843,-1.41460473600476,0.144806847230399 +230,2,4,-0.854064145631843,0.662825868088806,1.89877543200894 +230,3,4,-0.854064145631843,-0.652182540520576,0.308075121258965 +230,4,4,-0.854064145631843,0.68245012625534,1.76009392748307 +230,5,4,-0.854064145631843,1.61509807716483,2.02400699601978 +231,1,4,-1.83315433652527,-0.751082840375815,-0.520185148961388 +231,2,4,-1.83315433652527,-0.88514688326197,1.65944375180379 +231,3,4,-1.83315433652527,-0.295660470843824,-0.907336878398132 +231,4,4,-1.83315433652527,-0.495792414559182,-0.225431070997175 +231,5,4,-1.83315433652527,0.0418227109791764,2.81883558740669 +232,1,4,1.24837686380501,0.838224876289186,-0.621704789994096 +232,2,4,1.24837686380501,2.25510434548755,0.328480973937103 +232,3,4,1.24837686380501,2.10481475648221,2.88086825041376 +232,4,4,1.24837686380501,1.42429405303522,3.36866842489618 +232,5,4,1.24837686380501,1.05793232096196,6.97140678219277 +233,1,4,2.85114953488483,1.29139647396778,0.99238604179257 +233,2,4,2.85114953488483,1.8705653471451,-0.134331908602972 +233,3,4,2.85114953488483,2.20740279929195,3.07499052105055 +233,4,4,2.85114953488483,2.15171879921461,5.39633768858733 +233,5,4,2.85114953488483,2.80830718066296,8.80251164018261 +234,1,4,-0.936743330351871,-0.0945872932981996,-0.141865793981232 +234,2,4,-0.936743330351871,0.574562882075806,0.403537407215633 +234,3,4,-0.936743330351871,0.0997238820429076,-1.1903911691781 +234,4,4,-0.936743330351871,0.361027224824763,0.762696042186893 +234,5,4,-0.936743330351871,0.557639499328201,3.13537313460174 +235,1,4,1.47580185605354,0.00281937707094781,2.6165660705684 +235,2,4,1.47580185605354,1.34854747593014,2.41643446713607 +235,3,4,1.47580185605354,1.99294917196747,2.43476543091642 +235,4,4,1.47580185605354,1.6967718166645,5.85878317716718 +235,5,4,1.47580185605354,1.95520672221328,8.07589311734043 +236,1,4,0.208747033908035,0.0745711285520664,1.31236043050198 +236,2,4,0.208747033908035,1.20431328156382,1.33371052800362 +236,3,4,0.208747033908035,-0.0523967515138719,2.31728577207176 +236,4,4,0.208747033908035,0.307677941881186,3.09254540576663 +236,5,4,0.208747033908035,0.894146130541882,2.88336258223656 +237,1,4,0.896059204631448,1.18810550586458,1.30092916076486 +237,2,4,0.896059204631448,0.454227458333857,1.52099651693863 +237,3,4,0.896059204631448,1.35758023985095,1.35051819874402 +237,4,4,0.896059204631448,0.331892873266859,5.76870679312247 +237,5,4,0.896059204631448,1.41000502507945,6.23106759672651 +238,1,4,0.113951054067604,-0.220492161787358,0.481424354940191 +238,2,4,0.113951054067604,0.811282363216797,0.725121018412195 +238,3,4,0.113951054067604,-0.34398335882458,-1.56392926823064 +238,4,4,0.113951054067604,1.53352069000688,1.92056368213871 +238,5,4,0.113951054067604,1.65603526197291,3.2588267214286 +239,1,4,0.332304485256041,1.15620286359608,-1.6681412354686 +239,2,4,0.332304485256041,0.747202621907737,-0.314622856456936 +239,3,4,0.332304485256041,0.334322005229968,-0.308578372252669 +239,4,4,0.332304485256041,1.42756911312914,2.47095762222018 +239,5,4,0.332304485256041,0.5879033667627,3.50822725475215 +240,1,4,0.00970594683895151,-0.0704452216851235,3.04122955188286 +240,2,4,0.00970594683895151,0.76395694446961,2.95416927363478 +240,3,4,0.00970594683895151,-0.046594574847339,2.45553375362412 +240,4,4,0.00970594683895151,0.769993965782161,3.94403688734056 +240,5,4,0.00970594683895151,-0.443491592376686,5.34311300206848 +241,1,4,-0.223773244924263,-0.65278284105429,-1.48828544189116 +241,2,4,-0.223773244924263,0.593489908952763,0.370330080113892 +241,3,4,-0.223773244924263,0.545104944921187,0.891140750941535 +241,4,4,-0.223773244924263,1.64379566963664,3.74751639708699 +241,5,4,-0.223773244924263,0.846644395352223,4.99132096127366 +242,1,4,0.311444780258236,1.48178803288983,-0.380767459864371 +242,2,4,0.311444780258236,1.2734006735771,0.0259808435259069 +242,3,4,0.311444780258236,-0.109772227045998,1.50016359137237 +242,4,4,0.311444780258236,0.593862883300123,2.71532805813609 +242,5,4,0.311444780258236,1.72865594619768,3.97334193486166 +243,1,4,0.341619072387833,-0.207498945762728,-0.820909073821433 +243,2,4,0.341619072387833,0.385145039448137,-0.841004612792712 +243,3,4,0.341619072387833,0.368659884620509,0.0844219376525748 +243,4,4,0.341619072387833,1.22635035866907,0.952682324592303 +243,5,4,0.341619072387833,1.76880167951173,4.90943609337553 +244,1,4,-2.04126212237833,-1.60140313192775,0.663827237102985 +244,2,4,-2.04126212237833,-0.930248302703456,1.58493805703856 +244,3,4,-2.04126212237833,0.346794972361608,0.410523261284975 +244,4,4,-2.04126212237833,-0.401524382975107,1.79390821724782 +244,5,4,-2.04126212237833,0.273694848325163,1.61289090300176 +245,1,4,-0.36739874796181,0.384491144940372,-0.570341416585646 +245,2,4,-0.36739874796181,-0.0554327743052686,1.77983314229872 +245,3,4,-0.36739874796181,1.17310166519544,1.2281890200107 +245,4,4,-0.36739874796181,1.24012454957959,2.79092718189971 +245,5,4,-0.36739874796181,0.779841451650018,4.30924212933595 +246,1,4,-0.231984127628662,0.476599317012363,0.43657726339478 +246,2,4,-0.231984127628662,0.248843711953803,0.863012870447415 +246,3,4,-0.231984127628662,0.274345464327848,1.55119128297576 +246,4,4,-0.231984127628662,0.473713731438912,3.09769523693271 +246,5,4,-0.231984127628662,0.222520430314378,3.86862426455256 +247,1,4,-0.0956398371392947,0.149491999232215,0.598097954992508 +247,2,4,-0.0956398371392947,0.022604092587618,-0.0664439657673578 +247,3,4,-0.0956398371392947,0.220212399721201,0.524357762567856 +247,4,4,-0.0956398371392947,1.00365977468176,2.46031605710149 +247,5,4,-0.0956398371392947,1.28279241725496,4.85795540887373 +248,1,4,-1.70619282588905,-0.645361101889452,1.9916344801831 +248,2,4,-1.70619282588905,-0.862927516651366,3.66791717271056 +248,3,4,-1.70619282588905,0.0127762391683118,0.609025656290565 +248,4,4,-1.70619282588905,-0.658014594679568,3.18183436799844 +248,5,4,-1.70619282588905,-0.072577383389055,2.07262299675295 +249,1,4,-0.927336330190315,0.0705125883849166,2.91410539149122 +249,2,4,-0.927336330190315,-0.151899489836396,2.18979215037809 +249,3,4,-0.927336330190315,-0.42190651916858,1.65888100186867 +249,4,4,-0.927336330190315,0.460398446536725,4.08669056464587 +249,5,4,-0.927336330190315,0.600612524859274,5.90915200490956 +250,1,4,-1.14162801235656,0.206590383641819,0.28352920602639 +250,2,4,-1.14162801235656,-0.47704255454694,1.449317489013 +250,3,4,-1.14162801235656,-0.263536574659807,-0.4116398920302 +250,4,4,-1.14162801235656,0.292921465867566,0.36941772834878 +250,5,4,-1.14162801235656,0.326599649673672,3.25456142230352 +251,1,4,-0.533133577076554,0.331363034011417,-1.68870149736653 +251,2,4,-0.533133577076554,-0.117821643463392,0.857656318646477 +251,3,4,-0.533133577076554,1.30481063021976,0.226720234297566 +251,4,4,-0.533133577076554,-0.0456515009391002,1.62908564662487 +251,5,4,-0.533133577076554,0.336734673783287,2.6538556184386 +252,1,4,-1.4242396017973,-0.616487643209825,-1.16197382550618 +252,2,4,-1.4242396017973,0.809377735235426,0.0307873435046787 +252,3,4,-1.4242396017973,0.105432878183446,-0.0608605853037751 +252,4,4,-1.4242396017973,-0.0919608429902865,2.1255165629107 +252,5,4,-1.4242396017973,1.14213687390222,1.39823536766718 +253,1,4,0.0421130196327412,1.03467783928002,-1.11463016065861 +253,2,4,0.0421130196327412,0.426069218333878,-0.766654289126376 +253,3,4,0.0421130196327412,0.434427914234819,1.22879798728331 +253,4,4,0.0421130196327412,0.956344900243424,1.95082296497428 +253,5,4,0.0421130196327412,0.995569508719923,6.3550670944955 +254,1,4,1.26480281838255,1.44835886628602,1.17559904360065 +254,2,4,1.26480281838255,0.882706330710806,1.49438786415408 +254,3,4,1.26480281838255,1.32459160552475,2.15149912950138 +254,4,4,1.26480281838255,0.784870571407219,2.86896633076918 +254,5,4,1.26480281838255,1.15274927647445,6.9490189531146 +255,1,4,-1.90012999787815,0.673194082270379,-0.096756989707168 +255,2,4,-1.90012999787815,-1.02133154445786,0.000225348399189085 +255,3,4,-1.90012999787815,-0.50678712210121,-1.02312348114859 +255,4,4,-1.90012999787815,0.542192852658531,0.997428462853665 +255,5,4,-1.90012999787815,0.273626936566653,0.92625656443509 +256,1,4,-0.252346344034527,1.03251230536201,0.683415454947388 +256,2,4,-0.252346344034527,0.161490907558304,0.168102835634015 +256,3,4,-0.252346344034527,0.498605839954082,1.01539631672537 +256,4,4,-0.252346344034527,0.110756718531093,2.9865347314332 +256,5,4,-0.252346344034527,0.798434708656316,5.63224386538034 +257,1,4,0.295978757655662,0.216684978603956,-1.77045143244746 +257,2,4,0.295978757655662,0.0586862830669549,1.44714523155237 +257,3,4,0.295978757655662,0.523459956256675,0.538358101141446 +257,4,4,0.295978757655662,1.56990095241408,2.09305803603714 +257,5,4,0.295978757655662,1.22199253431524,3.45003271456258 +258,1,4,1.45826398970798,1.73386027058223,-0.407308799164423 +258,2,4,1.45826398970798,1.61805486454142,-0.604501564966646 +258,3,4,1.45826398970798,2.15750962591565,-0.0310382347093021 +258,4,4,1.45826398970798,1.2065993232119,2.57988120880643 +258,5,4,1.45826398970798,2.3927575283945,4.378101946541 +259,1,4,-1.74927828242113,-0.9582717131597,-0.291874873747854 +259,2,4,-1.74927828242113,-0.0841825243742936,2.55602592722196 +259,3,4,-1.74927828242113,-0.66186210802338,1.08527973151208 +259,4,4,-1.74927828242113,-1.08468288544549,0.00776297528025038 +259,5,4,-1.74927828242113,0.0597058834678753,2.38968504577223 +260,1,4,-0.137485383994025,0.792215203761203,-1.10257299306267 +260,2,4,-0.137485383994025,0.0812225692570001,0.457900169155973 +260,3,4,-0.137485383994025,0.0805787039345165,1.08921496380089 +260,4,4,-0.137485383994025,0.443699607906665,3.65248584537344 +260,5,4,-0.137485383994025,0.574308585361864,2.1595938602977 +261,1,4,0.499633628174919,1.13992947289452,-1.70794926705912 +261,2,4,0.499633628174919,0.522811863197591,1.86087969706738 +261,3,4,0.499633628174919,0.56806718260539,2.17218766492301 +261,4,4,0.499633628174919,1.01730076322439,1.34763443391338 +261,5,4,0.499633628174919,1.47679969319364,2.79623267918103 +262,1,4,1.08067198678011,1.17717605152764,1.20733511818297 +262,2,4,1.08067198678011,1.1751219890645,2.17608523804642 +262,3,4,1.08067198678011,1.701187483165,3.52833996545476 +262,4,4,1.08067198678011,1.41892071374161,4.73777070820504 +262,5,4,1.08067198678011,1.7323885465477,7.84386251335008 +263,1,4,-0.56060719046659,0.105163095142194,1.2071490768394 +263,2,4,-0.56060719046659,0.0198785644593,0.831932700091331 +263,3,4,-0.56060719046659,0.449070786540603,1.66734205810187 +263,4,4,-0.56060719046659,-0.0394458634330443,2.59047103292563 +263,5,4,-0.56060719046659,0.678285785599231,3.15533257937058 +264,1,4,0.0907057898496138,-0.47278090245472,1.58095980850717 +264,2,4,0.0907057898496138,0.446766877366091,-1.38279888216449 +264,3,4,0.0907057898496138,0.544943667505921,2.71674869734128 +264,4,4,0.0907057898496138,0.76573621541547,2.96574195549868 +264,5,4,0.0907057898496138,1.41895561912996,1.95434593015983 +265,1,4,1.08467352421369,1.3456816273122,-0.708797764954616 +265,2,4,1.08467352421369,1.08450782081211,-1.78097508051042 +265,3,4,1.08467352421369,1.39961924915943,-1.79699087295665 +265,4,4,1.08467352421369,1.17295798482399,4.10392366788344 +265,5,4,1.08467352421369,1.25389946236374,3.77002388463445 +266,1,4,-0.663235486302357,-0.156636189295917,-0.913206940327043 +266,2,4,-0.663235486302357,-0.157832130555018,-1.0445091310822 +266,3,4,-0.663235486302357,1.16715836241289,0.447228752399067 +266,4,4,-0.663235486302357,0.216898343439123,1.26839928522657 +266,5,4,-0.663235486302357,0.953672436944933,2.48130106007881 +267,1,4,0.507544490493503,0.417972511745019,0.941617354075974 +267,2,4,0.507544490493503,1.18950643981523,1.9862507863618 +267,3,4,0.507544490493503,0.371223098149939,2.95823701428986 +267,4,4,0.507544490493503,1.54379395353631,4.3867244894278 +267,5,4,0.507544490493503,1.78172598951428,7.97799074265316 +268,1,4,-1.58517361075372,-0.449522195882874,0.360124536502925 +268,2,4,-1.58517361075372,-0.636906574393382,-0.0564251355860901 +268,3,4,-1.58517361075372,-0.498536311791312,-0.784646559316506 +268,4,4,-1.58517361075372,0.215602396572638,-0.0850944418558538 +268,5,4,-1.58517361075372,-0.190835436734482,1.64915421342684 +269,1,4,-0.451581849130411,0.788061274122034,0.443360546254768 +269,2,4,-0.451581849130411,-0.255544152496929,0.0991548889815735 +269,3,4,-0.451581849130411,-0.0456743309929889,-0.0154119586940922 +269,4,4,-0.451581849130411,0.863064056517142,1.38119858118668 +269,5,4,-0.451581849130411,0.375685752463198,3.70360413505207 +270,1,4,0.291875229266022,1.16560069998419,-0.0623894341601451 +270,2,4,0.291875229266022,0.478483492965743,1.60919552841901 +270,3,4,0.291875229266022,0.893078366696526,2.0977066590772 +270,4,4,0.291875229266022,1.55374186798575,2.24487890667299 +270,5,4,0.291875229266022,0.608647637970957,3.53143962307269 +271,1,4,0.0779558094282166,0.341207463349526,1.23452468056018 +271,2,4,0.0779558094282166,0.952023617349011,0.641523269725003 +271,3,4,0.0779558094282166,0.809678561125316,1.4314431131351 +271,4,4,0.0779558094282166,0.998192867745152,1.96103023715827 +271,5,4,0.0779558094282166,0.727489746127947,4.93574553674648 +272,1,4,0.302325606233118,-0.313005897459611,2.22744223449703 +272,2,4,0.302325606233118,1.26633743489874,0.947006120888524 +272,3,4,0.302325606233118,0.456127163622586,0.915729413013545 +272,4,4,0.302325606233118,0.613026738962035,2.78755041110836 +272,5,4,0.302325606233118,1.51074045322368,4.97690784001007 +273,1,4,-0.598886389459515,-1.23214633546578,2.85034802997392 +273,2,4,-0.598886389459515,0.469231509795185,2.28936950149414 +273,3,4,-0.598886389459515,0.0653801138614925,5.32178136606866 +273,4,4,-0.598886389459515,1.01017905748285,3.99470307015833 +273,5,4,-0.598886389459515,1.47174857219337,5.94702329102721 +274,1,4,-0.67930601368561,0.636870921277184,1.62418182853493 +274,2,4,-0.67930601368561,0.10978632365971,-0.016826567941179 +274,3,4,-0.67930601368561,1.28079905770633,-0.540404005824146 +274,4,4,-0.67930601368561,0.580204977086341,3.20791975425798 +274,5,4,-0.67930601368561,0.429730028989504,4.02565224395567 +275,1,4,-0.11883036286453,0.0987066122506285,0.341097128148806 +275,2,4,-0.11883036286453,0.333640883169436,0.775998225006508 +275,3,4,-0.11883036286453,0.118838622926395,1.93875270343894 +275,4,4,-0.11883036286453,0.663691666488021,2.02147912609745 +275,5,4,-0.11883036286453,0.776990997101618,3.70891660244939 +276,1,4,-1.05250893889619,0.207706706604471,1.55724848283629 +276,2,4,-1.05250893889619,-0.331494328078079,1.9362374432695 +276,3,4,-1.05250893889619,-0.319789048714261,2.05616909833095 +276,4,4,-1.05250893889619,-0.143506203445625,2.9038522125567 +276,5,4,-1.05250893889619,0.799605265472782,4.45063561169586 +277,1,4,-0.795844037443362,0.19773315538162,-1.8378169583471 +277,2,4,-0.795844037443362,0.270395158483434,0.6880310785137 +277,3,4,-0.795844037443362,0.472347160923808,1.47613775270217 +277,4,4,-0.795844037443362,-0.768088395024826,2.51168159079353 +277,5,4,-0.795844037443362,0.547199254652822,4.15777050391864 +278,1,4,0.173615046018517,0.965776968467884,1.13520464623337 +278,2,4,0.173615046018517,0.588715357259776,1.60995251889149 +278,3,4,0.173615046018517,0.503368361124506,1.16655823323351 +278,4,4,0.173615046018517,0.299150390000537,4.05486954578688 +278,5,4,0.173615046018517,0.827024303091238,4.9639610608935 +279,1,4,0.0416951645261407,0.866967622081922,1.18077004074956 +279,2,4,0.0416951645261407,0.0551413833702407,-0.0854470487913126 +279,3,4,0.0416951645261407,1.53638709831634,1.26794253836204 +279,4,4,0.0416951645261407,1.13716982550896,2.83178478428392 +279,5,4,0.0416951645261407,1.12145927780378,5.62157478688342 +280,1,4,1.7584858414703,0.539505404090267,1.42838736347038 +280,2,4,1.7584858414703,1.99518281921814,0.520484304504163 +280,3,4,1.7584858414703,1.29540090344877,3.68015674163707 +280,4,4,1.7584858414703,2.616567296328,5.95731680505034 +280,5,4,1.7584858414703,1.86573904906666,8.38399941725952 +281,1,4,0.294307873629322,0.367016285926457,0.290612600985405 +281,2,4,0.294307873629322,0.255139421662298,1.35557946058075 +281,3,4,0.294307873629322,1.89254428009209,0.47947229670969 +281,4,4,0.294307873629322,0.756565102470336,5.02955675119337 +281,5,4,0.294307873629322,0.59436204587968,5.65770391957791 +282,1,4,-0.668920427496635,-0.16090516774787,-0.0220861603980491 +282,2,4,-0.668920427496635,-0.293932941387419,-0.860657413226595 +282,3,4,-0.668920427496635,-0.617888639151153,-1.15166725712085 +282,4,4,-0.668920427496635,0.888558954552449,2.92589475136101 +282,5,4,-0.668920427496635,0.966788957178054,2.38434469294651 +283,1,4,-0.147887549805199,1.21883984593654,0.201493397033268 +283,2,4,-0.147887549805199,-0.104507144604202,1.05560905284092 +283,3,4,-0.147887549805199,0.644480146848447,2.5893567926083 +283,4,4,-0.147887549805199,0.689421659305855,2.97529728089224 +283,5,4,-0.147887549805199,0.0146694646590244,2.7385080198275 +284,1,4,1.44872204277006,1.52566993336519,-1.09833942670499 +284,2,4,1.44872204277006,1.13121617916432,-0.763523549678274 +284,3,4,1.44872204277006,1.02618553930428,2.14434055999589 +284,4,4,1.44872204277006,2.32644315791059,4.70517765580124 +284,5,4,1.44872204277006,1.6632694223249,4.67184316918207 +285,1,4,-1.73133492647527,-1.1333333954352,0.530923214738404 +285,2,4,-1.73133492647527,-0.0797624768335474,2.04554158053777 +285,3,4,-1.73133492647527,0.279531137402414,1.22563072312665 +285,4,4,-1.73133492647527,0.324317655202742,1.53333743270017 +285,5,4,-1.73133492647527,-0.0842300049235476,3.21185361312661 +286,1,4,0.573428126344241,0.794394353051903,4.01969753025436 +286,2,4,0.573428126344241,0.283988223371164,5.20447479100824 +286,3,4,0.573428126344241,0.822028322194829,4.55993273559606 +286,4,4,0.573428126344241,0.36957610586572,4.07308490588507 +286,5,4,0.573428126344241,0.461021582718634,8.45361684973381 +287,1,4,1.49761969936007,0.38082553869154,3.75268269573446 +287,2,4,1.49761969936007,1.89226501243119,2.33915199737279 +287,3,4,1.49761969936007,2.11120221547793,3.87823256197682 +287,4,4,1.49761969936007,0.575220036330622,7.54075988529905 +287,5,4,1.49761969936007,1.56730512370106,10.87439030553 +288,1,4,0.120198304783001,0.221838332294035,-0.0343789585175994 +288,2,4,0.120198304783001,0.912273335999318,0.486857046923631 +288,3,4,0.120198304783001,0.46617645674522,-0.324957673735775 +288,4,4,0.120198304783001,0.833832779999747,1.28965171934442 +288,5,4,0.120198304783001,1.97307799051121,2.86679713845684 +289,1,4,-1.21503732454083,0.242954873846912,2.83881937447877 +289,2,4,-1.21503732454083,-0.0432160852840555,0.078823737728758 +289,3,4,-1.21503732454083,-0.427710871312259,0.70638669292398 +289,4,4,-1.21503732454083,-0.556300707912749,0.928782733937543 +289,5,4,-1.21503732454083,-0.135667200525247,2.68144692998541 +290,1,4,-1.1571502810733,0.210540410190643,0.338661393530131 +290,2,4,-1.1571502810733,-0.539918911846132,1.72072318693133 +290,3,4,-1.1571502810733,0.484706650698575,1.59241521439714 +290,4,4,-1.1571502810733,-0.405062434886484,4.11562329467502 +290,5,4,-1.1571502810733,0.981369158137991,3.20289878600007 +291,1,4,-0.373521121162934,0.0230948219244378,1.26027391139345 +291,2,4,-0.373521121162934,-0.256186240139986,2.11891529051296 +291,3,4,-0.373521121162934,-0.441530085576679,1.62632162393161 +291,4,4,-0.373521121162934,0.838881189609289,2.97479271232015 +291,5,4,-0.373521121162934,0.445423620665624,5.739502917094 +292,1,4,0.893632838246333,0.649527073294073,-0.0466265337101809 +292,2,4,0.893632838246333,1.23151636406837,2.32570939009992 +292,3,4,0.893632838246333,1.67383795568239,2.78685583010959 +292,4,4,0.893632838246333,1.18427970593881,2.58308217320773 +292,5,4,0.893632838246333,1.10217795369724,4.17545088830462 +293,1,4,-0.17329984570749,-0.495703964986854,0.426598483323649 +293,2,4,-0.17329984570749,-0.20694697270213,1.43844736703838 +293,3,4,-0.17329984570749,0.545163558474171,3.17427049488256 +293,4,4,-0.17329984570749,0.401595214074252,3.6781335251007 +293,5,4,-0.17329984570749,0.177816834539147,3.23938527767826 +294,1,4,0.326497399280871,-0.13472193785092,1.25809685074472 +294,2,4,0.326497399280871,0.034022762598704,2.84557217537418 +294,3,4,0.326497399280871,0.598162799541413,1.46014716927973 +294,4,4,0.326497399280871,0.126200469466238,3.65678460464985 +294,5,4,0.326497399280871,0.164798225218632,5.61297214487013 +295,1,4,-1.69765547653391,-0.154280889366906,2.75895647414194 +295,2,4,-1.69765547653391,0.688766738833268,2.00287035833896 +295,3,4,-1.69765547653391,-0.476175631328628,2.83643637564895 +295,4,4,-1.69765547653391,-0.427259753858562,3.00758562895917 +295,5,4,-1.69765547653391,0.0358480577355416,4.42414645411934 +296,1,4,0.633650933641627,-0.821113587145336,-0.195984269286924 +296,2,4,0.633650933641627,-0.0816294051250152,0.955935691449413 +296,3,4,0.633650933641627,0.46272800410014,3.04460414246167 +296,4,4,0.633650933641627,1.24145014749022,2.25349804152533 +296,5,4,0.633650933641627,0.91046353957579,4.10649106176556 +297,1,4,0.573362320259365,0.211954671830595,-1.22089638368291 +297,2,4,0.573362320259365,0.531850561306936,-1.36805062084361 +297,3,4,0.573362320259365,0.294502809667766,-0.368782377037484 +297,4,4,0.573362320259365,1.6726363990039,3.37390164758102 +297,5,4,0.573362320259365,0.904425320734592,4.48803997278305 +298,1,4,0.777482844206898,0.532731775655578,-0.69093420369047 +298,2,4,0.777482844206898,1.23200658222154,2.17322151719419 +298,3,4,0.777482844206898,1.11553239844212,1.00522514103251 +298,4,4,0.777482844206898,1.50951732571487,2.41323059679681 +298,5,4,0.777482844206898,1.70040147775544,3.75537833523442 +299,1,4,0.662331180984266,0.822473966761673,-0.573524004218574 +299,2,4,0.662331180984266,-0.150138919613316,0.704459464206402 +299,3,4,0.662331180984266,0.912328456619516,2.77438102760412 +299,4,4,0.662331180984266,0.677139268370734,3.18581770685075 +299,5,4,0.662331180984266,1.45162176721901,3.32255926170751 +300,1,4,-1.7765642769854,0.0682655892671309,-2.41606241019692 +300,2,4,-1.7765642769854,-0.596264264781815,-2.58896938493558 +300,3,4,-1.7765642769854,0.244954817789035,-2.86925495415941 +300,4,4,-1.7765642769854,0.716148966232101,0.454508967483654 +300,5,4,-1.7765642769854,-0.508267542335298,0.917821961357774 diff --git a/benchmarks/data/twfeweights_unbalanced_panel.csv b/benchmarks/data/twfeweights_unbalanced_panel.csv new file mode 100644 index 000000000..4283b98c8 --- /dev/null +++ b/benchmarks/data/twfeweights_unbalanced_panel.csv @@ -0,0 +1,1501 @@ +"unit","period","first_treat","x1","xtv","outcome" +1,1,0,0.762280032796771,0.22170929102064,0.32572261663086 +1,2,0,0.762280032796771,1.68545437574437,0.32042267370171 +1,3,0,0.762280032796771,0.783974881767155,1.70068782526904 +1,4,0,0.762280032796771,0.151811746643227,2.97713297459989 +1,5,0,0.762280032796771,1.2007958635123,2.34523214422804 +1,6,0,0.762280032796771,1.78117297800612,3.02378983652724 +2,1,0,-0.503334604664531,0.003264242668709,1.68714543541815 +2,2,0,-0.503334604664531,-0.01160465898369,2.11519135601787 +2,3,0,-0.503334604664531,0.431326801314968,2.52345814716303 +2,4,0,-0.503334604664531,0.770960476173647,1.13187660893405 +2,5,0,-0.503334604664531,0.557599780535241,3.50475895194589 +2,6,0,-0.503334604664531,0.882451795015903,2.56076108897434 +3,1,0,0.639896047927709,-0.0821545846502019,0.575850883278317 +3,2,0,0.639896047927709,0.420510283615455,1.46869634270462 +3,3,0,0.639896047927709,1.13004442178635,1.12926487305912 +3,4,0,0.639896047927709,1.54970030934366,3.75188178767779 +3,5,0,0.639896047927709,1.33314318152428,3.63591530837704 +3,6,0,0.639896047927709,2.22273056340426,4.58792223153239 +4,1,0,0.550362540546485,0.365760271855874,1.09027190546647 +4,2,0,0.550362540546485,0.832282303943132,3.26795580759371 +4,3,0,0.550362540546485,0.308471950710698,2.58526381483487 +4,4,0,0.550362540546485,1.56094152897056,2.56528027385053 +4,5,0,0.550362540546485,0.975725496813423,4.77071046622743 +4,6,0,0.550362540546485,2.03906796710312,2.64882999446538 +5,1,0,-1.29901420371325,-0.460999451990144,-0.354880543831845 +5,2,0,-1.29901420371325,0.501202768546227,1.69427801668797 +5,3,0,-1.29901420371325,-0.71046414492934,-0.134441277863296 +5,4,0,-1.29901420371325,-0.404803176271545,0.955977315050681 +5,5,0,-1.29901420371325,1.19643256869914,0.94227680755701 +5,6,0,-1.29901420371325,0.302674962248639,0.661806868491084 +6,1,0,-0.28270754060892,0.509984546718241,-1.60615679868968 +6,2,0,-0.28270754060892,0.795840872958493,1.42057064874697 +6,3,0,-0.28270754060892,0.498743320252149,2.56986086833164 +6,4,0,-0.28270754060892,0.444741791228927,2.43785042499056 +6,5,0,-0.28270754060892,0.0659811927562748,2.03911739958511 +6,6,0,-0.28270754060892,0.434119468978059,2.66364554745197 +7,1,0,-0.228457094854756,-0.125665405042128,-1.50029270012919 +7,2,0,-0.228457094854756,0.579836444928065,-0.271374116596818 +7,3,0,-0.228457094854756,-0.125065661168389,0.134719536136669 +7,4,0,-0.228457094854756,0.540215036454454,1.27719558483555 +7,5,0,-0.228457094854756,0.78024096522406,1.91860134504533 +7,6,0,-0.228457094854756,1.08320687793625,2.14194113990727 +8,1,0,0.904243934680486,1.63054948591081,2.44848344757429 +8,2,0,0.904243934680486,1.12467480171044,1.73465701246783 +8,3,0,0.904243934680486,1.14298117599048,3.13058617549245 +8,4,0,0.904243934680486,1.34982399747455,5.14687438054722 +8,5,0,0.904243934680486,1.24058911320076,5.20947125423497 +8,6,0,0.904243934680486,1.91183461909463,6.66336910996244 +9,1,0,1.0703229218455,0.74531880575199,-1.01317106446202 +9,2,0,1.0703229218455,1.76213986405735,0.295535989411215 +9,3,0,1.0703229218455,0.996599825632608,3.73789484050547 +9,4,0,1.0703229218455,1.5717667357908,3.12292622990361 +9,5,0,1.0703229218455,1.53030746151368,3.66172079419978 +9,6,0,1.0703229218455,2.28600306767279,4.01915059847461 +10,1,0,-0.143983755489064,-0.231058747953471,0.202498691570403 +10,2,0,-0.143983755489064,-0.459455998164224,2.02623199895792 +10,3,0,-0.143983755489064,0.696408286328998,1.20596875375505 +10,4,0,-0.143983755489064,-0.00560727561101249,2.99415578809309 +10,5,0,-0.143983755489064,1.05054222823483,1.7483927621027 +10,6,0,-0.143983755489064,2.11038914740653,1.80106981071404 +11,1,0,0.586058897064396,0.547033869273745,-0.104895161161322 +11,2,0,0.586058897064396,0.782832252459021,-0.858059059928484 +11,3,0,0.586058897064396,0.979859018356124,-0.889891395153455 +11,4,0,0.586058897064396,1.13205981238157,0.418028183041052 +11,5,0,0.586058897064396,1.94312488363535,2.62673794895776 +11,6,0,0.586058897064396,1.34556107555346,1.69235327979137 +12,1,0,-0.0328133420909616,-0.718664318284347,1.01256054018714 +12,2,0,-0.0328133420909616,0.820872201858118,-0.951074969175408 +12,3,0,-0.0328133420909616,-0.123187668525215,0.899700408876431 +12,4,0,-0.0328133420909616,0.78432993049315,-0.16904574962847 +12,5,0,-0.0328133420909616,1.5789293870534,2.50565159085796 +12,6,0,-0.0328133420909616,1.13043900063265,2.18474286754862 +13,1,0,-0.261074291075106,-0.886304628526536,0.190059587043397 +13,2,0,-0.261074291075106,1.18300828926824,1.73607216665735 +13,3,0,-0.261074291075106,0.558340815315284,2.39851236830549 +13,4,0,-0.261074291075106,0.677849256703997,1.93832673654187 +13,5,0,-0.261074291075106,1.24279144956035,4.16803650110529 +13,6,0,-0.261074291075106,1.60879506475197,1.84949612465038 +14,1,0,-2.01934334017198,-0.823487619952887,0.965002660772585 +14,2,0,-2.01934334017198,-0.707032294505178,0.205238668207667 +14,3,0,-2.01934334017198,-0.61048067361937,-0.481964402432323 +14,4,0,-2.01934334017198,-0.471995841450106,0.539239702125543 +14,5,0,-2.01934334017198,0.20463530567174,0.600661309672454 +14,6,0,-2.01934334017198,0.563781281949436,0.603161656459746 +15,1,0,0.361957454931056,0.0189565591765361,-0.129555386494278 +15,2,0,0.361957454931056,0.106085329301318,0.425642861619322 +15,3,0,0.361957454931056,0.451190373474882,2.23576139017725 +15,4,0,0.361957454931056,1.28538019707699,4.29088314743058 +15,5,0,0.361957454931056,1.20237818127509,3.214356355133 +15,6,0,0.361957454931056,1.4677696966727,4.14629939829314 +16,1,0,-0.599632942668391,-0.0865504039587467,-2.49861640755116 +16,2,0,-0.599632942668391,0.730243732022659,0.509031417314836 +16,3,0,-0.599632942668391,-0.152765685660319,0.369550346158571 +16,4,0,-0.599632942668391,0.437564273403713,0.0251070431731277 +16,5,0,-0.599632942668391,1.25673745360795,-1.26356621386106 +16,6,0,-0.599632942668391,0.396948169632596,-1.19812926828942 +17,1,0,0.198694952835121,1.48026675231997,-0.0914373018488306 +17,2,0,0.198694952835121,0.602080106111401,-1.13650828313566 +17,3,0,0.198694952835121,0.66717979268474,1.28036383890753 +17,4,0,0.198694952835121,0.744336945605951,0.31249515532039 +17,5,0,0.198694952835121,0.734442176603276,3.41280747785275 +17,6,0,0.198694952835121,1.14792728063429,0.520393061796443 +18,1,0,-0.560769887816829,0.740566418840177,-1.13242281217491 +18,2,0,-0.560769887816829,-0.800802303021818,-1.39936169368382 +18,3,0,-0.560769887816829,1.64284271435801,-1.02033967144853 +18,4,0,-0.560769887816829,-0.33064398708322,0.436977600914218 +18,5,0,-0.560769887816829,1.52038443046078,1.20604277293126 +18,6,0,-0.560769887816829,1.07184454667575,0.846352514375142 +19,1,0,0.426318371985953,0.615622372338839,1.52561631350584 +19,2,0,0.426318371985953,1.1829040046622,3.60645868325585 +19,3,0,0.426318371985953,0.60962893875821,2.6921711792421 +19,4,0,0.426318371985953,1.43857945487693,3.86330852828638 +19,5,0,0.426318371985953,1.66282398015716,4.83728207603734 +19,6,0,0.426318371985953,1.40236230094667,4.44066588875079 +20,1,0,0.0661747368407101,-0.468916572476273,0.944687218273834 +20,2,0,0.0661747368407101,0.0698315654648725,2.09812105308828 +20,3,0,0.0661747368407101,1.0687390568495,3.04216485731978 +20,4,0,0.0661747368407101,1.15786295583289,3.54263125509707 +20,5,0,0.0661747368407101,1.27377826018422,1.96382561369836 +20,6,0,0.0661747368407101,0.499986760723536,3.63781888104593 +21,1,0,0.836932519114268,1.19554747601385,0.744833504737609 +21,2,0,0.836932519114268,0.859628392812328,2.36872962935447 +21,3,0,0.836932519114268,0.748417270018536,2.15303566865993 +21,4,0,0.836932519114268,1.59274446404155,3.29582632329775 +21,5,0,0.836932519114268,1.4555910021538,4.40482843122191 +21,6,0,0.836932519114268,1.58763855557056,4.8251150631344 +22,1,0,0.0941874351853353,0.466497866418425,2.31236487898151 +22,2,0,0.0941874351853353,0.261894899289606,1.20207512086356 +22,3,0,0.0941874351853353,-0.199883909412536,2.61279927072359 +22,4,0,0.0941874351853353,1.06060738068883,2.92500959946553 +22,5,0,0.0941874351853353,0.83115437138983,2.86255384244105 +22,6,0,0.0941874351853353,0.547611602018412,3.86553389086576 +23,1,0,2.24292067852269,1.65272226128883,0.512316531737125 +23,2,0,2.24292067852269,1.10093214898091,0.278294515654655 +23,3,0,2.24292067852269,1.91261347599136,1.64154746116044 +23,4,0,2.24292067852269,1.70894499719591,3.9889543067309 +23,5,0,2.24292067852269,1.88316975411293,5.14776444387616 +23,6,0,2.24292067852269,2.22298347240385,5.60218491495817 +24,1,0,-1.35767302263554,-0.532569922147083,0.854035384193373 +24,2,0,-1.35767302263554,0.293236980944925,0.859939323644371 +24,3,0,-1.35767302263554,0.400452813605349,-0.601293901404389 +24,4,0,-1.35767302263554,-0.0464606324392122,0.377478681201744 +24,5,0,-1.35767302263554,1.30355241324811,0.0307348313527933 +24,6,0,-1.35767302263554,0.640401721123196,-0.291861720912245 +25,1,0,0.536762929066239,0.494542956395155,1.13128088027847 +25,2,0,0.536762929066239,0.413659545685529,0.175996686768765 +25,3,0,0.536762929066239,0.951090791785006,0.89142621783185 +25,4,0,0.536762929066239,1.42138214987661,1.83478859337311 +25,5,0,0.536762929066239,0.197687553568442,3.82746389246305 +25,6,0,0.536762929066239,0.997008528348205,3.29347248061781 +26,1,0,0.080424492077766,0.55057496426079,-1.57794939371139 +26,2,0,0.080424492077766,0.857425330645063,0.204173346072951 +26,3,0,0.080424492077766,0.435846052366296,0.467273164232521 +26,4,0,0.080424492077766,0.984228608534135,-0.358590158898051 +26,5,0,0.080424492077766,1.72601324978517,-0.249224951584627 +26,6,0,0.080424492077766,1.09283259694636,2.05876710749068 +27,1,0,-0.661809943463178,-0.351151472617987,1.3332812259713 +27,2,0,-0.661809943463178,-0.769094233439663,1.02937579325852 +27,3,0,-0.661809943463178,0.529301492309873,0.670624009578854 +27,4,0,-0.661809943463178,0.340211950424195,2.83276083639353 +27,5,0,-0.661809943463178,0.672804544960685,1.33367085251827 +27,6,0,-0.661809943463178,0.836997883685823,1.64463392166496 +28,1,0,1.18763780487985,0.857112953615082,0.918371137986335 +28,2,0,1.18763780487985,0.609455036547718,-0.1953916519845 +28,3,0,1.18763780487985,0.957415327060119,1.64710330460386 +28,4,0,1.18763780487985,1.70678167693069,2.82206969876212 +28,5,0,1.18763780487985,2.28994975704781,2.81441799704668 +28,6,0,1.18763780487985,2.70532991473682,6.64412995317486 +29,1,0,0.171569793047149,-0.398831901085214,0.103682763935853 +29,2,0,0.171569793047149,0.185275047950682,0.183298060409325 +29,3,0,0.171569793047149,1.16669238180421,1.92277150882252 +29,4,0,0.171569793047149,1.12694852976321,1.54493766913653 +29,5,0,0.171569793047149,1.04381631290082,4.12170013580083 +29,6,0,0.171569793047149,1.43494779620057,2.35016615646919 +30,1,0,0.214899638814364,0.990047580435093,0.0329317907015836 +30,2,0,0.214899638814364,0.00318616298013463,3.1538826374806 +30,3,0,0.214899638814364,-0.146108580250217,1.40058443711373 +30,4,0,0.214899638814364,0.538522428274927,3.45129621770057 +30,5,0,0.214899638814364,1.52956640764869,4.19427236133207 +30,6,0,0.214899638814364,1.99778295611785,3.56034464827219 +31,1,0,2.32254390100494,1.15644468484223,2.77259843088733 +31,2,0,2.32254390100494,1.7483266658964,2.02564757997795 +31,3,0,2.32254390100494,2.2699980030887,1.64606771516423 +31,4,0,2.32254390100494,2.71526529576935,3.37656661650927 +31,5,0,2.32254390100494,2.46194873478394,6.10188976491978 +31,6,0,2.32254390100494,2.97510578028662,8.52554433662228 +32,1,0,-0.169502937020473,0.693215490975842,2.05141406053865 +32,2,0,-0.169502937020473,0.468165521466207,4.61437618451743 +32,3,0,-0.169502937020473,0.205806456493878,1.11339736229636 +32,4,0,-0.169502937020473,1.25812023117349,4.1166099867286 +32,5,0,-0.169502937020473,1.44654158298815,3.40523553515562 +32,6,0,-0.169502937020473,1.31004230358703,1.01835012296617 +33,1,0,0.552770969821111,-0.306006044577907,1.25412066983468 +33,2,0,0.552770969821111,0.995602390305268,1.80917798576773 +33,3,0,0.552770969821111,1.01655508934542,0.757291376613842 +33,4,0,0.552770969821111,1.1036479557666,2.96417691332605 +33,5,0,0.552770969821111,1.45864948387186,2.46197479844314 +33,6,0,0.552770969821111,0.859567156764491,5.97071719266024 +34,1,0,0.481846461393583,0.269302303860423,0.526376540360734 +34,2,0,0.481846461393583,0.0461613847557073,2.19364928878128 +34,3,0,0.481846461393583,1.12068944272412,2.44731864429638 +34,4,0,0.481846461393583,0.783270153094354,2.52781800340864 +34,5,0,0.481846461393583,1.97211005145252,3.13973621130369 +34,6,0,0.481846461393583,1.03839429696509,2.94271775797344 +35,1,0,0.180785291726702,0.590217475976332,2.01882790659905 +35,2,0,0.180785291726702,0.408881961264579,4.08885751899202 +35,3,0,0.180785291726702,0.581997504350181,3.11714635871518 +35,4,0,0.180785291726702,1.49965041737788,5.38773518005293 +35,5,0,0.180785291726702,2.02563496208288,5.9748136314136 +35,6,0,0.180785291726702,2.28603451086318,2.7474872302793 +36,1,0,0.937530562434575,0.211835239755195,0.264811770320571 +36,2,0,0.937530562434575,0.97223160253963,3.72358171039763 +36,3,0,0.937530562434575,2.03844428066716,2.52837086994306 +36,4,0,0.937530562434575,1.35584019846206,3.46668468173737 +36,5,0,0.937530562434575,2.31452132526835,4.30663345338415 +36,6,0,0.937530562434575,1.52220257385175,5.90295382804056 +37,1,0,0.605297706819014,1.33501016334775,-1.22155851591522 +37,2,0,0.605297706819014,0.273646824543883,0.94685268474313 +37,3,0,0.605297706819014,1.18861405551878,-1.06749360477619 +37,4,0,0.605297706819014,0.430022662342312,2.49683112338337 +37,5,0,0.605297706819014,0.980782441057555,2.87859515158806 +37,6,0,0.605297706819014,1.68491715613623,3.18312049166695 +38,1,0,-0.150046075996628,0.907283121370997,-0.269990850232008 +38,2,0,-0.150046075996628,0.578967394515049,-0.886845866605914 +38,3,0,-0.150046075996628,1.31322956121607,2.17397969319074 +38,4,0,-0.150046075996628,0.770060624548639,0.629084998485956 +38,5,0,-0.150046075996628,0.858759768922271,0.248192249762817 +38,6,0,-0.150046075996628,1.31157900399226,3.13842264488913 +39,1,0,-1.53940169474244,0.386233806928156,-0.523153781143094 +39,2,0,-1.53940169474244,-0.184038667457276,-1.58742257093027 +39,3,0,-1.53940169474244,-0.973488223644379,-0.749417744231022 +39,4,0,-1.53940169474244,0.510965070488413,-0.735701532265713 +39,5,0,-1.53940169474244,0.766370381872722,-1.75342467837839 +39,6,0,-1.53940169474244,0.687218453403146,0.252169837110935 +40,1,0,0.391843330512968,0.704055957164002,0.409086287780538 +40,2,0,0.391843330512968,0.519946271805668,-1.89191667178992 +40,3,0,0.391843330512968,0.432251147082645,0.648330099168622 +40,4,0,0.391843330512968,1.21713541238615,1.31189584887626 +40,5,0,0.391843330512968,1.27472438671965,0.103667887130483 +40,6,0,0.391843330512968,1.66786350472797,0.685042322957091 +41,1,0,-0.0373260431491352,0.0317324594708311,-0.747942068188531 +41,2,0,-0.0373260431491352,0.558761507456241,-1.19204731827254 +41,3,0,-0.0373260431491352,0.627774174828559,-0.00444322772313305 +41,4,0,-0.0373260431491352,1.22057537184321,0.0157895549428625 +41,5,0,-0.0373260431491352,-0.13463140571047,2.03834883872972 +41,6,0,-0.0373260431491352,1.26326673119073,2.38013129655104 +42,1,0,0.827626116517122,0.619582666001477,2.05284651791961 +42,2,0,0.827626116517122,0.728960630288116,2.63692144330927 +42,3,0,0.827626116517122,1.51573241849325,2.82734402300804 +42,4,0,0.827626116517122,2.32134629984684,4.70066367856648 +42,5,0,0.827626116517122,1.26080310077716,2.71057083119057 +42,6,0,0.827626116517122,1.56560512998824,4.83856397972047 +43,1,0,-0.665331979033663,-0.225102401789033,2.56456608500982 +43,2,0,-0.665331979033663,-0.553172893008135,4.61457088107317 +43,3,0,-0.665331979033663,0.972861442802118,1.82443898633191 +43,4,0,-0.665331979033663,0.32326929160598,2.64950447327976 +43,5,0,-0.665331979033663,0.306489791796068,2.60561890553988 +43,6,0,-0.665331979033663,1.03909552234328,3.64564011990368 +44,1,0,0.942288464073336,0.00874783812863711,0.056977549744632 +44,2,0,0.942288464073336,0.596281511258369,1.12468821275525 +44,3,0,0.942288464073336,1.39468851953371,2.90909190594441 +44,4,0,0.942288464073336,2.11675576968934,4.23166721482138 +44,5,0,0.942288464073336,1.58330641741654,5.43308550177595 +44,6,0,0.942288464073336,1.16350891633872,4.71261506783713 +45,1,0,-0.0809416298757052,0.807692153394734,1.0789586456094 +45,2,0,-0.0809416298757052,0.752887274713188,0.785720766407489 +45,3,0,-0.0809416298757052,1.24054098972764,2.55331843603172 +45,4,0,-0.0809416298757052,1.34412295664021,2.48536763140985 +45,5,0,-0.0809416298757052,-0.0307181568992582,1.1879236197245 +45,6,0,-0.0809416298757052,1.78022693303965,3.71848890245285 +46,1,0,0.521163361059873,-0.723702502547912,1.09867938696003 +46,2,0,0.521163361059873,1.1499608686645,3.55842561079306 +46,3,0,0.521163361059873,1.15232123544905,3.17634759665484 +46,4,0,0.521163361059873,1.42926586195998,3.39003436513758 +46,5,0,0.521163361059873,0.879129584097901,2.80180340599266 +46,6,0,0.521163361059873,0.914156746191509,2.46863004623085 +47,1,0,1.61466175286344,0.643006559192123,0.836204718894284 +47,2,0,1.61466175286344,1.13661030532442,2.30379657813442 +47,3,0,1.61466175286344,1.19295732982162,2.64086502103959 +47,4,0,1.61466175286344,0.239328559267526,3.65153083011424 +47,5,0,1.61466175286344,1.68375842695897,5.33901616867935 +47,6,0,1.61466175286344,2.02785308242104,4.85439287872028 +48,1,0,-1.7565659395326,-0.218238640820971,1.78646195020932 +48,2,0,-1.7565659395326,-0.969625592296388,0.784269865689745 +48,3,0,-1.7565659395326,-0.419721278011531,2.20671186654619 +48,4,0,-1.7565659395326,-0.0179174885255089,0.521674280505722 +48,5,0,-1.7565659395326,0.0262037737083689,0.883000983935286 +48,6,0,-1.7565659395326,0.434997896953442,1.76393693136589 +49,1,0,0.0236704073602128,0.342406228293342,-0.0627973687877335 +49,2,0,0.0236704073602128,1.339042197954,0.510408069568147 +49,3,0,0.0236704073602128,0.396305118083889,0.128070410293366 +49,4,0,0.0236704073602128,0.301229123953579,2.34822787611627 +49,5,0,0.0236704073602128,1.84984351148937,2.02095085612302 +49,6,0,0.0236704073602128,1.14237872441368,3.88976774422889 +50,1,0,-1.05243771447958,0.290206750660849,-0.27631290661132 +50,2,0,-1.05243771447958,-0.331976147509505,0.0784093174185179 +50,3,0,-1.05243771447958,0.644389922826399,1.85558157163888 +50,4,0,-1.05243771447958,-0.0363544318974775,1.14349827947337 +50,5,0,-1.05243771447958,-0.63902084988463,0.0282478533308865 +50,6,0,-1.05243771447958,0.277763327171488,1.60530873660652 +51,1,0,1.41893961966826,1.72807036319162,1.23790039735326 +51,2,0,1.41893961966826,1.68895995675209,2.07276778859 +51,3,0,1.41893961966826,0.971263584985605,3.6394918544787 +51,4,0,1.41893961966826,1.11899195431195,3.41900141304584 +51,5,0,1.41893961966826,1.87654473407966,4.68940054213404 +51,6,0,1.41893961966826,3.04105530464514,4.8924758822659 +52,1,0,-1.1910633756242,0.200524510461652,-2.7265059697081 +52,2,0,-1.1910633756242,-1.2115909345573,-1.40963894581555 +52,3,0,-1.1910633756242,0.386175892807225,-1.47451140247512 +52,4,0,-1.1910633756242,1.02789828620209,0.146871306849964 +52,5,0,-1.1910633756242,-0.273264466564166,-0.0141430110384286 +52,6,0,-1.1910633756242,0.578879852223559,-1.17416016414042 +53,1,0,0.195514781381017,-0.742554152921851,2.23579400276791 +53,2,0,0.195514781381017,0.23337503458647,1.10762600155431 +53,3,0,0.195514781381017,0.263301882945963,2.51597948959044 +53,4,0,0.195514781381017,-0.199791624518697,3.42840858712109 +53,5,0,0.195514781381017,1.60088371927206,1.87400311570008 +53,6,0,0.195514781381017,1.23132242768948,2.10936229576561 +54,1,0,0.111145488608917,0.41248587564804,-1.23962257171445 +54,2,0,0.111145488608917,0.650566893014723,0.757366835792272 +54,3,0,0.111145488608917,0.638921948130227,-0.317803478051985 +54,4,0,0.111145488608917,-0.29731490602435,0.682731490142251 +54,5,0,0.111145488608917,1.38608083061868,1.22724299967546 +54,6,0,0.111145488608917,1.38422425480489,3.12874530862489 +55,1,0,0.666862478709229,1.66603184966515,-0.96199867245643 +55,2,0,0.666862478709229,0.633112902252682,-1.17037332152456 +55,3,0,0.666862478709229,0.343015660916893,0.874503842048745 +55,4,0,0.666862478709229,0.770817827624448,-0.0109943396516543 +55,5,0,0.666862478709229,0.964533553442544,1.71046636135371 +55,6,0,0.666862478709229,1.76201281537328,1.8951412953794 +56,1,0,-1.77404829500802,-1.40580752868787,1.39863275680412 +56,2,0,-1.77404829500802,0.870111926030201,-0.982599170611262 +56,3,0,-1.77404829500802,-0.830299171612467,-0.778719888508792 +56,4,0,-1.77404829500802,0.334381341446428,2.10682625136074 +56,5,0,-1.77404829500802,0.0402025096737488,0.21272995862292 +56,6,0,-1.77404829500802,-0.102134961440965,-0.482035354350842 +57,1,0,-1.69162901504091,-0.937011635623388,-1.822960278522 +57,2,0,-1.69162901504091,-0.854090558628001,0.246599401891903 +57,3,0,-1.69162901504091,0.306483912711003,-1.10550309471832 +57,4,0,-1.69162901504091,-0.375186713716243,-0.403217811110663 +57,5,0,-1.69162901504091,-0.0372694866470827,-1.2397278955316 +57,6,0,-1.69162901504091,0.042822212170315,-1.72546983425334 +58,1,0,-0.849031394079241,-0.365462147353985,-0.312601656922766 +58,2,0,-0.849031394079241,-0.0832361702771065,0.103741778870931 +58,3,0,-0.849031394079241,0.142581169967471,-0.482286945873843 +58,4,0,-0.849031394079241,0.460384860409781,1.40366770538517 +58,5,0,-0.849031394079241,0.996609986700584,0.871624823978852 +58,6,0,-0.849031394079241,0.85264265006985,1.89123646594172 +59,1,0,1.21025798910521,1.04847972204707,-0.575583913121781 +59,2,0,1.21025798910521,1.38937084327584,1.37186521680745 +59,3,0,1.21025798910521,1.2547790752355,0.529851722231607 +59,4,0,1.21025798910521,1.17327232495747,1.47840784888763 +59,5,0,1.21025798910521,1.172880927698,3.91882955565966 +59,6,0,1.21025798910521,1.39574007441178,2.28500196266698 +60,1,0,0.789386089516239,0.179951915237443,0.435733939422799 +60,2,0,0.789386089516239,0.438493621227776,1.47709316141718 +60,3,0,0.789386089516239,1.21539128818204,3.37967447648071 +60,4,0,0.789386089516239,1.76191512346368,2.94762787653769 +60,5,0,0.789386089516239,1.35758016131045,2.48842314882934 +60,6,0,0.789386089516239,1.50244938572149,4.71258184189825 +61,1,0,1.15707226166531,0.695992482122723,0.366873356023154 +61,2,0,1.15707226166531,1.30843852180442,0.25928295987725 +61,3,0,1.15707226166531,1.23140830848896,0.670889744195337 +61,4,0,1.15707226166531,1.52680662397394,3.10031216794104 +61,5,0,1.15707226166531,1.97553314383926,4.52201393669106 +61,6,0,1.15707226166531,1.89587096185583,4.17000041602687 +62,1,0,-0.344242039937823,-0.012607129644549,1.53095259638453 +62,2,0,-0.344242039937823,-0.190659504558242,2.99016158987811 +62,3,0,-0.344242039937823,0.129929099988597,2.55026017406343 +62,4,0,-0.344242039937823,0.75025985280716,0.871172605987694 +62,5,0,-0.344242039937823,-0.088414077084213,2.66185824077775 +62,6,0,-0.344242039937823,1.68941056098916,2.3342515589607 +63,1,0,-0.520156154475325,0.115284773651995,-0.554998777313392 +63,2,0,-0.520156154475325,0.380215358847244,0.198926509461979 +63,3,0,-0.520156154475325,-0.045325702695873,0.599340066190899 +63,4,0,-0.520156154475325,0.403372511379866,3.05428355693047 +63,5,0,-0.520156154475325,0.339830867502976,2.03433756350707 +63,6,0,-0.520156154475325,0.952544521227367,1.17519783476071 +64,1,0,0.203435284900822,0.515493855780151,-0.749085263516488 +64,2,0,0.203435284900822,0.746187070355182,2.1052938488016 +64,3,0,0.203435284900822,0.374895246973636,1.58265151593793 +64,4,0,0.203435284900822,0.506751948936316,2.14495255559874 +64,5,0,0.203435284900822,1.51739942863017,0.95825975606708 +64,6,0,0.203435284900822,1.09147631899005,3.41111423514602 +65,1,0,-0.55341970038927,-0.131380062378949,-1.32195592808742 +65,2,0,-0.55341970038927,-0.241965241577032,-1.02423916263361 +65,3,0,-0.55341970038927,1.15300408189915,-0.535841679982353 +65,4,0,-0.55341970038927,0.0532746291132736,-0.968231722746471 +65,5,0,-0.55341970038927,0.745558003100886,0.832777050424564 +65,6,0,-0.55341970038927,-0.00751406091164042,1.96077688545181 +66,1,0,1.16897479134018,0.184878831174988,-1.28620870245401 +66,2,0,1.16897479134018,0.987928053916087,0.166753297701763 +66,3,0,1.16897479134018,1.27696264162646,1.68247659314794 +66,4,0,1.16897479134018,1.55536067948273,3.87746819914394 +66,5,0,1.16897479134018,2.15354648101136,4.79993387221023 +66,6,0,1.16897479134018,1.71085429487955,4.04950544043591 +67,1,0,-0.311033874726052,0.0263159579110574,0.871500540829715 +67,2,0,-0.311033874726052,0.213806039758833,1.10566090191848 +67,3,0,-0.311033874726052,1.16580579545896,2.62828302528987 +67,4,0,-0.311033874726052,0.941962746311597,1.57046531053 +67,5,0,-0.311033874726052,0.185785078920236,1.0287988657371 +67,6,0,-0.311033874726052,0.0881200987592246,4.48340110608756 +68,1,0,0.609445260914666,0.160720456451015,-0.875045477717211 +68,2,0,0.609445260914666,0.398587442880102,1.97447918279283 +68,3,0,0.609445260914666,0.855870863727989,1.76870737136006 +68,4,0,0.609445260914666,1.43803622432617,3.07264275168721 +68,5,0,0.609445260914666,1.53294248674142,3.1268484428726 +68,6,0,0.609445260914666,2.30216153797432,4.90626806484531 +69,1,0,-1.70140877068258,-0.772137510934144,-3.67033050406739 +69,2,0,-1.70140877068258,-0.0887373122839057,0.00741645039112426 +69,3,0,-1.70140877068258,0.23462709293423,-0.35143639642592 +69,4,0,-1.70140877068258,-0.0133450385366955,-3.11261503664916 +69,5,0,-1.70140877068258,0.647212644285179,-1.22067385183629 +69,6,0,-1.70140877068258,-0.318090934527394,-1.7703701960703 +70,1,0,-0.701528235373902,0.419756517766687,-0.392170361021321 +70,2,0,-0.701528235373902,1.00520991718909,-1.24465679664118 +70,3,0,-0.701528235373902,0.397714193248576,1.73346811162478 +70,4,0,-0.701528235373902,0.141784792120088,2.62165510171308 +70,5,0,-0.701528235373902,-0.229555991095407,1.66547134257899 +70,6,0,-0.701528235373902,0.457270807196878,3.5583092350656 +71,1,0,0.505874716504533,0.0328685484582639,1.22528702490137 +71,2,0,0.505874716504533,1.59522458519428,2.23505630653046 +71,3,0,0.505874716504533,1.76396196449465,4.50976634967245 +71,4,0,0.505874716504533,2.02663440047412,5.34020271673294 +71,5,0,0.505874716504533,1.16929077274353,5.67948718050403 +71,6,0,0.505874716504533,0.987516672556885,4.01479530785995 +72,1,0,-0.844325706369891,-0.160769095993306,1.77828123515083 +72,2,0,-0.844325706369891,-0.626002441128783,2.86943080366521 +72,3,0,-0.844325706369891,0.624503195437875,1.13819663453126 +72,4,0,-0.844325706369891,0.4128826402263,2.34116236266462 +72,5,0,-0.844325706369891,0.64321904824842,6.00376292914209 +72,6,0,-0.844325706369891,1.30388473771754,4.39817205983043 +73,1,0,0.35463969852429,0.175853581251931,1.54901516888919 +73,2,0,0.35463969852429,0.255347723083929,3.75759086722241 +73,3,0,0.35463969852429,0.842048899742575,2.74674519300349 +73,4,0,0.35463969852429,0.843858032144758,3.48463769028479 +73,5,0,0.35463969852429,2.02272986936328,2.63274428096982 +73,6,0,0.35463969852429,1.756515220362,3.36491283175246 +74,1,0,0.634049900150801,1.06843050846366,0.631806643103645 +74,2,0,0.634049900150801,1.09459023941769,1.10185630635582 +74,3,0,0.634049900150801,1.16047718202649,2.21277078081482 +74,4,0,0.634049900150801,0.194209434370495,4.49679293334212 +74,5,0,0.634049900150801,1.57111541788119,3.78301141707104 +74,6,0,0.634049900150801,1.8709185053691,3.83801701206847 +75,1,0,-0.144614154426945,-0.1812136320715,-1.50303099629943 +75,2,0,-0.144614154426945,0.0812045686177193,-0.30813252539654 +75,3,0,-0.144614154426945,0.592606669784812,-0.87606327177109 +75,4,0,-0.144614154426945,2.21284582539953,1.49034525451696 +75,5,0,-0.144614154426945,0.89070743973123,1.99305042503676 +75,6,0,-0.144614154426945,1.22531924408363,1.04904749337575 +76,1,0,-0.965620194716236,-0.416127905137339,-0.767970743384194 +76,2,0,-0.965620194716236,0.895930664804138,-0.185344950505385 +76,3,0,-0.965620194716236,0.77433449793566,0.860449806370312 +76,4,0,-0.965620194716236,0.70085446817956,1.07172526057056 +76,5,0,-0.965620194716236,1.05852067016297,0.51916850372457 +76,6,0,-0.965620194716236,0.638859520759945,1.33787453176867 +77,1,0,-1.86580715673843,-0.391278486536145,-0.0920440713171948 +77,2,0,-1.86580715673843,-1.3757502672685,-0.560366699602578 +77,3,0,-1.86580715673843,0.308335534756901,-0.371286061286313 +77,4,0,-1.86580715673843,0.345233743814379,-0.20407441671565 +77,5,0,-1.86580715673843,0.381411993188402,-1.14332176885771 +77,6,0,-1.86580715673843,-0.584252643983715,-1.27669748966723 +78,1,0,-1.36945985986056,-0.317388985722178,0.757961675843857 +78,2,0,-1.36945985986056,-0.931319605141151,-1.47231495687832 +78,3,0,-1.36945985986056,0.64939703346783,0.947760079117256 +78,4,0,-1.36945985986056,0.141848675286346,0.427700713693576 +78,5,0,-1.36945985986056,0.645979505797502,0.123611381550555 +78,6,0,-1.36945985986056,0.853461304724385,-1.30724575909746 +79,1,0,-1.4085132336069,-1.4450148783583,0.727494818634948 +79,2,0,-1.4085132336069,-0.599025158604699,0.291135574546722 +79,3,0,-1.4085132336069,-0.406989701850758,0.882944383939482 +79,4,0,-1.4085132336069,-0.0202306741507156,0.290743072184474 +79,5,0,-1.4085132336069,0.418122082147065,-0.554654948017118 +79,6,0,-1.4085132336069,0.161492862233551,-0.104305091084399 +80,1,0,1.0465570019994,0.83467245617602,1.79474289967135 +80,2,0,1.0465570019994,0.112243844479774,2.42789304208095 +80,3,0,1.0465570019994,0.333555656213314,4.11924338896377 +80,4,0,1.0465570019994,1.01788286394161,3.88357495448787 +80,5,0,1.0465570019994,1.53660910957585,4.5182271100794 +80,6,0,1.0465570019994,1.62573779201925,6.09413820742505 +81,1,0,-1.41453829223088,-0.422101284770018,0.769101887017708 +81,2,0,-1.41453829223088,-0.315168451954516,0.729566856970148 +81,3,0,-1.41453829223088,-0.272040188359851,-0.671347616711595 +81,4,0,-1.41453829223088,0.183840000922275,-0.614516066633574 +81,5,0,-1.41453829223088,0.299215308065219,0.0802272383475152 +81,6,0,-1.41453829223088,0.732800101297085,0.510417085488192 +82,1,0,0.160539536166133,0.959530186512375,-0.215930107181071 +82,2,0,0.160539536166133,-0.326281366208948,1.07948246934508 +82,3,0,0.160539536166133,0.324000083908395,1.52302152579562 +82,4,0,0.160539536166133,-0.258160129826018,1.74745221913454 +82,5,0,0.160539536166133,1.46023105551091,2.33039670539431 +82,6,0,0.160539536166133,0.788817140506763,2.32858808738527 +83,1,0,-0.567801186208016,0.231833577654445,1.9093919950695 +83,2,0,-0.567801186208016,-0.406252558339809,1.71838187145903 +83,3,0,-0.567801186208016,0.500105335834885,0.424872683579041 +83,4,0,-0.567801186208016,1.6055462707676,1.14871256154913 +83,5,0,-0.567801186208016,1.27321097698752,2.60872052340111 +83,6,0,-0.567801186208016,0.495575788605087,1.73434907598337 +84,1,0,-0.782213005252933,-1.24210551325893,0.62824879952858 +84,2,0,-0.782213005252933,0.998799311582334,-0.285355853362829 +84,3,0,-0.782213005252933,0.152353660632812,1.08574270218729 +84,4,0,-0.782213005252933,0.437770170026559,0.992180904182235 +84,5,0,-0.782213005252933,0.555952746039179,2.04941499620541 +84,6,0,-0.782213005252933,0.890162464146768,-0.894620656851764 +85,1,0,-0.487130227305875,0.47292213649988,-0.82059178734108 +85,2,0,-0.487130227305875,0.492301356302156,0.262528389117469 +85,3,0,-0.487130227305875,1.29009677447065,1.17301382240356 +85,4,0,-0.487130227305875,1.43591257033689,2.60956505836633 +85,5,0,-0.487130227305875,1.82677210590248,0.693530436043082 +85,6,0,-0.487130227305875,1.32378375915425,1.21715970534733 +86,1,0,-0.384373119535855,-0.170904424548637,0.498232930318139 +86,2,0,-0.384373119535855,-0.295740805748663,0.301854807254213 +86,3,0,-0.384373119535855,0.578383091390104,0.695014709575832 +86,4,0,-0.384373119535855,0.528019648180887,-0.737125962869525 +86,5,0,-0.384373119535855,1.01985265304516,-0.0830034763200127 +86,6,0,-0.384373119535855,1.67834611857395,2.49504179213634 +87,1,0,0.408020963230945,0.962156203251732,1.94439889343524 +87,2,0,0.408020963230945,0.99999055549027,1.16647176165158 +87,3,0,0.408020963230945,0.970786735481066,1.69236048882497 +87,4,0,0.408020963230945,1.41338326575805,4.12905636501844 +87,5,0,0.408020963230945,1.49273145808118,2.17402526741817 +87,6,0,0.408020963230945,1.37163125752385,5.37627582349371 +88,1,0,0.599584079099597,0.730924181364043,-1.31376569334669 +88,2,0,0.599584079099597,1.44699595117378,-0.623281047649964 +88,3,0,0.599584079099597,0.28253615169462,-0.693242466503484 +88,4,0,0.599584079099597,1.16021618973925,1.01881668516363 +88,5,0,0.599584079099597,0.964519264352473,1.21574324307197 +88,6,0,0.599584079099597,1.84858002344773,2.57058311503664 +89,1,0,-0.311048261021212,0.172929318481353,2.7900643892079 +89,2,0,-0.311048261021212,0.913627245775031,1.24443702713629 +89,3,0,-0.311048261021212,-0.781248428169652,1.03938341222443 +89,4,0,-0.311048261021212,0.602717775024287,0.461064451496916 +89,5,0,-0.311048261021212,0.77172145901647,2.32861716047523 +89,6,0,-0.311048261021212,0.875048468268877,4.46406247718196 +90,1,0,-0.812370112719702,0.508688758563729,0.663812620865317 +90,2,0,-0.812370112719702,0.0978888128213179,0.604934747365124 +90,3,0,-0.812370112719702,-0.227169219633528,0.483064206170982 +90,4,0,-0.812370112719702,-0.0789620142453338,1.30016784410719 +90,5,0,-0.812370112719702,0.474031207045202,-0.35852658877042 +90,6,0,-0.812370112719702,0.362037836323042,2.62002321031834 +91,1,0,-0.835160216707722,0.0821439607244709,-1.31297017293814 +91,2,0,-0.835160216707722,-0.215052170622971,-0.00952574873257295 +91,3,0,-0.835160216707722,-0.515795699189277,0.93911380476138 +91,4,0,-0.835160216707722,0.0994435769261413,0.29534073274764 +91,5,0,-0.835160216707722,0.640632783084201,0.631494983316459 +91,6,0,-0.835160216707722,0.444237884051304,-0.614904284648697 +92,1,0,-0.0347979035615357,-0.425071253260572,-0.276129691013229 +92,2,0,-0.0347979035615357,-0.00558881648247656,2.6914081615219 +92,3,0,-0.0347979035615357,0.12042472026752,0.139596914798406 +92,4,0,-0.0347979035615357,0.19294717753885,-0.0358857413034133 +92,5,0,-0.0347979035615357,0.961964380434135,2.00578635972552 +92,6,0,-0.0347979035615357,1.17135420091513,2.59638281957783 +93,1,0,-0.875486806182028,0.130202709056072,0.0515320316326158 +93,2,0,-0.875486806182028,-1.41949223413023,0.503122856462025 +93,3,0,-0.875486806182028,-0.109791084167903,-1.1180468841845 +93,4,0,-0.875486806182028,0.659965080718873,-0.368352880340779 +93,5,0,-0.875486806182028,0.067354181000394,0.0520547024496831 +93,6,0,-0.875486806182028,1.4060340511004,-1.52569833771551 +94,1,0,-0.257821470178811,0.103041140124536,3.00578229260879 +94,2,0,-0.257821470178811,0.0510455966337183,2.81600739821049 +94,3,0,-0.257821470178811,1.28170651334834,1.88866826165356 +94,4,0,-0.257821470178811,0.524450670730671,4.24222926203413 +94,5,0,-0.257821470178811,0.543757953082137,3.89014943444177 +94,6,0,-0.257821470178811,0.837694358790485,5.33137502521585 +95,1,0,-1.06522102729796,0.338860131607811,1.32692462722341 +95,2,0,-1.06522102729796,-0.127369030427613,-0.0832637940592554 +95,3,0,-1.06522102729796,0.0207887615637769,-0.11264669981747 +95,4,0,-1.06522102729796,0.226974834769246,2.89905849310711 +95,5,0,-1.06522102729796,0.495476577660107,2.54073657860863 +95,6,0,-1.06522102729796,0.231418798298941,0.847408625748489 +96,1,0,0.55656405447025,1.84356052443496,2.97044415315513 +96,2,0,0.55656405447025,0.403515009694641,4.91746159042525 +96,3,0,0.55656405447025,1.18467123093859,3.82888905681828 +96,4,0,0.55656405447025,1.45829670995655,4.519569208739 +96,5,0,0.55656405447025,2.31876334410251,2.96806867089888 +96,6,0,0.55656405447025,1.45175015135933,4.95375559492303 +97,1,0,0.968155189610845,0.731474808343252,2.00819632359671 +97,2,0,0.968155189610845,1.81966272467269,2.41871187476564 +97,3,0,0.968155189610845,1.32541288635434,2.85560582347779 +97,4,0,0.968155189610845,1.02840238957306,1.99543376514698 +97,5,0,0.968155189610845,0.72402655805433,6.36901987614095 +97,6,0,0.968155189610845,1.62812732719515,5.45740806337183 +98,1,0,-0.642161084239234,-0.285123013459829,-0.000642853350642403 +98,2,0,-0.642161084239234,-0.598332460697446,2.16895969756833 +98,3,0,-0.642161084239234,0.455815886578303,0.909661380366459 +98,4,0,-0.642161084239234,0.799759144823868,2.97501355379955 +98,5,0,-0.642161084239234,0.742981106632073,2.0103010060523 +98,6,0,-0.642161084239234,1.28403169128433,3.27781589866413 +99,1,0,0.489663886549812,1.58049905561598,0.63224613149953 +99,2,0,0.489663886549812,0.372598634840632,0.153880335628493 +99,3,0,0.489663886549812,1.4943622173095,1.81815428213934 +99,4,0,0.489663886549812,1.5378597808116,0.935298801828264 +99,5,0,0.489663886549812,1.56518687462083,2.92415904318258 +99,6,0,0.489663886549812,0.602576741552375,2.44369000347583 +100,1,0,0.83152065605674,0.478913216287586,0.746784055135776 +100,2,0,0.83152065605674,1.4235529798745,2.04540124845096 +100,3,0,0.83152065605674,0.968193684526933,1.32348962919775 +100,4,0,0.83152065605674,1.07833225792295,0.547613792696509 +100,5,0,0.83152065605674,1.06746551895251,3.31761291520659 +100,6,0,0.83152065605674,1.85523532671524,3.08316773932352 +101,1,0,-0.919726134671162,-0.0153717261852472,2.7788719289477 +101,2,0,-0.919726134671162,0.0258159670995393,2.62781854772703 +101,3,0,-0.919726134671162,-0.210141999024792,2.44073176765192 +101,4,0,-0.919726134671162,0.524799330002078,1.78160512517281 +101,5,0,-0.919726134671162,-0.0437030303186893,3.04165380941417 +101,6,0,-0.919726134671162,1.3026199543081,5.32428649347593 +102,1,0,0.641255824240951,0.532093223730807,-2.09251354544218 +102,2,0,0.641255824240951,0.0941130407259139,0.733006859014464 +102,3,0,0.641255824240951,1.55861283645998,-0.23990034173479 +102,4,0,0.641255824240951,1.21632707133294,-0.176646402386964 +102,5,0,0.641255824240951,1.77130479077965,4.52178543711811 +102,6,0,0.641255824240951,2.00779025241614,1.91109596784127 +103,1,0,1.14326851073882,0.0119837643154777,2.22043326561435 +103,2,0,1.14326851073882,0.362873618700088,4.6975800148237 +103,3,0,1.14326851073882,1.97187455363987,2.20278367481176 +103,4,0,1.14326851073882,0.549403280336012,4.53412190050463 +103,5,0,1.14326851073882,0.74357105113021,4.04691725702911 +103,6,0,1.14326851073882,1.41192305654462,5.38945740422014 +104,1,0,-1.08072100327974,-0.36568086508946,-0.464035936891244 +104,2,0,-1.08072100327974,-0.98836432896696,-0.356166326023515 +104,3,0,-1.08072100327974,0.467378110330098,-0.0636497459678805 +104,4,0,-1.08072100327974,-0.935126213484022,-0.716266458809519 +104,5,0,-1.08072100327974,0.502717496775458,-0.660129046615434 +104,6,0,-1.08072100327974,1.47173852447056,2.01181621362982 +105,1,0,-0.308977011536411,-0.179345642251318,-0.562705733283704 +105,2,0,-0.308977011536411,0.70656302910021,0.18167481062825 +105,3,0,-0.308977011536411,1.21257374180067,0.813478648356054 +105,4,0,-0.308977011536411,0.381854939551222,0.207736765764304 +105,5,0,-0.308977011536411,1.2106178506653,0.0126906713691238 +105,6,0,-0.308977011536411,0.964448644390534,1.03945464476557 +106,1,0,1.50226005393992,1.34925401718096,2.64319435904704 +106,2,0,1.50226005393992,0.98662538563711,1.3596364327739 +106,3,0,1.50226005393992,2.12823788384978,3.59777809891119 +106,4,0,1.50226005393992,2.30362907733559,5.55472033802837 +106,5,0,1.50226005393992,1.37909145501961,4.84064460409266 +106,6,0,1.50226005393992,1.71825363617274,4.87571321763657 +107,1,0,-0.89457220946846,-0.463955575441369,-3.48745734772936 +107,2,0,-0.89457220946846,-0.0519919382569897,0.125749218439857 +107,3,0,-0.89457220946846,0.520988865165158,-1.70695509814757 +107,4,0,-0.89457220946846,0.33236786268323,0.725662474427429 +107,5,0,-0.89457220946846,1.03760437559963,1.39038624668232 +107,6,0,-0.89457220946846,0.256795459568391,-0.335002534800443 +108,1,0,-1.22324299509621,-0.526114758338154,-0.692133068311805 +108,2,0,-1.22324299509621,0.18477604071721,0.429974089819244 +108,3,0,-1.22324299509621,0.286754140761167,0.372277811842254 +108,4,0,-1.22324299509621,0.542204942504875,0.697878652044274 +108,5,0,-1.22324299509621,0.399442852524572,1.14482729349123 +108,6,0,-1.22324299509621,1.27799739480744,0.760227606864043 +109,1,0,-0.758669466253882,-0.783003297910145,0.0513878206820995 +109,2,0,-0.758669466253882,0.285020986239263,-0.434316127626099 +109,3,0,-0.758669466253882,0.898896938193642,-1.10042010645776 +109,4,0,-0.758669466253882,0.828259841590877,0.643677661471694 +109,5,0,-0.758669466253882,0.380786238406674,1.65897088961278 +109,6,0,-0.758669466253882,0.456410360436729,1.16614884094835 +110,1,0,-2.28836186963232,-1.42240905596012,1.56363050881366 +110,2,0,-2.28836186963232,-1.21031092491361,2.01509164036705 +110,3,0,-2.28836186963232,-0.766586478955116,0.996018770236634 +110,4,0,-2.28836186963232,-0.412915805901137,1.21019829707552 +110,5,0,-2.28836186963232,-1.10526766855183,1.11050770085309 +110,6,0,-2.28836186963232,-0.0558900611664474,0.75444310848228 +111,1,0,0.79771570665335,-0.553133989340675,1.36541391115453 +111,2,0,0.79771570665335,0.246665282521328,2.08614423172338 +111,3,0,0.79771570665335,0.972160689944622,2.5652568519032 +111,4,0,0.79771570665335,0.376996285841898,2.30036239478882 +111,5,0,0.79771570665335,1.4268445074441,3.26085042603327 +111,6,0,0.79771570665335,1.94231661484459,5.31180379360411 +112,1,0,0.12870078897994,0.307506803447317,0.692955762744788 +112,2,0,0.12870078897994,-0.0268741505941875,2.4379863824427 +112,3,0,0.12870078897994,1.00343228065177,1.30208928752089 +112,4,0,0.12870078897994,0.131765893134764,2.86354301083545 +112,5,0,0.12870078897994,0.933523097093359,2.94208138077174 +112,6,0,0.12870078897994,1.76925438035231,3.72716043227665 +113,1,0,-0.00931074971757802,-0.397545128543512,-0.304337726872656 +113,2,0,-0.00931074971757802,0.504335269263596,-1.47696043309091 +113,3,0,-0.00931074971757802,0.69556377664362,0.706831865285902 +113,4,0,-0.00931074971757802,0.952311641049705,1.58895831015197 +113,5,0,-0.00931074971757802,1.72201418175824,1.80348015736981 +113,6,0,-0.00931074971757802,1.00346429177833,2.07938285516792 +114,1,0,-1.33814559514493,-0.13084645095529,0.258563622937842 +114,2,0,-1.33814559514493,-0.328313780755942,-0.755357255404722 +114,3,0,-1.33814559514493,0.767190991540338,0.547314979192388 +114,4,0,-1.33814559514493,0.551545429130956,1.51813317281849 +114,5,0,-1.33814559514493,0.280707403536271,0.585742471844865 +114,6,0,-1.33814559514493,1.87073018189421,1.79129333628737 +115,1,0,-0.0307450017978104,-0.203832382523738,-0.547529905930902 +115,2,0,-0.0307450017978104,0.803531851748503,-1.94762176066094 +115,3,0,-0.0307450017978104,-0.328550830765483,1.15802858200512 +115,4,0,-0.0307450017978104,1.08894515645791,0.479216783688697 +115,5,0,-0.0307450017978104,1.00890444877836,2.146119939363 +115,6,0,-0.0307450017978104,2.54984322597835,2.60041936200757 +116,1,0,0.416210502177554,0.800227902503501,-1.13776949509889 +116,2,0,0.416210502177554,0.72177376351145,-1.29095759397637 +116,3,0,0.416210502177554,1.05092852687046,0.163616534216729 +116,4,0,0.416210502177554,1.07841177778874,0.318569963738997 +116,5,0,0.416210502177554,1.27216409348855,2.62571443309508 +116,6,0,0.416210502177554,2.20291413903546,1.40992567038327 +117,1,0,0.185682053987045,0.0154632265573412,0.126843635097071 +117,2,0,0.185682053987045,-0.294568652078405,-1.59502600932928 +117,3,0,0.185682053987045,0.3434221572718,-0.0954442306696356 +117,4,0,0.185682053987045,0.736971931659641,0.410252145148292 +117,5,0,0.185682053987045,1.5296434146358,1.91722421838383 +117,6,0,0.185682053987045,1.67782346096464,1.01747266376817 +118,1,0,-1.59532950598604,-0.798373104920996,-1.00968400645432 +118,2,0,-1.59532950598604,-0.204361624072532,0.370830252251386 +118,3,0,-1.59532950598604,-0.896149987574084,0.484139473997054 +118,4,0,-1.59532950598604,0.141700472907281,-0.466405186376612 +118,5,0,-1.59532950598604,1.00839653260119,-0.026721952059122 +118,6,0,-1.59532950598604,0.332064509443496,0.821121599465879 +119,1,0,0.121268709978484,0.181681164145453,0.570191327984123 +119,2,0,0.121268709978484,0.519288120645323,2.03742277716536 +119,3,0,0.121268709978484,0.742228904360963,0.281887395792038 +119,4,0,0.121268709978484,0.580678820252124,2.26926858657859 +119,5,0,0.121268709978484,0.646598927456894,0.783493583393684 +119,6,0,0.121268709978484,1.40112832662989,0.82104317069335 +120,1,0,-0.523538252561795,-0.179295568123839,0.22212232449333 +120,2,0,-0.523538252561795,0.10946622333482,0.335398721679965 +120,3,0,-0.523538252561795,0.89486358654903,0.578696943291453 +120,4,0,-0.523538252561795,0.176361255514506,1.50645843591311 +120,5,0,-0.523538252561795,0.706148111155747,4.27936001732894 +120,6,0,-0.523538252561795,1.38773488273727,3.45033864189556 +121,1,3,-1.3209898980621,-0.659030256624808,-1.11457919929402 +121,2,3,-1.3209898980621,-0.141217731860885,1.28435903211288 +121,3,3,-1.3209898980621,-0.791920375267903,0.685288874745403 +121,4,3,-1.3209898980621,-0.0734820400267869,3.25785366872356 +121,5,3,-1.3209898980621,-0.49656205388661,2.02276889139347 +121,6,3,-1.3209898980621,-0.965589275967642,5.21429798519067 +122,1,3,0.65557245598294,0.507227828874015,-0.250570912893117 +122,2,3,0.65557245598294,1.03929914437131,-0.420264983423434 +122,3,3,0.65557245598294,1.34473746170499,1.76685337912262 +122,4,3,0.65557245598294,0.87359591944603,3.4820602234097 +122,5,3,0.65557245598294,0.892485396451848,5.95175331176178 +122,6,3,0.65557245598294,2.52111655846667,7.40612320147572 +123,1,3,0.0144327369651312,0.897110333973617,-0.887945988958214 +123,2,3,0.0144327369651312,0.562597501739654,0.177689505327719 +123,3,3,0.0144327369651312,1.19295988538516,2.78876965030089 +123,4,3,0.0144327369651312,-0.152912404575872,2.14857714673905 +123,5,3,0.0144327369651312,0.354572848563497,5.03305075612627 +123,6,3,0.0144327369651312,0.11038766791842,6.34960584861583 +124,1,3,0.06591379512311,0.0534203651908569,-0.843874058964572 +124,2,3,0.06591379512311,0.270394743140451,2.86834595033898 +124,3,3,0.06591379512311,0.129802985645562,3.91738873706921 +124,4,3,0.06591379512311,0.873770307631855,4.42769196153781 +124,5,3,0.06591379512311,0.983498849130909,7.0964325924724 +124,6,3,0.06591379512311,1.20719003484248,9.48929410346392 +125,1,3,-0.275046326426422,0.293249514156622,1.81714098956309 +125,2,3,-0.275046326426422,0.229476930121458,0.185804954844934 +125,3,3,-0.275046326426422,0.268736748455175,2.76347481395949 +125,4,3,-0.275046326426422,0.466683387349963,4.5530580706989 +125,5,3,-0.275046326426422,1.15612694018968,7.60432928703051 +125,6,3,-0.275046326426422,0.396102765747029,5.50714869110561 +126,1,3,-0.556089165281445,-0.204073561439021,0.917827508857894 +126,2,3,-0.556089165281445,0.19698200850372,2.39608998202929 +126,3,3,-0.556089165281445,1.33309830602468,2.51649382846799 +126,4,3,-0.556089165281445,0.813016580393121,3.75983978920604 +126,5,3,-0.556089165281445,0.279270401241487,5.81598295015132 +126,6,3,-0.556089165281445,0.767015417251097,6.57061869757607 +127,1,3,-0.481761250465059,-0.507168035910364,-0.907680621931876 +127,2,3,-0.481761250465059,0.782178870451505,-1.068489152912 +127,3,3,-0.481761250465059,-0.112056540437024,1.85024307065664 +127,4,3,-0.481761250465059,0.398656788397188,5.22249311314911 +127,5,3,-0.481761250465059,0.423006794578998,3.11951784142225 +127,6,3,-0.481761250465059,0.671986212079901,5.44040257022184 +128,1,3,-1.32740904200675,-1.65144181693399,1.33668575620608 +128,2,3,-1.32740904200675,0.13845682943084,-0.164424994709493 +128,3,3,-1.32740904200675,-0.186641909443694,1.74404216922443 +128,4,3,-1.32740904200675,-0.0193098809394121,1.16926685493817 +128,5,3,-1.32740904200675,0.260278867551969,3.41050128420705 +128,6,3,-1.32740904200675,0.426968699743162,5.81478718328758 +129,1,3,-0.835635156209953,0.365077037473928,1.17563749280077 +129,2,3,-0.835635156209953,0.0255047424585454,0.463487723132613 +129,3,3,-0.835635156209953,0.238261763497732,1.81748991235753 +129,4,3,-0.835635156209953,1.14995136482981,3.02643141094252 +129,5,3,-0.835635156209953,-0.368831270681908,4.53077069457151 +129,6,3,-0.835635156209953,0.494855790328605,3.7369366925089 +130,1,3,0.014617056865741,-0.920193437062664,-0.468859478425729 +130,2,3,0.014617056865741,-0.409771247075564,-0.195779473995372 +130,3,3,0.014617056865741,0.143316992860131,3.35793046974238 +130,4,3,0.014617056865741,1.16344796814541,2.90092316112646 +130,5,3,0.014617056865741,1.92474818134048,7.05539446585689 +130,6,3,0.014617056865741,1.23561893888835,7.4371222065699 +131,1,3,0.438622701805894,1.00256898424452,0.535588551592362 +131,2,3,0.438622701805894,0.891298196316596,-0.51181417381938 +131,3,3,0.438622701805894,1.09060025694346,2.97164644917054 +131,4,3,0.438622701805894,1.34453810826578,2.25079123994776 +131,5,3,0.438622701805894,1.40699884669438,3.59911808574875 +131,6,3,0.438622701805894,0.939875156111874,7.29199956126919 +132,1,3,-1.76243810418353,-1.55642713172473,-0.805043158381241 +132,2,3,-1.76243810418353,-0.391682064940616,-2.0263867595638 +132,3,3,-1.76243810418353,-0.666139512388104,-1.05765094965184 +132,4,3,-1.76243810418353,0.0238185660916042,0.817677818881005 +132,5,3,-1.76243810418353,0.400946115070112,2.91380467653736 +132,6,3,-1.76243810418353,0.913005144984613,2.69049089180893 +133,1,3,1.36534824167319,0.763831528913434,0.577023159081942 +133,2,3,1.36534824167319,0.844309096416079,0.042038282212332 +133,3,3,1.36534824167319,0.947415268176042,2.18378635404389 +133,4,3,1.36534824167319,1.43995238461224,4.92965591378078 +133,5,3,1.36534824167319,2.89808347087941,8.12380754151815 +133,6,3,1.36534824167319,1.10219087008148,9.40004516409809 +134,1,3,1.29446825743501,0.608144869107727,1.08618105043624 +134,2,3,1.29446825743501,1.41973826830158,2.32331581819127 +134,3,3,1.29446825743501,1.51878532055069,3.65140165693938 +134,4,3,1.29446825743501,1.69646592321119,5.36297031018112 +134,5,3,1.29446825743501,1.56911161772488,8.48595978983258 +134,6,3,1.29446825743501,1.90002513778021,9.35068411797129 +135,1,3,-0.178549336156913,0.151158553945093,0.828144325264999 +135,2,3,-0.178549336156913,-0.471060546327792,3.9089119030933 +135,3,3,-0.178549336156913,0.103993512674667,6.07766753358876 +135,4,3,-0.178549336156913,1.08664644183412,5.71413666854425 +135,5,3,-0.178549336156913,1.68404926041005,5.99317910927949 +135,6,3,-0.178549336156913,0.577078077823259,9.483960218053 +136,1,3,-1.07692090882478,-0.401929333738954,1.04747553448351 +136,2,3,-1.07692090882478,-0.545038593804227,0.890555345254023 +136,3,3,-1.07692090882478,0.508562503370516,0.983523883778991 +136,4,3,-1.07692090882478,0.00140170282829649,2.45797787912278 +136,5,3,-1.07692090882478,0.13965802945591,4.85004362804588 +136,6,3,-1.07692090882478,0.935084069218439,5.46362892185911 +137,1,3,0.728454193443001,-0.0114977260864277,0.974364596857209 +137,2,3,0.728454193443001,0.7135343982297,0.67155879830354 +137,3,3,0.728454193443001,0.655179531278547,2.90146090914518 +137,4,3,0.728454193443001,1.38622403146752,5.33426163164905 +137,5,3,0.728454193443001,1.12156269309693,7.6727252728244 +137,6,3,0.728454193443001,1.15693441030068,8.86462680988762 +138,1,3,-1.46938141572406,-0.0883249129282256,0.722072099170389 +138,2,3,-1.46938141572406,-0.142449306634193,0.823877896167457 +138,3,3,-1.46938141572406,-0.0298040848424808,2.30322550088994 +138,4,3,-1.46938141572406,-0.322736256121953,2.31264370539726 +138,5,3,-1.46938141572406,0.124282773210025,6.14394845899594 +138,6,3,-1.46938141572406,1.1023978140493,5.08916101818471 +139,1,3,-0.549298690203426,0.301171733824676,-0.227928347441122 +139,2,3,-0.549298690203426,0.402237345797786,-0.737524166727403 +139,3,3,-0.549298690203426,0.683518954814073,-1.0638203854028 +139,4,3,-0.549298690203426,0.403594704560704,2.16485431597404 +139,5,3,-0.549298690203426,0.55999493763121,4.51625222775326 +139,6,3,-0.549298690203426,1.06915987676031,4.39776263951282 +140,1,3,-1.07513120640263,-0.218917779907026,0.741651877086614 +140,2,3,-1.07513120640263,-0.412691091490353,-0.171855503087313 +140,3,3,-1.07513120640263,0.367202949190019,0.58558380183808 +140,4,3,-1.07513120640263,1.24324583274143,2.70757401857652 +140,5,3,-1.07513120640263,0.424084621174276,1.96616566887664 +140,6,3,-1.07513120640263,0.534559115396839,4.59790385921341 +141,1,3,0.589903379971464,0.518152792811317,0.532857161426118 +141,2,3,0.589903379971464,1.74119554926273,0.583690811233439 +141,3,3,0.589903379971464,1.14384744906574,1.11047142482919 +141,4,3,0.589903379971464,1.43423263831571,5.00913061015248 +141,5,3,0.589903379971464,1.74498549823101,6.56368416729218 +141,6,3,0.589903379971464,1.67289590250084,9.02370809506211 +142,1,3,0.657929099074857,-0.527630501829893,2.23509371293706 +142,2,3,0.657929099074857,0.790313265176156,3.28376796139343 +142,3,3,0.657929099074857,1.30011710750872,1.89310004655243 +142,4,3,0.657929099074857,1.27724881504038,5.63969392962316 +142,5,3,0.657929099074857,0.767272841138719,6.94525677505373 +142,6,3,0.657929099074857,2.32556023319219,9.74629505340234 +143,1,3,0.918209578132046,0.389990741964451,1.73002264959394 +143,2,3,0.918209578132046,0.324124967116553,2.51685777627825 +143,3,3,0.918209578132046,0.864710010354607,4.10998227389613 +143,4,3,0.918209578132046,0.721106454137181,5.74373385047243 +143,5,3,0.918209578132046,2.35220290569798,6.31425548414344 +143,6,3,0.918209578132046,1.06086818047735,9.11988048390896 +144,1,3,1.03723233079857,0.780710632652874,0.212208576948754 +144,2,3,1.03723233079857,1.05604424585774,-0.667835264144556 +144,3,3,1.03723233079857,0.452858041598415,2.84834674243207 +144,4,3,1.03723233079857,1.974918457812,5.49179716277523 +144,5,3,1.03723233079857,1.04103576092729,6.15510734290007 +144,6,3,1.03723233079857,2.13124097931571,10.0580109007073 +145,1,3,0.432864420882128,0.0546511643881509,2.06577211453651 +145,2,3,0.432864420882128,0.542564453524272,0.826589014872826 +145,3,3,0.432864420882128,-0.102986095234928,3.30224330441748 +145,4,3,0.432864420882128,0.818817860108457,5.56089893194988 +145,5,3,0.432864420882128,0.961458251664878,7.7240973086671 +145,6,3,0.432864420882128,1.27895634770374,8.02085741371424 +146,1,3,0.494887951274981,0.906621933718628,1.35833730554146 +146,2,3,0.494887951274981,0.609584098447873,1.50672641372537 +146,3,3,0.494887951274981,1.66806988803927,2.68420111093815 +146,4,3,0.494887951274981,1.17243386474274,5.77226847745143 +146,5,3,0.494887951274981,1.7367792550661,6.86343483473947 +146,6,3,0.494887951274981,1.32962006663717,7.39856300627146 +147,1,3,0.337514991797093,0.714474564182813,3.77502417711785 +147,2,3,0.337514991797093,-0.120717345754449,2.39855290111283 +147,3,3,0.337514991797093,0.979181408407107,3.3951013798209 +147,4,3,0.337514991797093,0.765822372463936,4.51187239398326 +147,5,3,0.337514991797093,1.1636592509228,7.32871256559361 +147,6,3,0.337514991797093,1.69073725736773,7.89889121515618 +148,1,3,0.940914420491235,0.32599988959407,-2.68432419188846 +148,2,3,0.940914420491235,0.902667702594575,-0.502223665400725 +148,3,3,0.940914420491235,1.79711547389592,2.72616983157362 +148,4,3,0.940914420491235,1.89564536968462,3.39439552886371 +148,5,3,0.940914420491235,1.86546638275212,4.46411002977733 +148,6,3,0.940914420491235,1.85126607219131,5.8921851033331 +149,1,3,0.457967792328404,0.378698935479082,4.12572103429067 +149,2,3,0.457967792328404,0.625145122806749,4.12170173700891 +149,3,3,0.457967792328404,0.657407339867929,5.67039906683291 +149,4,3,0.457967792328404,0.894496831842379,8.9524316705688 +149,5,3,0.457967792328404,1.27815558368229,9.34940165660425 +149,6,3,0.457967792328404,1.21757039960575,9.72894783793935 +150,1,3,1.28822951042595,0.929280238078882,-0.681541858459146 +150,2,3,1.28822951042595,0.37556044800664,-2.14029291717815 +150,3,3,1.28822951042595,0.955960970286821,1.49994481205458 +150,4,3,1.28822951042595,0.666797039509379,3.18273482219049 +150,5,3,1.28822951042595,1.93863479694882,6.10370652383497 +150,6,3,1.28822951042595,1.3261231323096,5.09851390581681 +151,1,3,0.799327933544311,0.752383684129809,0.998835172417796 +151,2,3,0.799327933544311,1.48794166496513,2.50739666840068 +151,3,3,0.799327933544311,0.636647413978102,3.39248141555062 +151,4,3,0.799327933544311,0.43183050965539,6.2241784706405 +151,5,3,0.799327933544311,1.60186143581385,6.57526813277581 +151,6,3,0.799327933544311,1.48009902563213,9.68047322868392 +152,1,3,0.641075436191527,0.410381761209829,0.181288378872355 +152,2,3,0.641075436191527,0.833387068122976,0.440335510368963 +152,3,3,0.641075436191527,1.02324616435653,1.92203052266619 +152,4,3,0.641075436191527,1.77222798638308,3.95065028029737 +152,5,3,0.641075436191527,1.50187251866824,2.58165781357217 +152,6,3,0.641075436191527,0.913951107889827,6.90369945886804 +153,1,3,0.647106275420512,-0.237400237086652,-2.02164808965544 +153,2,3,0.647106275420512,0.257978734055656,0.228139264140656 +153,3,3,0.647106275420512,1.75480644001202,-0.169015192170175 +153,4,3,0.647106275420512,0.41350545765392,3.79498507213573 +153,5,3,0.647106275420512,1.07445417959469,4.38245420741927 +153,6,3,0.647106275420512,0.98341038645438,6.61511100339346 +154,1,3,1.04666451517136,0.341354827336876,3.42913227448808 +154,2,3,1.04666451517136,0.672471846348521,0.616635516261048 +154,3,3,1.04666451517136,1.27811698735899,2.7748662991019 +154,4,3,1.04666451517136,1.48649127087623,6.01209090483621 +154,5,3,1.04666451517136,1.61543543912908,7.37520919143932 +154,6,3,1.04666451517136,2.02536607455159,9.48209414318584 +155,1,3,0.576975845608666,0.0914320774480271,0.722851227802534 +155,2,3,0.576975845608666,0.108522327794117,1.6890512266516 +155,3,3,0.576975845608666,0.259628061168946,2.58597206970974 +155,4,3,0.576975845608666,1.01811928925711,6.21498745277975 +155,5,3,0.576975845608666,0.719060589759526,7.07453482145899 +155,6,3,0.576975845608666,1.92334185803099,10.009696002852 +156,1,3,0.17037629608571,0.784349410920146,2.11235841463052 +156,2,3,0.17037629608571,0.214232082289887,-0.697328657084354 +156,3,3,0.17037629608571,1.25013551445887,3.29808927709344 +156,4,3,0.17037629608571,1.7086115578765,4.77325299010961 +156,5,3,0.17037629608571,1.11415545041536,5.70327263182313 +156,6,3,0.17037629608571,0.847156272453002,6.739097993206 +157,1,3,0.396115427083706,0.923645518828156,0.864477515155358 +157,2,3,0.396115427083706,0.648033309117745,3.31731508957503 +157,3,3,0.396115427083706,1.14817613766954,3.60719373954189 +157,4,3,0.396115427083706,0.420299068553708,5.09071471525354 +157,5,3,0.396115427083706,1.14833117298122,6.75113263123557 +157,6,3,0.396115427083706,1.07770858221266,8.92370742505083 +158,1,3,-0.38503981699399,0.249091044450303,-0.455481254742036 +158,2,3,-0.38503981699399,0.652965111326567,0.513194049834224 +158,3,3,-0.38503981699399,-0.203526544921249,3.97850435486181 +158,4,3,-0.38503981699399,0.178102153152252,3.50022698865287 +158,5,3,-0.38503981699399,0.238824839896899,5.90492745057116 +158,6,3,-0.38503981699399,1.41010656792478,6.139197871099 +159,1,3,-0.606966899370249,-0.235070624391252,1.67459208020634 +159,2,3,-0.606966899370249,0.397058569815924,-0.983876672431654 +159,3,3,-0.606966899370249,-0.0987236791816803,2.11597559238572 +159,4,3,-0.606966899370249,0.0175133414764335,4.93858440091871 +159,5,3,-0.606966899370249,1.07180450901133,6.65025467972839 +159,6,3,-0.606966899370249,1.03878178919866,4.64039122565767 +160,1,3,0.183609061947842,0.922644868366452,1.06263247541716 +160,2,3,0.183609061947842,-0.521822880184819,3.84919810086592 +160,3,3,0.183609061947842,1.26182116417092,2.87398711588164 +160,4,3,0.183609061947842,1.1296518783099,4.97402442219597 +160,5,3,0.183609061947842,1.16806292711789,6.59628616328911 +160,6,3,0.183609061947842,1.54910874984912,7.78160398847949 +161,1,3,-1.5739802437226,-0.738750875893334,0.646798997035059 +161,2,3,-1.5739802437226,-0.817190355326895,-0.199794544446061 +161,3,3,-1.5739802437226,0.00978813383769062,1.53768978518043 +161,4,3,-1.5739802437226,-0.0692027187562883,4.22559029952303 +161,5,3,-1.5739802437226,-0.321648430686517,2.49489898430561 +161,6,3,-1.5739802437226,1.80291081637126,2.97534946673223 +162,1,3,-1.5135182536791,-1.21415498016749,1.08344375432323 +162,2,3,-1.5135182536791,0.00862446836171632,0.366096436545889 +162,3,3,-1.5135182536791,-0.580560672020834,0.386396282908581 +162,4,3,-1.5135182536791,0.284802496374087,2.65431055942517 +162,5,3,-1.5135182536791,0.481985729465867,4.08004569199787 +162,6,3,-1.5135182536791,-0.164457779819998,5.44486509744182 +163,1,3,0.17709443103689,0.282609663048505,1.2187601244373 +163,2,3,0.17709443103689,0.411582509889598,3.46647644603637 +163,3,3,0.17709443103689,0.847495649227131,6.24284448978847 +163,4,3,0.17709443103689,1.00780264364244,6.26054223704011 +163,5,3,0.17709443103689,0.567458999918712,6.75896719170673 +163,6,3,0.17709443103689,1.73124741621038,8.55153680922603 +164,1,3,0.403250756295859,0.492850083886365,2.48897701813397 +164,2,3,0.403250756295859,0.540083950359147,2.10861656799587 +164,3,3,0.403250756295859,0.731550492755462,5.78483021182663 +164,4,3,0.403250756295859,0.458474142754207,6.25304959523288 +164,5,3,0.403250756295859,1.4995702617515,7.24373626356283 +164,6,3,0.403250756295859,0.984707497276815,8.31693696597542 +165,1,3,-1.75890966008881,-0.492384152667725,-3.0165269004572 +165,2,3,-1.75890966008881,-0.114806693432703,1.51062699564072 +165,3,3,-1.75890966008881,-0.48595164161601,-1.00828460304626 +165,4,3,-1.75890966008881,0.617535901530216,3.78881684855537 +165,5,3,-1.75890966008881,-0.477578070519953,4.53617118871654 +165,6,3,-1.75890966008881,0.0165545486318738,2.41304871152758 +166,1,3,0.550444344652174,0.720896381917308,-0.0902337840097769 +166,2,3,0.550444344652174,1.07643595531667,1.67174317972133 +166,3,3,0.550444344652174,1.23330497683997,3.30095700316899 +166,4,3,0.550444344652174,0.239620344875785,5.01348139892779 +166,5,3,0.550444344652174,0.572685885770811,5.04458155047439 +166,6,3,0.550444344652174,1.90034510518233,8.66633628083652 +167,1,3,-0.177518472281448,0.559127011742701,-2.84585414560063 +167,2,3,-0.177518472281448,0.671231625413232,0.97648769056059 +167,3,3,-0.177518472281448,0.596607991549095,0.67623314360395 +167,4,3,-0.177518472281448,0.28242453789717,2.26063148836001 +167,5,3,-0.177518472281448,0.772759320165847,4.26137543656067 +167,6,3,-0.177518472281448,0.0529782047803846,4.03342527507047 +168,1,3,-0.283378689715887,0.56880477901632,0.566393501073422 +168,2,3,-0.283378689715887,0.687228894918578,1.33767774653906 +168,3,3,-0.283378689715887,0.716981777257136,3.20528354672777 +168,4,3,-0.283378689715887,0.589459989413421,4.6418581200693 +168,5,3,-0.283378689715887,0.896581550535394,5.35234524586164 +168,6,3,-0.283378689715887,1.16641332038933,5.89859615929218 +169,1,3,0.0949255069380952,-0.138844826813786,0.880456211503483 +169,2,3,0.0949255069380952,0.899054228996397,1.70093245545876 +169,3,3,0.0949255069380952,0.375520297802488,2.34651154150143 +169,4,3,0.0949255069380952,0.765847242429938,7.0065917773773 +169,5,3,0.0949255069380952,0.770509647956809,4.98968023938993 +169,6,3,0.0949255069380952,1.34483793919838,6.9101710957784 +170,1,3,0.109708934474219,0.386120730420973,2.85160717995104 +170,2,3,0.109708934474219,0.407879747082585,1.33752020856814 +170,3,3,0.109708934474219,0.637178862432442,4.20098703126136 +170,4,3,0.109708934474219,0.207565281405001,4.77374671451531 +170,5,3,0.109708934474219,1.85419902965847,6.3939200558063 +170,6,3,0.109708934474219,1.61088867029117,6.59919416619873 +171,1,3,-1.02075201769416,-0.356310807707324,1.02592651163283 +171,2,3,-1.02075201769416,0.0424193927151826,0.741657135044775 +171,3,3,-1.02075201769416,0.0317264399471442,1.71370818238931 +171,4,3,-1.02075201769416,0.467235778780831,5.47811263281214 +171,5,3,-1.02075201769416,0.0791128775261391,4.12329110395588 +171,6,3,-1.02075201769416,0.895303915215752,5.00292845449822 +172,1,3,-0.214471204357173,-0.727646095799363,1.35131925370431 +172,2,3,-0.214471204357173,0.210944497892413,2.37175026318938 +172,3,3,-0.214471204357173,0.570841146082983,3.65951143912713 +172,4,3,-0.214471204357173,0.461650424820328,6.3022531708993 +172,5,3,-0.214471204357173,0.342228491003671,8.22419778472385 +172,6,3,-0.214471204357173,1.28117396857179,7.84810036822843 +173,1,3,0.95312723300276,0.587136635676131,2.27117731500969 +173,2,3,0.95312723300276,0.467985411032795,2.69182871995691 +173,3,3,0.95312723300276,0.768091000867892,6.44965476196657 +173,4,3,0.95312723300276,1.53518612893103,6.20379049823067 +173,5,3,0.95312723300276,0.734032956366248,8.37840205107447 +173,6,3,0.95312723300276,0.880826456225791,10.6071742329299 +174,1,3,-0.299503059426975,-0.0143918053073009,0.165378654819905 +174,2,3,-0.299503059426975,0.351702004390223,1.14652818851001 +174,3,3,-0.299503059426975,1.35443661771852,1.34930878825417 +174,4,3,-0.299503059426975,0.641451213769214,2.97573642323419 +174,5,3,-0.299503059426975,0.792151951680742,3.2784114112508 +174,6,3,-0.299503059426975,0.559389848134574,7.10103746423127 +175,1,3,-1.35656841055337,-0.68822256639622,1.09383253065236 +175,2,3,-1.35656841055337,0.165950349733725,2.32705481637459 +175,3,3,-1.35656841055337,-0.0394690022428881,1.58889766941899 +175,4,3,-1.35656841055337,-0.304479039922878,0.857436014714444 +175,5,3,-1.35656841055337,1.07142210914254,2.26750720166773 +175,6,3,-1.35656841055337,1.22342980686212,3.62877035996801 +176,1,3,-0.722906834501834,0.163769122541646,-0.632232933093176 +176,2,3,-0.722906834501834,0.0157880302844722,0.938801879860888 +176,3,3,-0.722906834501834,0.317548010285299,3.67834365574177 +176,4,3,-0.722906834501834,0.320655034651504,2.99198807090912 +176,5,3,-0.722906834501834,1.20799091004943,3.5296721551273 +176,6,3,-0.722906834501834,1.4715576177269,3.80599815112298 +177,1,3,-0.873027113678424,-0.134710889146149,1.20554182330847 +177,2,3,-0.873027113678424,-0.343203700912485,1.08472943367274 +177,3,3,-0.873027113678424,0.68383414113027,4.85152313687422 +177,4,3,-0.873027113678424,-0.0307043288901434,4.54226192906532 +177,5,3,-0.873027113678424,0.476565413251091,8.21586862915938 +177,6,3,-0.873027113678424,0.692592828471272,7.78871269859934 +178,1,3,0.04325400319841,0.483812986689953,2.23748077948003 +178,2,3,0.04325400319841,0.124723740750307,0.0257740658283268 +178,3,3,0.04325400319841,-0.0236415679346836,2.07893121569823 +178,4,3,0.04325400319841,0.38645207413081,4.73747920429682 +178,5,3,0.04325400319841,0.911079572477391,6.13334250277362 +178,6,3,0.04325400319841,1.56328403641471,5.10634606436181 +179,1,3,-0.705597138651361,-0.203512657961043,0.0705300276670152 +179,2,3,-0.705597138651361,-0.500641996603977,1.80939716425173 +179,3,3,-0.705597138651361,0.443055984083823,1.22782580215329 +179,4,3,-0.705597138651361,0.785627941108739,3.54952364932024 +179,5,3,-0.705597138651361,0.370592747588609,5.44620669723755 +179,6,3,-0.705597138651361,0.784335634233162,6.91807435557 +180,1,3,2.06398866391933,1.05930399943233,-0.00767017810784587 +180,2,3,2.06398866391933,0.974937384643123,1.3889518162926 +180,3,3,2.06398866391933,1.67488127697509,4.941828354601 +180,4,3,2.06398866391933,1.54166970733265,6.37226663940419 +180,5,3,2.06398866391933,1.55908201581147,7.63019049127558 +180,6,3,2.06398866391933,3.13182760994635,11.0824508424514 +181,1,3,0.360194044770426,0.137219648791359,-0.120984723180791 +181,2,3,0.360194044770426,0.38551064785073,1.30910385832044 +181,3,3,0.360194044770426,0.414240176764444,2.40758725231587 +181,4,3,0.360194044770426,1.03798805335224,4.04969508727083 +181,5,3,0.360194044770426,1.26482943472362,7.05018989645842 +181,6,3,0.360194044770426,0.674817592128119,8.06113032044475 +182,1,3,-1.68147033404543,-0.529992351263284,-0.468300262616531 +182,2,3,-1.68147033404543,-0.361074846598351,-0.82128675411022 +182,3,3,-1.68147033404543,0.193761651635151,2.11230702234137 +182,4,3,-1.68147033404543,0.217264661905855,1.47293064025627 +182,5,3,-1.68147033404543,-0.887821017937981,2.24634296880867 +182,6,3,-1.68147033404543,0.741792955070734,3.71110597293242 +183,1,3,-0.196778992696935,-0.646729030890644,-1.00205754402073 +183,2,3,-0.196778992696935,0.0139864871831406,0.953921068284142 +183,3,3,-0.196778992696935,0.0795937312953826,0.860583458955781 +183,4,3,-0.196778992696935,0.837209243653747,4.51956682189724 +183,5,3,-0.196778992696935,0.601170803412534,4.97723339242763 +183,6,3,-0.196778992696935,1.47422885794644,5.32121969953623 +184,1,3,0.730421144749429,-0.139311224235713,0.171081394747952 +184,2,3,0.730421144749429,1.53996280134507,1.94344486906202 +184,3,3,0.730421144749429,0.782149116082784,3.23383839820865 +184,4,3,0.730421144749429,0.609823463790785,2.62356551558351 +184,5,3,0.730421144749429,1.89101976789857,6.35341741228886 +184,6,3,0.730421144749429,1.72754545713741,7.05230497645335 +185,1,3,-0.0863000134300777,0.953213880450527,0.0349463777858372 +185,2,3,-0.0863000134300777,0.24448596475191,0.745217744798797 +185,3,3,-0.0863000134300777,0.817473988609581,3.84902626581413 +185,4,3,-0.0863000134300777,0.341876847218706,3.75460775000166 +185,5,3,-0.0863000134300777,0.648765871009181,5.82819842016675 +185,6,3,-0.0863000134300777,1.0361709300895,6.16150290917458 +186,1,3,0.506421572168642,0.367175153198149,-0.150215409953131 +186,2,3,0.506421572168642,0.567547043778752,1.16468293926085 +186,3,3,0.506421572168642,0.622465315259915,2.3988406787079 +186,4,3,0.506421572168642,0.394663508169827,6.48536024802799 +186,5,3,0.506421572168642,0.884648088357487,5.87931879381252 +186,6,3,0.506421572168642,1.2150386952591,7.23309433364219 +187,1,3,0.241411363100991,0.926007128421722,0.230728131241308 +187,2,3,0.241411363100991,0.613940209317918,0.0577344374340867 +187,3,3,0.241411363100991,1.98051616149487,2.11188576701728 +187,4,3,0.241411363100991,-0.0482207260230793,1.6364886842494 +187,5,3,0.241411363100991,1.13745180089868,3.05389500156471 +187,6,3,0.241411363100991,1.43270154624331,4.83798493222586 +188,1,3,-0.291937535972332,-0.432249249948712,-3.09229985520743 +188,2,3,-0.291937535972332,0.267479708595359,-1.57889714548199 +188,3,3,-0.291937535972332,0.415797657047746,0.287257962183195 +188,4,3,-0.291937535972332,-0.473326451583243,1.62858283902274 +188,5,3,-0.291937535972332,0.250647366273356,1.21906452860634 +188,6,3,-0.291937535972332,1.39466252891878,4.89754797997687 +189,1,3,-1.42673622400026,0.0874786130907167,-0.492174801618482 +189,2,3,-1.42673622400026,-0.360288142109093,-0.590640207208217 +189,3,3,-1.42673622400026,-0.00295717580758077,0.594372840951305 +189,4,3,-1.42673622400026,0.250174512466053,4.34575202958559 +189,5,3,-1.42673622400026,0.331734399880543,2.33611571216873 +189,6,3,-1.42673622400026,0.496807974220238,4.54737715581265 +190,1,3,-0.2115984836605,0.60223614723348,0.833538926670778 +190,2,3,-0.2115984836605,-0.731387152145866,2.22483251339626 +190,3,3,-0.2115984836605,1.278125494643,2.18564298307356 +190,4,3,-0.2115984836605,0.78864245668234,5.51233312764889 +190,5,3,-0.2115984836605,1.26236600590622,6.24849207856658 +190,6,3,-0.2115984836605,1.05785986019084,8.69015711295789 +191,1,5,0.983592225005866,0.685269042736118,1.38335066496562 +191,2,5,0.983592225005866,0.544502809451078,2.63622648994195 +191,3,5,0.983592225005866,1.27722602650698,1.7975669580347 +191,4,5,0.983592225005866,0.921971281461675,4.00241556657792 +191,5,5,0.983592225005866,1.16159622356324,3.40768352363552 +191,6,5,0.983592225005866,1.87368858096347,6.01936776577239 +192,1,5,-0.283157026746315,0.0104512285659374,1.27997348572735 +192,2,5,-0.283157026746315,-0.0834672788684822,-1.10599779071778 +192,3,5,-0.283157026746315,-0.232655886244044,-0.454134990455851 +192,4,5,-0.283157026746315,0.723728064985206,1.68155470824088 +192,5,5,-0.283157026746315,1.6857236853256,1.87155358218702 +192,6,5,-0.283157026746315,1.44827549669874,5.95064220847003 +193,1,5,0.00523970150119662,0.276436364797033,-1.93364739079216 +193,2,5,0.00523970150119662,-0.286764607187908,-1.76658683663304 +193,3,5,0.00523970150119662,0.830021732791269,-1.31917039846352 +193,4,5,0.00523970150119662,0.0683583315476364,-0.585846233099063 +193,5,5,0.00523970150119662,1.41795184080921,1.07536146125153 +193,6,5,0.00523970150119662,0.820462889803177,4.78603679649615 +194,1,5,0.92545213318825,1.31461163819231,2.92951584215972 +194,2,5,0.92545213318825,1.90310839616577,0.226299788014934 +194,3,5,0.92545213318825,1.87138107688812,3.63924728317667 +194,4,5,0.92545213318825,1.47734438773685,4.09859230112481 +194,5,5,0.92545213318825,1.61762449653127,5.5875297297966 +194,6,5,0.92545213318825,1.53505759071181,5.73026764736178 +195,1,5,-0.635670393667737,0.92906119673608,0.727083910123489 +195,2,5,-0.635670393667737,0.945527892927173,1.89213182433102 +195,3,5,-0.635670393667737,-0.0488719634014442,2.509396606672 +195,4,5,-0.635670393667737,0.589292270566448,1.32663557253803 +195,5,5,-0.635670393667737,0.481316722409301,3.19923450497982 +195,6,5,-0.635670393667737,1.08897508741881,4.12014467427658 +196,1,5,1.15670868958139,0.524265461334005,1.14361307382569 +196,2,5,1.15670868958139,1.09018652226291,1.7777630785721 +196,3,5,1.15670868958139,1.17365751121989,2.80845429477206 +196,4,5,1.15670868958139,0.519026773875079,4.76364255763518 +196,5,5,1.15670868958139,1.52994359394921,4.67118251896483 +196,6,5,1.15670868958139,1.00861805373324,7.97521550687341 +197,1,5,1.78969021425029,0.4434355361539,1.75105876560352 +197,2,5,1.78969021425029,2.26138915469364,3.18412757427007 +197,3,5,1.78969021425029,2.03441329772634,4.87704223939755 +197,4,5,1.78969021425029,2.19733416987764,5.07213816231237 +197,5,5,1.78969021425029,1.86882625281979,7.36445273426155 +197,6,5,1.78969021425029,0.869774157574492,7.89045853098738 +198,1,5,-0.149019232380438,0.906656488536891,0.617582157062771 +198,2,5,-0.149019232380438,-0.303441296388933,1.74706459113415 +198,3,5,-0.149019232380438,-0.0386216673435962,1.41188339797885 +198,4,5,-0.149019232380438,0.227646993112056,2.47893997311591 +198,5,5,-0.149019232380438,1.44183369734344,3.68740351555302 +198,6,5,-0.149019232380438,0.337064359767777,4.93171424134225 +199,1,5,-0.261067529046921,1.10193513549883,4.07397809127892 +199,2,5,-0.261067529046921,0.831821005947784,2.64115271413386 +199,3,5,-0.261067529046921,1.0507459873153,2.5984635365937 +199,4,5,-0.261067529046921,0.954457862577376,3.15005778434285 +199,5,5,-0.261067529046921,0.726355188233059,6.09140004175636 +199,6,5,-0.261067529046921,0.925631026478668,6.26570200917389 +200,1,5,-0.279422558021935,-1.14039912066845,-3.71994209961769 +200,2,5,-0.279422558021935,-1.06989083106309,-1.03919624114515 +200,3,5,-0.279422558021935,1.36983043403129,-0.00638631184268384 +200,4,5,-0.279422558021935,1.08189923332103,0.228665527854604 +200,5,5,-0.279422558021935,0.902665178547284,2.26697554552666 +200,6,5,-0.279422558021935,0.731355545786545,6.36441088594964 +201,1,5,-0.867508001059406,-0.233675811236789,-1.49758007707428 +201,2,5,-0.867508001059406,-0.550883931428525,0.374386856211282 +201,3,5,-0.867508001059406,0.0857728512645791,0.0294364273898407 +201,4,5,-0.867508001059406,-0.232389250145715,1.42191855779375 +201,5,5,-0.867508001059406,1.33590509096511,2.55688670984156 +201,6,5,-0.867508001059406,0.534449929648672,1.42124688873966 +202,1,5,-0.266028106354418,-0.128148387025209,0.975074215504778 +202,2,5,-0.266028106354418,0.446467815751043,-0.0510663436929477 +202,3,5,-0.266028106354418,0.0897105442552565,1.67391335328299 +202,4,5,-0.266028106354418,0.268223994946335,2.19974815286627 +202,5,5,-0.266028106354418,0.298937620984014,3.08651582214762 +202,6,5,-0.266028106354418,1.19175101086346,5.35184933489553 +203,1,5,0.0257161972054362,-0.00777182454554659,0.887316769481998 +203,2,5,0.0257161972054362,0.195171140233948,-2.34645529619854 +203,3,5,0.0257161972054362,0.943185858944576,-0.886172854562971 +203,4,5,0.0257161972054362,0.97297438051478,0.999551256331918 +203,5,5,0.0257161972054362,1.5413637280623,1.46967773439603 +203,6,5,0.0257161972054362,1.09237538286599,4.74829328190536 +204,1,5,0.116551434634878,0.778053422766819,0.923901269733919 +204,2,5,0.116551434634878,0.283905891981172,0.912512995323872 +204,3,5,0.116551434634878,0.939725254375677,1.49793081521857 +204,4,5,0.116551434634878,-0.105034571045226,1.4170606846868 +204,5,5,0.116551434634878,0.0742100832343512,2.88003343428499 +204,6,5,0.116551434634878,2.19181543302645,4.89563372429911 +205,1,5,-0.553125937781561,0.0859988450814071,-0.126962865847536 +205,2,5,-0.553125937781561,-0.113794244285939,1.04790901416167 +205,3,5,-0.553125937781561,1.50265709885282,-0.0121647533712959 +205,4,5,-0.553125937781561,0.753642118076197,-1.87801353698332 +205,5,5,-0.553125937781561,0.798974629882657,1.81444078247793 +205,6,5,-0.553125937781561,1.22940991518573,2.72413883940338 +206,1,5,0.756255260174997,1.37286170808534,-0.0618669650195494 +206,2,5,0.756255260174997,0.0270268243862444,3.87004440747301 +206,3,5,0.756255260174997,0.100304660700929,2.62002685969236 +206,4,5,0.756255260174997,1.50297292169296,2.10325982384873 +206,5,5,0.756255260174997,1.47481298806093,5.08851039988119 +206,6,5,0.756255260174997,1.49517633071853,5.329207300942 +207,1,5,0.402919292696054,0.448184026029819,1.6851353088623 +207,2,5,0.402919292696054,-0.309345001830238,-0.0480430976597863 +207,3,5,0.402919292696054,1.22136648402251,1.95683410550332 +207,4,5,0.402919292696054,1.38816029165409,1.56554848066498 +207,5,5,0.402919292696054,1.12448389959517,4.80323957292905 +207,6,5,0.402919292696054,0.670591444723001,7.61710463034625 +208,1,5,-0.269406763993592,0.206667648141733,2.22207660012001 +208,2,5,-0.269406763993592,0.0288576177863964,1.86463949430405 +208,3,5,-0.269406763993592,1.3635139605955,2.09011945100763 +208,4,5,-0.269406763993592,1.17538271934532,1.67784711763015 +208,5,5,-0.269406763993592,0.615716880287763,4.45742510031184 +208,6,5,-0.269406763993592,0.974785266523716,3.3847857749142 +209,1,5,1.37322747966558,1.51005515038624,-1.95142063469521 +209,2,5,1.37322747966558,0.66548558946035,-0.00933320143897454 +209,3,5,1.37322747966558,0.368067356457127,1.92149373004633 +209,4,5,1.37322747966558,2.2044979582161,2.22552051395123 +209,5,5,1.37322747966558,1.32532189498429,4.42699163117127 +209,6,5,1.37322747966558,2.29794574303181,6.54842437979203 +210,1,5,-1.21546340892904,-1.09510776302794,2.16313335716541 +210,2,5,-1.21546340892904,-0.0869249651887154,-0.113012776498633 +210,3,5,-1.21546340892904,-0.279230661394171,1.56240828061579 +210,4,5,-1.21546340892904,-0.049521274309301,4.51715372142149 +210,5,5,-1.21546340892904,0.00657525039390539,2.86563277524601 +210,6,5,-1.21546340892904,-0.0645347156398759,2.47481350145028 +211,1,5,-0.0701117965547371,-0.715929786867697,0.0255299412037402 +211,2,5,-0.0701117965547371,1.36168105909399,0.998667609709119 +211,3,5,-0.0701117965547371,-0.135807675651325,1.56559791547375 +211,4,5,-0.0701117965547371,1.51851847154815,1.02220638575337 +211,5,5,-0.0701117965547371,0.165993948214305,4.03246492986913 +211,6,5,-0.0701117965547371,0.959425763838153,3.69187032029544 +212,1,5,1.52424799429573,0.754050279225861,1.82652570977404 +212,2,5,1.52424799429573,1.36008329509545,1.87592392729932 +212,3,5,1.52424799429573,1.75505071496496,4.67880928526713 +212,4,5,1.52424799429573,1.22602677201853,6.36476797191923 +212,5,5,1.52424799429573,1.44265784000527,6.45907084431469 +212,6,5,1.52424799429573,1.81542507698388,8.88774350019465 +213,1,5,-1.13898322177537,-0.704850530259102,0.989954312743611 +213,2,5,-1.13898322177537,0.269910398419923,2.64079703302344 +213,3,5,-1.13898322177537,-0.30661493225157,0.234400241959169 +213,4,5,-1.13898322177537,0.173200709565968,0.115256042285581 +213,5,5,-1.13898322177537,-0.174767967493852,2.37243004034063 +213,6,5,-1.13898322177537,0.062705181813378,3.15196239199807 +214,1,5,0.0136635721568709,0.281286858491064,0.308737894502187 +214,2,5,0.0136635721568709,0.709632912619912,2.86447518064835 +214,3,5,0.0136635721568709,0.885759818762133,1.60433385861157 +214,4,5,0.0136635721568709,0.508252732894141,2.9767702147517 +214,5,5,0.0136635721568709,0.995181587793084,4.04309942636804 +214,6,5,0.0136635721568709,0.952559089661326,4.772002205928 +215,1,5,-0.524976137743338,-0.608745128024079,-0.623987422658077 +215,2,5,-0.524976137743338,0.0441390976086862,0.3386677182339 +215,3,5,-0.524976137743338,1.013040638404,0.859440864419736 +215,4,5,-0.524976137743338,0.368850033899774,-0.818130100579811 +215,5,5,-0.524976137743338,1.28996274747004,0.747054019336613 +215,6,5,-0.524976137743338,0.521169572898015,1.66670573365348 +216,1,5,0.528119708420632,0.651477084437435,-0.227441920101093 +216,2,5,0.528119708420632,0.581493185777966,0.0857339576458841 +216,3,5,0.528119708420632,1.74913980629871,0.254230180328475 +216,4,5,0.528119708420632,0.897716621442405,2.42565796191354 +216,5,5,0.528119708420632,1.71655243592702,2.22339067686188 +216,6,5,0.528119708420632,1.34115667416258,4.07122551146554 +217,1,5,-0.278139356660236,-0.432553538863004,-0.781011633085371 +217,2,5,-0.278139356660236,0.422378930946401,-0.567678688198085 +217,3,5,-0.278139356660236,0.562491377327001,-0.965369032587661 +217,4,5,-0.278139356660236,0.500297950693926,0.934629473125308 +217,5,5,-0.278139356660236,1.31421861315264,0.758231973389853 +217,6,5,-0.278139356660236,1.35139823865584,2.03912728069027 +218,1,5,0.49043594073852,0.321110560676538,0.510384585688465 +218,2,5,0.49043594073852,0.706729675579629,-1.04299998018294 +218,3,5,0.49043594073852,0.880495568887448,1.15469874941526 +218,4,5,0.49043594073852,0.865795526845137,2.38447041558231 +218,5,5,0.49043594073852,1.36781795204138,3.44280884506035 +218,6,5,0.49043594073852,2.36944881552513,8.16161080730789 +219,1,5,-1.98958128344518,-0.534903390163236,1.42588349306103 +219,2,5,-1.98958128344518,-0.373290925434039,1.36822515439796 +219,3,5,-1.98958128344518,0.158331617880881,-0.926249260297009 +219,4,5,-1.98958128344518,-0.607765725191721,-1.20272573134459 +219,5,5,-1.98958128344518,0.499900456385467,-0.65847289715537 +219,6,5,-1.98958128344518,-0.128902846011836,2.52464534710552 +220,1,5,-0.213493340618975,0.185391211106394,1.26700247541726 +220,2,5,-0.213493340618975,0.0140852828787207,-1.4427375813286 +220,3,5,-0.213493340618975,0.0174615772931458,-0.306210357001648 +220,4,5,-0.213493340618975,0.539954347150334,1.81353252436983 +220,5,5,-0.213493340618975,0.663050221600149,3.42996813535437 +220,6,5,-0.213493340618975,0.611350423560622,4.1990652063131 +221,1,5,-0.877521252652733,-0.558494533889764,-1.24506994830942 +221,2,5,-0.877521252652733,-0.054670778540151,-1.05245973258249 +221,3,5,-0.877521252652733,-0.434196160948057,-0.49282114431209 +221,4,5,-0.877521252652733,0.178911659606747,-2.59280555529188 +221,5,5,-0.877521252652733,0.68317178405947,1.53212805684875 +221,6,5,-0.877521252652733,1.00294622561415,2.4801642646124 +222,1,5,1.81905705063474,1.37293696497615,1.80371980454759 +222,2,5,1.81905705063474,1.72673008537909,2.32717660429447 +222,3,5,1.81905705063474,1.37469608885288,2.83327951835427 +222,4,5,1.81905705063474,1.44072915243679,5.45277832067784 +222,5,5,1.81905705063474,1.94592738842887,8.26489174542936 +222,6,5,1.81905705063474,2.65622088626748,8.25819620853794 +223,1,5,0.852925428022894,1.67333324392399,0.922675753730593 +223,2,5,0.852925428022894,1.16539499869153,3.04908307330675 +223,3,5,0.852925428022894,1.73698490260616,1.69916384611494 +223,4,5,0.852925428022894,1.13029237034851,4.50861305072888 +223,5,5,0.852925428022894,0.725190204392934,4.0963264949155 +223,6,5,0.852925428022894,1.49173024477212,7.68586493395678 +224,1,5,-0.517878257327904,-1.35732639663923,-1.23119186049687 +224,2,5,-0.517878257327904,1.11491866568095,-0.315709039321787 +224,3,5,-0.517878257327904,0.587780945805903,-0.369763432693986 +224,4,5,-0.517878257327904,0.927163157395854,-0.506417691026338 +224,5,5,-0.517878257327904,0.488711607226898,0.686305714138264 +224,6,5,-0.517878257327904,0.385312701261675,3.62404644990348 +225,1,5,-1.11257146628431,-0.475426417311899,-1.04280843910047 +225,2,5,-1.11257146628431,-1.18634048699766,-2.30358967343468 +225,3,5,-1.11257146628431,0.091173974222244,0.945624995795265 +225,4,5,-1.11257146628431,0.238514872098877,-1.99598194201174 +225,5,5,-1.11257146628431,0.425818445029469,1.50873654886708 +225,6,5,-1.11257146628431,1.05295883081395,2.07497420733641 +226,1,5,-1.81489141645341,-1.24259664095849,1.51719448330531 +226,2,5,-1.81489141645341,-0.554428904590342,1.41446259490764 +226,3,5,-1.81489141645341,-0.299740806185905,2.18468422027469 +226,4,5,-1.81489141645341,0.542734837538821,0.163871949464976 +226,5,5,-1.81489141645341,-0.298329304331962,0.435908366646806 +226,6,5,-1.81489141645341,0.414989726238207,1.96641824618135 +227,1,5,-0.260227815842348,-1.03005150290773,-0.019407951568893 +227,2,5,-0.260227815842348,0.468446628646404,0.0285719838885757 +227,3,5,-0.260227815842348,0.470855524402432,-0.174855338773838 +227,4,5,-0.260227815842348,1.44176617213929,0.69418426946141 +227,5,5,-0.260227815842348,0.226607556826976,4.60969384532652 +227,6,5,-0.260227815842348,0.906000249868074,3.70417043721014 +228,1,5,-0.248887440272954,-0.418765638490002,-0.0863136410068941 +228,2,5,-0.248887440272954,0.314102203471253,2.38681030771552 +228,3,5,-0.248887440272954,0.362571171005034,2.76383374351119 +228,4,5,-0.248887440272954,0.47788363062241,3.55624065831369 +228,5,5,-0.248887440272954,0.520531301129979,3.77951398822541 +228,6,5,-0.248887440272954,1.24345219848178,4.81422068100672 +229,1,5,-0.300211489798693,-0.603458799036538,0.129483577367022 +229,2,5,-0.300211489798693,0.379696846594982,2.39422380550729 +229,3,5,-0.300211489798693,0.285898323584938,0.56453671209308 +229,4,5,-0.300211489798693,-0.343957165096895,1.81928926476601 +229,5,5,-0.300211489798693,1.34665793798712,1.60250849670397 +229,6,5,-0.300211489798693,1.38600178614644,4.44490888146164 +230,1,5,2.18079606684413,0.0328589209121795,1.38593922837778 +230,2,5,2.18079606684413,2.20428194064908,3.38086181845178 +230,3,5,2.18079606684413,1.69611948536251,3.41464459135443 +230,4,5,2.18079606684413,2.64727843019363,4.4219208488337 +230,5,5,2.18079606684413,2.91539831678562,6.44465284207725 +230,6,5,2.18079606684413,2.02577390016517,10.3074908553895 +231,1,5,0.682258613138577,0.232553052940103,0.575192037116431 +231,2,5,0.682258613138577,0.934575833052082,2.93725402467604 +231,3,5,0.682258613138577,1.79765272992026,4.02869272214191 +231,4,5,0.682258613138577,0.700136142980352,5.18621671240476 +231,5,5,0.682258613138577,1.29305205341976,7.67676120198773 +231,6,5,0.682258613138577,1.47254977346884,6.40465083371395 +232,1,5,0.738564646474746,0.255708737415919,2.17066071793656 +232,2,5,0.738564646474746,1.30570948732827,1.61187659853227 +232,3,5,0.738564646474746,1.3389155585084,3.07698051626951 +232,4,5,0.738564646474746,2.51777450060082,3.29898038024719 +232,5,5,0.738564646474746,2.01416388727751,4.04999085321885 +232,6,5,0.738564646474746,1.14429575274794,6.22284218610352 +233,1,5,1.31891652792444,0.943432331705388,-0.0289136497162035 +233,2,5,1.31891652792444,0.978239063144288,2.06170299473121 +233,3,5,1.31891652792444,0.310622694521734,3.69752660270523 +233,4,5,1.31891652792444,0.68052796604515,1.5204378915292 +233,5,5,1.31891652792444,1.70099796725061,5.71280736379098 +233,6,5,1.31891652792444,2.00471022400003,5.58278554448763 +234,1,5,1.28907990310246,1.65412984713345,-0.602892382604196 +234,2,5,1.28907990310246,0.853258601018456,0.907141408866408 +234,3,5,1.28907990310246,1.60150771388762,3.24796159869254 +234,4,5,1.28907990310246,1.96933258012763,4.19436697567741 +234,5,5,1.28907990310246,1.27632561755083,4.99253953190101 +234,6,5,1.28907990310246,1.77582680830315,4.8548769273247 +235,1,5,0.978070722904636,0.948150182775719,-1.03874117061971 +235,2,5,0.978070722904636,0.543444277286829,0.925766870985002 +235,3,5,0.978070722904636,1.49477068531528,2.21839953506068 +235,4,5,0.978070722904636,1.56819489132473,1.93251425438558 +235,5,5,0.978070722904636,1.59527502367316,3.20318666486669 +235,6,5,0.978070722904636,1.35543245600248,6.86424987830461 +236,1,5,0.00643884641450284,-0.641212010033781,-0.575014663395219 +236,2,5,0.00643884641450284,0.937855659547919,2.06384914461294 +236,3,5,0.00643884641450284,0.739732482846854,1.41647034926797 +236,4,5,0.00643884641450284,1.10190774770209,3.79348717413345 +236,5,5,0.00643884641450284,1.02193117870911,3.04259873537341 +236,6,5,0.00643884641450284,0.0204587745911464,7.00891684792682 +237,1,5,0.0165814801249611,-0.305408606389655,1.76392967553631 +237,2,5,0.0165814801249611,0.267051028226902,-1.10004204212255 +237,3,5,0.0165814801249611,0.319994248367331,2.40546891854285 +237,4,5,0.0165814801249611,1.43776249361238,2.12131653600467 +237,5,5,0.0165814801249611,1.14569000055294,2.5595375324857 +237,6,5,0.0165814801249611,0.435442855090185,4.89078063646968 +238,1,5,1.91587682170855,1.28061275956914,-0.647304703918855 +238,2,5,1.91587682170855,0.682339854322889,1.91715709450631 +238,3,5,1.91587682170855,0.682233140309582,2.42845110380224 +238,4,5,1.91587682170855,1.19451909853685,5.00427468249585 +238,5,5,1.91587682170855,1.36885315652453,5.70056465024374 +238,6,5,1.91587682170855,1.52792382399838,8.49809489229398 +239,1,5,-1.76542576063516,-1.30871766053306,0.221759257176214 +239,2,5,-1.76542576063516,-0.706967060937396,-0.283031516205847 +239,3,5,-1.76542576063516,-0.0826534122809553,-0.199081921554736 +239,4,5,-1.76542576063516,0.553569425457922,0.804541456652704 +239,5,5,-1.76542576063516,-0.425046079826805,0.893639807836897 +239,6,5,-1.76542576063516,1.118581133231,2.48466150451466 +240,1,5,-1.47447432420743,-0.590776716929888,-2.04062301930976 +240,2,5,-1.47447432420743,-0.0386169438484504,-1.89717081094193 +240,3,5,-1.47447432420743,0.210964261447659,1.17208473689677 +240,4,5,-1.47447432420743,-0.640970403235413,-1.29107091503318 +240,5,5,-1.47447432420743,0.758534883730153,0.323916789750266 +240,6,5,-1.47447432420743,1.2629442004812,1.9383167523482 +241,1,5,0.444480192099237,0.618633887994417,-2.68240112037008 +241,2,5,0.444480192099237,0.386435895591268,-2.42533638325667 +241,3,5,0.444480192099237,0.0306556682985822,-0.208110794073889 +241,4,5,0.444480192099237,1.65754806459742,1.05287475605699 +241,5,5,0.444480192099237,1.65965765467233,3.85979072939338 +241,6,5,0.444480192099237,0.390040101044708,5.13972097810125 +242,1,5,-0.109370786918,0.180835200749669,0.469693708300177 +242,2,5,-0.109370786918,-0.241649823954673,1.68019556392288 +242,3,5,-0.109370786918,1.63054915289421,2.99946048149086 +242,4,5,-0.109370786918,1.79876724716694,1.70281395029541 +242,5,5,-0.109370786918,1.50933658861389,1.92243530107267 +242,6,5,-0.109370786918,1.44996003366407,3.62636572590989 +243,1,5,0.336040969026486,0.395984087004319,-0.510470501592253 +243,2,5,0.336040969026486,1.49213107911934,0.341057260343169 +243,3,5,0.336040969026486,1.0322243877222,1.5963528291938 +243,4,5,0.336040969026486,1.13980145115043,3.22875490267963 +243,5,5,0.336040969026486,1.02657502168611,2.99297209111054 +243,6,5,0.336040969026486,1.51889009798494,5.66833238619657 +244,1,5,0.0246226254591995,-0.310292434088861,0.138069972547055 +244,2,5,0.0246226254591995,0.141199815446702,1.29756197057001 +244,3,5,0.0246226254591995,0.643356826099255,1.84884603919037 +244,4,5,0.0246226254591995,1.15478197762161,1.39675030805328 +244,5,5,0.0246226254591995,0.339075524589951,4.5868265068948 +244,6,5,0.0246226254591995,1.31437861623794,5.24668297480788 +245,1,5,0.885781948931482,0.806314046758624,2.2654923874529 +245,2,5,0.885781948931482,1.18408803633258,3.43352381181165 +245,3,5,0.885781948931482,0.227934592981508,3.16745054969171 +245,4,5,0.885781948931482,1.55756174226992,3.35340671191667 +245,5,5,0.885781948931482,2.33813525718331,4.78860640648161 +245,6,5,0.885781948931482,1.99813377457044,6.51661354997833 +246,1,5,-0.722659484881553,-0.384792945271069,-1.98735622308168 +246,2,5,-0.722659484881553,0.314472242298869,-0.285445896415748 +246,3,5,-0.722659484881553,1.11495488518433,-1.08780307763077 +246,4,5,-0.722659484881553,0.467747700320704,0.240682059810906 +246,5,5,-0.722659484881553,1.33148113439858,0.926984623451972 +246,6,5,-0.722659484881553,1.01374841804555,2.50113616748474 +247,1,5,1.59162124543521,0.801382985950288,0.157749357181468 +247,2,5,1.59162124543521,1.08447653818498,0.966600766348528 +247,3,5,1.59162124543521,1.64504914121795,2.97737955065785 +247,4,5,1.59162124543521,1.66859567663211,4.27374442535286 +247,5,5,1.59162124543521,1.20129706760012,3.93965864036669 +247,6,5,1.59162124543521,1.85582140799441,7.47447291305973 +248,1,5,0.00628203166510098,0.967574078834003,-0.277598332529191 +248,2,5,0.00628203166510098,0.251219158257299,-1.35116285419275 +248,3,5,0.00628203166510098,0.37795289944178,0.394806803517492 +248,4,5,0.00628203166510098,1.38110864533361,0.390223429962069 +248,5,5,0.00628203166510098,2.14074652648849,3.0551896743306 +248,6,5,0.00628203166510098,1.2842173284638,5.41393151214885 +249,1,5,-0.157830993412902,0.225604784343245,-0.63926524186835 +249,2,5,-0.157830993412902,0.815653032758968,1.08592736946653 +249,3,5,-0.157830993412902,-0.926936488629974,0.70719821540352 +249,4,5,-0.157830993412902,0.99159892459773,3.42146939341312 +249,5,5,-0.157830993412902,1.55574470790068,5.93041240162439 +249,6,5,-0.157830993412902,1.14913138457702,3.7143475809707 +250,1,5,-0.486593733281175,0.496828444822754,0.484715409441276 +250,2,5,-0.486593733281175,0.050098683873394,3.15320629989067 +250,3,5,-0.486593733281175,0.250532784667647,1.5334139004638 +250,4,5,-0.486593733281175,-1.07335766406325,-0.164378577396767 +250,5,5,-0.486593733281175,0.0773300025131354,3.30694573947018 +250,6,5,-0.486593733281175,1.12906863829978,4.46490543466803 diff --git a/changelog.d/20260831-twfe-weight-diagnostics.md b/changelog.d/20260831-twfe-weight-diagnostics.md new file mode 100644 index 000000000..25b3c8cd7 --- /dev/null +++ b/changelog.d/20260831-twfe-weight-diagnostics.md @@ -0,0 +1,20 @@ +### Added +- **TWFE weight diagnostics** (port of Brantly Callaway's `twfeweights` R + package, MIT): what a two-way fixed effects regression *implicitly* weights + on staggered-adoption data. + - `attgt_weights(results, aggregation="twfe"|"overall"|"simple")` reports the + weight a TWFE regression, ATT^O, or ATT^simple places on each ATT(g,t), + plus post-period negative-weight counts. Returns `ATTGTWeightsResult`. + - `decompose_twfe_weights(data, ..., method="fwl")` re-derives the estimate + from its ATT(g,t) building blocks and returns `TWFEDecompositionResult` + with `pretrend_bias` - the contribution of pre-treatment cells, i.e. of + parallel-trends violations rather than of treatment - and, with + `balance_covariates=`, implicit-weight covariate balance. + `plot_twfe_weights()` renders either view (matplotlib or plotly). + - Validation: rejects NaN / `-inf` cohort labels, covariate-adjusted fits + under `aggregation="twfe"`, duplicated or non-finite ATT(g,t) cells, an + incomplete group-time grid, and invalid sampling weights. Two structural + gaps are handled as R does instead of raising: a cohort with no estimable + post cell is dropped (`did`'s first-period drop), and under + `control_group="not_yet_treated"` the CS estimands average over each + cohort's available post periods (`aggte`). diff --git a/diff_diff/__init__.py b/diff_diff/__init__.py index 868ec0b63..80afc5dfa 100644 --- a/diff_diff/__init__.py +++ b/diff_diff/__init__.py @@ -296,6 +296,14 @@ TROPResults, trop, ) +from diff_diff.twfe_weights import ( + attgt_weights, + decompose_twfe_weights, +) +from diff_diff.twfe_weights_results import ( + ATTGTWeightsResult, + TWFEDecompositionResult, +) from diff_diff.two_stage import ( TwoStageBootstrapResults, TwoStageDiD, @@ -321,6 +329,7 @@ plot_sensitivity, plot_staircase, plot_synth_weights, + plot_twfe_weights, ) from diff_diff.wooldridge import WooldridgeDiD from diff_diff.wooldridge_results import WooldridgeDiDResults @@ -457,6 +466,13 @@ def __getattr__(name: str) -> _Any: "TWFEWeightsResult", "chaisemartin_dhaultfoeuille", "twowayfeweights", + # TWFE weight diagnostics (Callaway `twfeweights` port) - distinct from + # the dCDH `twowayfeweights` surface above: these weight ATT(g,t) + # parameters, not (unit, time) cells. + "ATTGTWeightsResult", + "TWFEDecompositionResult", + "attgt_weights", + "decompose_twfe_weights", # WooldridgeDiD (ETWFE) "WooldridgeDiD", "WooldridgeDiDResults", @@ -473,6 +489,7 @@ def __getattr__(name: str) -> _Any: "SieveLearner", # Visualization "plot_bacon", + "plot_twfe_weights", "plot_event_study", "plot_group_effects", "plot_sensitivity", diff --git a/diff_diff/dml_did.py b/diff_diff/dml_did.py index aa0e15a2f..42ec832d2 100644 --- a/diff_diff/dml_did.py +++ b/diff_diff/dml_did.py @@ -2411,6 +2411,7 @@ def fit( # stays admitted). is_survey_fit=survey_metadata is not None, bootstrap_results=bootstrap_results, + covariates=covariates, ) self.results_ = results self.is_fitted_ = True diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 39e3cb9f5..bbcd3fde0 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1503,6 +1503,78 @@ results.print_summary() plot_bacon(results) ``` +### TWFE Weight Diagnostics + +What a TWFE regression implicitly weights on staggered data. Distinct from +`twowayfeweights` (dCDH), which weights (unit, time) cells: these weight +ATT(g,t) parameters. Ported from Brantly Callaway's `twfeweights` R package +(MIT); methodology Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna +(2025). + +```python +attgt_weights( + results, # CallawaySantAnnaResults, or a (g,t) frame + aggregation="twfe", # "twfe" | "overall" (ATT^O) | "simple" + data=None, unit=None, time=None, first_treat=None, # frame path only + weights=None, # unit-level sampling weights +) -> ATTGTWeightsResult + +decompose_twfe_weights( + data, # balanced long panel (it re-estimates) + outcome=, unit=, time=, first_treat=, + method="fwl", + covariates=None, + base_period="first_period", # or "gmin1" + balance_covariates=None, # enables result.covariate_balance() + weights=None, +) -> TWFEDecompositionResult + +plot_twfe_weights(result, kind="auto") # "weights" | "balance" +``` + +`aggregation="twfe"` requires a fit with `base_period="universal"`, +`control_group="never_treated"` AND no covariates (R twfe_weights' three +restrictions); it raises otherwise. ATT^O and ATT^simple weights are +non-negative and sum to one, so comparing `implied_att` across the three +aggregations shows what the TWFE specification costs. + +Both entry points fail closed on input R never faced: NaN / `-inf` cohort +labels (never-treated is exactly `0` or `+inf`), duplicated or non-finite +ATT(g,t) cells, an incomplete group-time grid (`"twfe"` needs every cohort x +period cell, the CS estimands every post cell), and sampling weights that are +not finite, non-negative and positive-mass. Two structural gaps mirror R +rather than raising, each with a `UserWarning`: a cohort with no estimable +post cell is dropped from the table and the cohort shares (`did`'s +first-period drop), and under `control_group="not_yet_treated"` the cells CS +marks `zero_treated_control` are treated as structurally absent, so +`"overall"`/`"simple"` average over each cohort's AVAILABLE post periods +(`aggte`). `n_negative_post` / `negative_post_weight_share` report the +pathology (negative weight on POST cells); `n_negative` counts pre cells too, +and is near-half in every staggered design because the TWFE weights sum to +zero over the full grid. + +### plot_twfe_weights + +```python +plot_twfe_weights( + results, # ATTGTWeightsResult | TWFEDecompositionResult + kind="auto", # "weights" | "balance" ("auto" picks balance + # when the result carries a balance table) + standardize=True, absolute_value=True, # balance view + annotate=False, ax=None, show=True, + backend="matplotlib", # or "plotly" +) +``` + +`kind="weights"` scatters weight against ATT(g,t), coloured by pre/post - points +left of the vertical zero line carry negative weight. `kind="balance"` scatters +unweighted against implicitly-weighted covariate differences; points near the +horizontal axis are covariates the implicit weights balance. + +`decompose_twfe_weights` takes the raw panel rather than a fitted result +because it re-estimates. It is tied to `attgt_weights` by an identity: +`attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate`. + ### StaggeredTripleDifference DEPRECATED in 3.9, removed in 4.0 (ledger row M-013). Use @@ -1967,6 +2039,33 @@ Returned by `BaconDecomposition.fit()` (and the deprecated `bacon_decompose()` w **Methods:** `summary()`, `print_summary()`, `to_dataframe()` +### ATTGTWeightsResult + +Diagnostic result from `attgt_weights`. No inference quintet - the +decomposition is an algebraic identity. + +- `weights`: DataFrame with `group`, `time`, `post`, `weight`, `att` +- `implied_att`: `sum(weight * att)` - the TWFE coefficient when + `aggregation="twfe"` +- `n_negative`, `negative_weight_share`: the staggered-TWFE pathology +- `aggregation`, `source`, `control_group`, `base_period`, `n_cells` +- `summary()`, `to_dataframe()`, `to_dict()` + +### TWFEDecompositionResult + +Diagnostic result from `decompose_twfe_weights`. + +- `cells`: DataFrame with `group`, `time`, `post`, `att`, `weight`, `ess`, + `remainder` +- `estimate` == `decomposition` + `remainder` +- `pretrend_bias`: contribution of PRE-treatment cells, i.e. of + parallel-trends violations rather than of treatment +- `post_only`, `effective_sample_size`, `covariates`, `base_period` +- `covariate_balance(level="summary"|"cell", standardize=True, + post_only=True)`: implicit-weight covariate balance; raises when + `balance_covariates=` was not requested +- `summary()`, `to_dataframe()`, `to_dict()` + ### Comparison2x2 Individual 2x2 DiD comparison (used in BaconDecompositionResults). diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index f9220d90c..b81a2018e 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -90,6 +90,7 @@ The site is organized into 5 sections, each with a landing page: - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html): Cattaneo, Jansson & Ma (2020) density-discontinuity manipulation test (`RDDensityTest`), parity with R rddensity 3.0 - boundary-adaptive local polynomial density estimation at the cutoff, robust bias-corrected inference, unrestricted/restricted models, jackknife/plugin variances, data-driven bandwidths, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Placebo timing, group, permutation, and leave-one-out diagnostics +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html): Baker et al. (2025) implicit weights on ATT(g,t) - `attgt_weights(results, aggregation='twfe'|'overall'|'simple')` takes a fitted `CallawaySantAnnaResults` (raw ATT(g,t) frame + panel as fallback) and returns the weight each estimand places on each group-time effect, with the negative-weight share; `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, method='fwl', covariates=)` re-derives the TWFE estimate from its ATT(g,t) building blocks with `pretrend_bias`, and `result.covariate_balance()` reports implicit-weight covariate balance. Plot with `plot_twfe_weights`. R `twfeweights` 0.9.0 output parity - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html): Rambachan & Roth (2023) sensitivity analysis — robust CI under parallel trends violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html): Roth (2022) Section II.A-B no-individually-significant (NIS) box-probability pretest power + minimum detectable violation; `pretest_form='nis'` (default) implements the paper's primary form, `pretest_form='wald'` retained as paper-supported alternative (Propositions 1+3+4 all apply); linear-violation MDV in Roth's γ units when relative-time labels are threaded through `fit()`; full Σ_22 routing on non-bootstrap CallawaySantAnna and SunAbraham adapters and on admitted CS-/StackedDiD-sourced `aggregate('event_study')` containers (StackedDiD persists its ES VCV in every inference mode) - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html): Analytical and simulation-based power analysis — MDE, sample size, power curves for study design diff --git a/diff_diff/staggered.py b/diff_diff/staggered.py index 37a544907..8278d950c 100644 --- a/diff_diff/staggered.py +++ b/diff_diff/staggered.py @@ -7,7 +7,7 @@ import bisect import warnings -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple import numpy as np import pandas as pd @@ -3056,6 +3056,7 @@ def fit( group_time_effects, is_survey_fit=survey_metadata is not None, bootstrap_results=bootstrap_results, + covariates=covariates, ) self.is_fitted_ = True @@ -5151,6 +5152,7 @@ def _build_aggregation_kit( *, is_survey_fit: bool = False, bootstrap_results: Optional["CSBootstrapResults"] = None, + covariates: Optional[Sequence[str]] = None, ) -> Optional["AggregationKit"]: """Distil the fit-time state post-fit re-aggregation needs. @@ -5185,6 +5187,11 @@ def _build_aggregation_kit( # (or DDD) survey fit does not warn as "CallawaySantAnna" on post-fit # aggregate(). Legacy kits without the key default at the read site. bookkeeping["bootstrap_label"] = getattr(estimator, "_BOOTSTRAP_LABEL", "CallawaySantAnna") + # Covariate usage, recorded so downstream diagnostics can refuse designs + # their formulas do not cover (``attgt_weights(aggregation="twfe")`` + # mirrors R twfe_weights' ``xformla == ~1`` restriction). Column NAMES + # only - never values - so the data-minimization contract holds. + bookkeeping["covariates"] = tuple(covariates or ()) # Data minimization: the results object is picklable and users share # result artifacts, so the kit must not turn it into a carrier for raw diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py new file mode 100644 index 000000000..eda573824 --- /dev/null +++ b/diff_diff/twfe_weights.py @@ -0,0 +1,1625 @@ +"""Implicit TWFE weights on group-time average treatment effects. + +A two-way fixed effects regression run on staggered-adoption data does not +estimate a simple average of the underlying ATT(g, t). It estimates a +*weighted* average, and some of those weights can be negative - so the +coefficient need not lie in the convex hull of the effects it summarizes. +:func:`attgt_weights` reports those weights, next to the weights the target +estimands ATT^O and ATT^simple would use. :func:`decompose_twfe_weights` +re-derives the regression from its building blocks and separates the part +driven by pre-treatment parallel-trends violations. + +Distinct from :func:`diff_diff.twowayfeweights`, which implements the de +Chaisemartin & D'Haultfoeuille (2020) Theorem 1 decomposition: that one +weights ``(unit, time)`` cells, this one weights ATT(g, t) *parameters*. +Distinct also from :class:`diff_diff.BaconDecomposition`, which decomposes +TWFE into 2x2 DiD comparisons rather than into group-time effects. + +Ported from the R package ``twfeweights`` (version 0.9.0) by Brantly +Callaway, released under the MIT License. The upstream notice is reproduced +in full, as its terms require:: + + MIT License + + Copyright (c) 2023 Brantly Callaway + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Methodology: Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna (2025), +"Difference-in-Differences Designs: A Practitioner's Guide" +(arXiv:2503.13323); Callaway & Sant'Anna (2021) for the ATT^O / ATT^simple +weights. +""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union + +import numpy as np +import pandas as pd + +from diff_diff.linalg import solve_ols +from diff_diff.twfe_weights_results import ( + ATTGTWeightsResult, + TWFEDecompositionResult, +) +from diff_diff.utils import within_transform + +if TYPE_CHECKING: # pragma: no cover - typing only + from diff_diff.staggered_results import CallawaySantAnnaResults + +__all__ = ["attgt_weights", "decompose_twfe_weights"] + +_AGGREGATIONS = ("twfe", "overall", "simple") + + +def _is_never(values: np.ndarray) -> np.ndarray: + """Boolean mask for never-treated cohort labels. + + diff-diff and R ``did`` have both used ``0`` and ``+inf`` as the + never-treated sentinel over time; accept exactly those two and normalize + to ``0``. Every OTHER non-finite label (NaN, ``-inf``) is an input error - + see :func:`_validate_cohort_labels` - not a never-treated unit. + """ + arr = np.asarray(values, dtype=float) + return (arr == 0) | (arr == np.inf) + + +def _validate_cohort_labels( + values: np.ndarray, *, unit_ids: Optional[np.ndarray] = None, what: str = "first_treat" +) -> None: + """Reject NaN / ``-inf`` cohort labels instead of silently treating them as never-treated.""" + arr = np.asarray(values, dtype=float) + bad = np.isnan(arr) | (arr == -np.inf) + if bad.any(): + idx = np.flatnonzero(bad)[:5] + who = [unit_ids[i] for i in idx] if unit_ids is not None else idx.tolist() + raise ValueError( + f"{what!r} contains NaN or -inf cohort label(s) for unit(s) {who!r}; " + "never-treated units must be coded exactly 0 or +inf, and every " + "other unit needs a finite first-treatment period" + ) + + +def _validate_time_labels(values: np.ndarray, *, what: str = "time") -> None: + """Reject NaN / non-finite period labels before any grid is formed.""" + arr = pd.to_numeric(pd.Series(np.asarray(values)), errors="coerce").to_numpy(dtype=float) + bad = ~np.isfinite(arr) + if bad.any(): + raise ValueError( + f"{what!r} contains {int(bad.sum())} non-finite or non-numeric period " + f"label(s) (first at row {int(np.flatnonzero(bad)[0])}); every observation " + "must carry a finite period" + ) + + +def _positional_grid( + time_periods: Sequence[Any], +) -> Dict[float, int]: + """Map ordered period labels onto ``1..T``. + + R computes ``(maxT - g + 1) / length(tlist)`` directly on the raw period + labels, which is only correct when those labels are consecutive integers. + Working in positional time makes the same expression correct on gapped or + non-integer grids, and is bit-identical when the grid IS consecutive + (mpdta's 2003..2007 maps to 1..5 and both give 4/5 for g = 2004). + Recorded as a deviation in the methodology registry. + """ + ordered = sorted({float(t) for t in time_periods}) + return {t: i + 1 for i, t in enumerate(ordered)} + + +def _to_positional_cohort(cohorts: np.ndarray, grid: Dict[float, int]) -> np.ndarray: + """Cohort labels -> positional time; never-treated stays 0. + + Mirrors ``BMisc::orig2t``, which leaves the never-treated sentinel alone + under positional rescaling. + """ + out = np.zeros(len(cohorts), dtype=float) + never = _is_never(cohorts) + for i, (g, is_never) in enumerate(zip(cohorts, never)): + if is_never: + continue + key = float(g) + if key not in grid: + raise ValueError( + f"cohort label {g!r} is not one of the observed time periods " + f"{sorted(grid)!r}; cannot place it on the period grid" + ) + out[i] = grid[key] + return out + + +def _validate_unit_weights( + w: np.ndarray, is_never: np.ndarray, *, require_control_mass: bool +) -> None: + """Shared contract for unit-level sampling weights. + + Finite, non-negative, positive total, positive TREATED mass; positive + never-treated mass only where the never-treated group enters the formula + (``aggregation="twfe"`` and the decomposition) - ATT^O / ATT^simple are + defined without one. + """ + if not np.all(np.isfinite(w)): + raise ValueError("unit weights must be finite; got NaN or infinite weight(s)") + if (w < 0).any(): + idx = np.flatnonzero(w < 0)[:5].tolist() + raise ValueError( + f"unit weights must be non-negative; negative weight(s) at unit index {idx!r}" + ) + if w.sum() <= 0: + raise ValueError("unit weights sum to zero; cannot form cohort shares") + if w[~is_never].sum() <= 0: + raise ValueError( + "the ever-treated units carry zero total weight; cannot form cohort shares" + ) + if require_control_mass and w[is_never].sum() <= 0: + raise ValueError( + "the never-treated comparison group carries zero total weight, so " + "every group-time contrast is undefined" + ) + + +def _cohort_masses( + unit_cohorts: np.ndarray, + grid: Dict[float, int], + weights: Optional[np.ndarray], + *, + require_control_mass: bool = False, +) -> Tuple[Dict[int, float], Dict[int, float], Dict[int, float], float]: + """Cohort shares and treated-share-by-period, all in positional time. + + Returns + ------- + p_all : {positional g: share of ALL units in cohort g} + R's ``pg2`` - the denominator is every unit, never-treated included. + Used by the TWFE weights. + p_treated : {positional g: share of EVER-TREATED units in cohort g} + R's ``pg``. Used by the ATT^O / ATT^simple weights. + e_dt : {positional t: weighted share of units treated by t} + R's ``Edt(t)``. + mean_e_dt : float + R's ``mEdt`` - the average of ``e_dt`` over the period grid. + """ + g_pos = _to_positional_cohort(unit_cohorts, grid) + w = np.ones(len(g_pos)) if weights is None else np.asarray(weights, dtype=float) + if len(w) != len(g_pos): + raise ValueError(f"weights has length {len(w)} but there are {len(g_pos)} units") + _validate_unit_weights(w, g_pos == 0, require_control_mass=require_control_mass) + total = w.sum() + + treated = g_pos != 0 + treated_mass = w[treated].sum() + + cohorts = sorted({int(g) for g in g_pos if g != 0}) + p_all = {g: float(w[g_pos == g].sum() / total) for g in cohorts} + p_treated = {g: float(w[g_pos == g].sum() / treated_mass) for g in cohorts} + + periods = sorted(grid.values()) + e_dt = {t: float(w[treated & (g_pos <= t)].sum() / total) for t in periods} + mean_e_dt = float(np.mean([e_dt[t] for t in periods])) + return p_all, p_treated, e_dt, mean_e_dt + + +def _twfe_weight_vector( + groups: np.ndarray, + times: np.ndarray, + n_periods: int, + p_all: Dict[int, float], + e_dt: Dict[int, float], + mean_e_dt: float, +) -> np.ndarray: + """Weights a static TWFE regression places on each ATT(g, t). + + ``h(g,t) = 1[t >= g] - (maxT - g + 1)/T - E_t[D] + mean_t E_t[D]`` + ``num(g,t) = h(g,t) * p_g``, normalized by the sum over post cells. + + All arguments are in positional time, so ``maxT == n_periods``. + """ + h = ( + (times >= groups).astype(float) + - (n_periods - groups + 1.0) / n_periods + - np.array([e_dt[int(t)] for t in times]) + + mean_e_dt + ) + num = h * np.array([p_all[int(g)] for g in groups]) + post = times >= groups + denom = num[post].sum() + if denom == 0: + raise ValueError( + "TWFE weight normalization is degenerate (post-treatment weights " + "sum to zero); the regression has no identifying variation" + ) + return num / denom + + +def _overall_weight_vector( + groups: np.ndarray, + times: np.ndarray, + n_periods: int, + p_treated: Dict[int, float], + n_post_available: Optional[Dict[int, int]] = None, +) -> np.ndarray: + """ATT^O weights: ``1[t >= g] * pbar_g / (maxT - g + 1)``. + + Not renormalized - the ``(maxT - g + 1)`` divisor already makes them sum + to one over a complete post-treatment grid. ``n_post_available`` replaces + that divisor with each cohort's number of AVAILABLE post periods when + some post cells are structurally absent (``control_group="not_yet_treated"`` + runs out of comparison units) - what R ``aggte(type="group")`` averages + over on such a fit. + """ + if n_post_available is None: + divisor = n_periods - groups + 1.0 + else: + divisor = np.array([float(n_post_available[int(g)]) for g in groups]) + return (times >= groups).astype(float) * np.array([p_treated[int(g)] for g in groups]) / divisor + + +def _simple_weight_vector( + groups: np.ndarray, + times: np.ndarray, + p_treated: Dict[int, float], +) -> np.ndarray: + """ATT^simple weights: ``1[t >= g] * pbar_g``, normalized to sum to one.""" + raw = (times >= groups).astype(float) * np.array([p_treated[int(g)] for g in groups]) + total = raw.sum() + if total == 0: + raise ValueError( + "ATT^simple weight normalization is degenerate (no post-treatment " + "cells carry weight)" + ) + return raw / total + + +def _attgt_from_cs( + results: "CallawaySantAnnaResults", +) -> Tuple[pd.DataFrame, Dict[Tuple[Any, Any], Optional[str]]]: + """Extract the ``(g, t, att)`` table from a fitted CS result. + + Non-estimable cells (``skip_reason`` set, NaN effect) are left out of the + table and reported in the returned ``{(g, t): skip_reason}`` map, so the + caller can decide - per aggregation - whether the gap is structural, a + harmless pre-period drop, or a hard error. + """ + rows: List[Dict[str, Any]] = [] + skipped: Dict[Tuple[Any, Any], Optional[str]] = {} + for (g, t), cell in results.group_time_effects.items(): + effect = cell.get("effect", np.nan) + if cell.get("skip_reason") is not None or not np.isfinite(effect): + skipped[(g, t)] = cell.get("skip_reason") + continue + rows.append({"group": g, "time": t, "att": float(effect)}) + if not rows: + raise ValueError( + "the fitted result has no estimable group-time cells; there is nothing to weight" + ) + table = pd.DataFrame(rows).sort_values(["group", "time"]).reset_index(drop=True) + return table, skipped + + +def _attgt_from_frame( + frame: pd.DataFrame, +) -> Tuple[pd.DataFrame, Dict[Tuple[Any, Any], Optional[str]]]: + """Extract ``(g, t, att)`` from a user-supplied ATT(g, t) frame. + + ``effect`` is preferred over ``att`` because that is the column + ``CallawaySantAnnaResults.to_dataframe("group_time")`` emits - so the + fallback consumes our own frame verbatim, including its ``skip_reason`` + column when present. Duplicate cells and non-finite ``group`` / ``time`` + labels are rejected; a non-finite effect is reported in the skip map, not + silently kept (an ``inf`` ATT would otherwise propagate into + ``implied_att``). + """ + missing = {"group", "time"} - set(frame.columns) + if missing: + raise ValueError( + f"ATT(g,t) frame is missing required column(s) {sorted(missing)!r}; " + "expected 'group', 'time', and one of 'effect' / 'att'" + ) + for candidate in ("effect", "att"): + if candidate in frame.columns: + value_col = candidate + break + else: + raise ValueError( + "ATT(g,t) frame must carry an 'effect' or 'att' column; got " f"{list(frame.columns)!r}" + ) + groups = pd.to_numeric(frame["group"], errors="coerce").to_numpy(dtype=float) + times = pd.to_numeric(frame["time"], errors="coerce").to_numpy(dtype=float) + _validate_cohort_labels(groups, what="group") + _validate_time_labels(frame["time"].to_numpy(), what="time") + key = pd.MultiIndex.from_arrays([frame["group"].to_numpy(), frame["time"].to_numpy()]) + if key.duplicated().any(): + dupes = sorted({tuple(k) for k in key[key.duplicated()].tolist()})[:5] + raise ValueError( + f"ATT(g,t) frame has duplicated (group, time) cell(s) {dupes!r}; each " + "cell must appear exactly once" + ) + att = pd.to_numeric(frame[value_col], errors="coerce").to_numpy(dtype=float) + reasons = ( + frame["skip_reason"].tolist() if "skip_reason" in frame.columns else [None] * len(frame) + ) + table = pd.DataFrame( + {"group": frame["group"].to_numpy(), "time": frame["time"].to_numpy(), "att": att} + ) + finite = np.isfinite(att) + skipped: Dict[Tuple[Any, Any], Optional[str]] = {} + for i in np.flatnonzero(~finite): + reason = reasons[i] + skipped[(table["group"].iat[i], table["time"].iat[i])] = ( + None + if reason is None or (isinstance(reason, float) and np.isnan(reason)) + else str(reason) + ) + table = table[finite] + if table.empty: + raise ValueError("ATT(g,t) frame has no finite effects to weight") + _ = groups, times # validated above; positional mapping happens in the caller + return table.sort_values(["group", "time"]).reset_index(drop=True), skipped + + +def _unit_cohorts_from_frame( + data: pd.DataFrame, unit: str, time: str, first_treat: str +) -> Tuple[np.ndarray, np.ndarray, Optional[np.ndarray]]: + """Collapse a long panel to one cohort label per unit.""" + for col in (unit, time, first_treat): + if col not in data.columns: + raise ValueError(f"column {col!r} not found in data") + _validate_time_labels(data[time].to_numpy(), what=time) + # dropna=False: a unit whose label is NaN in one period must fail the + # invariance check, not slip through because nunique() skipped the NaN. + per_unit = data.groupby(unit, sort=True)[first_treat].nunique(dropna=False) + if (per_unit > 1).any(): + offenders = per_unit[per_unit > 1].index.tolist()[:5] + raise ValueError( + f"{first_treat!r} varies within unit(s) {offenders!r}; cohort " + "membership must be time-invariant" + ) + firsts = data.groupby(unit, sort=True)[first_treat].first() + cohorts = firsts.to_numpy() + _validate_cohort_labels(cohorts, unit_ids=firsts.index.to_numpy(), what=first_treat) + periods = np.asarray(sorted(data[time].unique())) + return cohorts, periods, None + + +def _resolve_cs_inputs( + results: "CallawaySantAnnaResults", +) -> Tuple[np.ndarray, Optional[np.ndarray]]: + """Read cohort labels (and survey weights) off a fitted CS result. + + The aggregation kit is package-internal, but it is the same channel + ``CallawaySantAnnaResults._aggregate_compute`` already uses - so this is + an established in-package coupling rather than a new one. When the kit is + absent (an old pickle), the caller is pointed at the ``data=`` fallback. + """ + kit = getattr(results, "_aggregation_kit", None) + if kit is None: + raise ValueError( + "this CallawaySantAnnaResults carries no aggregation bookkeeping " + "(it may have been unpickled from an older version), so cohort " + "shares cannot be recovered from it. Pass the panel explicitly:\n" + " attgt_weights(result.to_dataframe('group_time'), data=panel,\n" + " unit=..., time=..., first_treat=...)" + ) + bookkeeping = getattr(kit, "bookkeeping", {}) or {} + cohorts = bookkeeping.get("unit_cohorts") + if cohorts is None: + raise ValueError( + "aggregation bookkeeping does not carry 'unit_cohorts'; pass the " + "panel explicitly via data=/unit=/time=/first_treat=" + ) + weights = bookkeeping.get("survey_weights") + return np.asarray(cohorts), (None if weights is None else np.asarray(weights, dtype=float)) + + +def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> None: + """Reject fits whose design breaks the weight formulas. + + These are hard errors rather than warnings: a silently wrong weight table + is worse than no weight table, and every one of these has a concrete fix. + """ + from diff_diff.staggered_results import CallawaySantAnnaResults + + if not isinstance(results, CallawaySantAnnaResults): + raise TypeError( + "attgt_weights takes a CallawaySantAnna (or DMLDiD) fitted result, or an " + f"ATT(g,t) DataFrame; got {type(results).__name__}" + ) + if not getattr(results, "panel", True): + raise ValueError( + "attgt_weights requires a panel fit: E_t[D] and the cohort shares " + "average over a fixed set of units, which repeated cross-sections " + "do not provide. Refit with panel=True." + ) + if getattr(results, "used_rc_on_unbalanced_panel", False): + raise ValueError( + "this fit fell back to repeated-cross-section estimation on an " + "unbalanced panel, so the cohort shares are not comparable across " + "periods. Balance the panel (diff_diff.balance_panel) and refit." + ) + if aggregation != "twfe": + return + control_group = getattr(results, "control_group", None) + if control_group not in (None, "never_treated"): + raise ValueError( + f"aggregation='twfe' requires control_group='never_treated', got " + f"{control_group!r}. The TWFE weight formula is derived against a " + "never-treated comparison group (matching R's twfe_weights, which " + "raises the same restriction)." + ) + base_period = getattr(results, "base_period", None) + if base_period not in (None, "universal"): + raise ValueError( + f"aggregation='twfe' requires base_period='universal', got " + f"{base_period!r}. The formula needs the complete cohort x period " + "grid, including the pre-treatment cells that a varying base does " + "not report. Refit with base_period='universal'." + ) + # R's third restriction: xformla == ~1. The fit records its covariate + # column names on the aggregation kit; a kit without the key predates that + # bookkeeping (an old pickle) and can only be warned about. A missing kit + # is left to _resolve_cs_inputs, whose error is the useful one. + kit = getattr(results, "_aggregation_kit", None) + if kit is None: + return + bookkeeping = getattr(kit, "bookkeeping", {}) or {} + if "covariates" not in bookkeeping: + warnings.warn( + "this fit predates covariate bookkeeping, so attgt_weights cannot " + "verify it used no covariates; the TWFE weight formula assumes an " + "unadjusted regression (R twfe_weights requires xformla == ~1)", + UserWarning, + stacklevel=3, + ) + elif bookkeeping["covariates"]: + raise ValueError( + f"aggregation='twfe' requires a fit without covariates, but this one " + f"adjusted for {list(bookkeeping['covariates'])!r}. The TWFE weight " + "formula describes the unadjusted regression (R's twfe_weights stops " + "unless xformla == ~1); refit with covariates=None, or use " + "decompose_twfe_weights(covariates=...) for the covariate-adjusted " + "decomposition." + ) + + +def attgt_weights( + results: Union["CallawaySantAnnaResults", pd.DataFrame], + *, + aggregation: str = "twfe", + data: Optional[pd.DataFrame] = None, + unit: Optional[str] = None, + time: Optional[str] = None, + first_treat: Optional[str] = None, + weights: Optional[Union[str, np.ndarray]] = None, +) -> ATTGTWeightsResult: + """Weights an estimand places on each group-time effect ATT(g, t). + + Three estimands are available. ``"twfe"`` gives the weights implied by a + static two-way fixed effects regression - the ones that can go negative. + ``"overall"`` and ``"simple"`` give the weights of the Callaway & + Sant'Anna (2021) target parameters ATT^O and ATT^simple, which are + non-negative by construction. Comparing them shows how far the regression + is from the estimand you meant to report. + + Parameters + ---------- + results : CallawaySantAnnaResults or pd.DataFrame + A fitted Callaway & Sant'Anna result (preferred), or a frame with + ``group`` / ``time`` / ``effect`` (or ``att``) columns - the output of + ``result.to_dataframe("group_time")`` is consumed verbatim, including + its ``skip_reason`` column. On the frame path, ``data``, ``unit``, + ``time`` and ``first_treat`` are required so cohort shares can be + formed, and the caller is responsible for the fit having used no + covariates under ``aggregation="twfe"`` (a frame carries no record of + that; the fitted path checks it). + aggregation : {"twfe", "overall", "simple"}, default "twfe" + Which estimand's weights to report. + data : pd.DataFrame, optional + Balanced panel backing the ATT(g, t) frame. Only for the fallback + path; passing it alongside a fitted result raises. + unit, time, first_treat : str, optional + Column names in ``data``. Required together with ``data``. + weights : str or array-like, optional + Unit-level sampling weights (R's ``w=``): a column name in ``data``, + or one value per unit. Rejected when the fit already carries survey + weights, which take precedence. Must be finite and non-negative with + positive treated mass (and positive never-treated mass for ``"twfe"``). + + Returns + ------- + ATTGTWeightsResult + Per-cell weights plus the negative-weight roll-ups. + + Raises + ------ + ValueError + On an unknown ``aggregation``; on a design the formula does not + support (repeated cross-sections, unbalanced fallback, and - for + ``aggregation="twfe"`` - a non-never-treated control group, a + non-universal base period, or a covariate-adjusted fit); on NaN / + ``-inf`` cohort labels, invalid weights, duplicated or non-finite + cells; or on an INCOMPLETE grid: ``"twfe"`` needs every cohort x period + cell, ``"overall"`` / ``"simple"`` every post-treatment cell. + TypeError + When ``results`` is neither a CallawaySantAnna-family result nor a + DataFrame. + + Notes + ----- + Two structural gaps are handled rather than raised, mirroring R: + + * A cohort with NO estimable post-treatment cell (typically one treated in + the first observed period, which has no base period) is dropped from the + table AND from the cohort masses with a warning - what + ``did::pre_process_did`` does when it drops units already treated in the + first period. + * Under ``control_group="not_yet_treated"`` the last cohorts run out of + comparison units, and CS marks those post cells ``zero_treated_control``. + For ``"overall"`` / ``"simple"`` they are treated as structurally absent: + ``"overall"`` divides each cohort by its number of AVAILABLE post periods + and ``"simple"`` renormalizes over the available post cells - what + R ``aggte()`` computes on such a fit. A warning names the cells. + (``"twfe"`` requires a never-treated control group and never reaches + this branch.) + + R's ``keep_untreated=TRUE`` is not exposed. It synthesizes ``G = 0`` rows + with ``attgt = 0`` to mirror an internal vector layout; those rows are + excluded from every normalization and contribute exactly zero, so the + argument does not affect any number. + + Examples + -------- + >>> import diff_diff # doctest: +SKIP + >>> cs = diff_diff.CallawaySantAnna(base_period="universal") # doctest: +SKIP + >>> res = cs.fit(df, outcome="y", unit="id", time="t", + ... first_treat="g") # doctest: +SKIP + >>> w = diff_diff.attgt_weights(res, aggregation="twfe") # doctest: +SKIP + >>> print(w.summary()) # doctest: +SKIP + """ + if aggregation not in _AGGREGATIONS: + raise ValueError( + f"aggregation must be one of {list(_AGGREGATIONS)!r}, got " f"{aggregation!r}" + ) + + frame_path = isinstance(results, pd.DataFrame) + frame = results if isinstance(results, pd.DataFrame) else None + fallback_args = {"data": data, "unit": unit, "time": time, "first_treat": first_treat} + supplied = {k: v for k, v in fallback_args.items() if v is not None} + + if frame_path: + if len(supplied) != 4: + missing = sorted(set(fallback_args) - set(supplied)) + raise ValueError( + "the DataFrame path needs the panel too, so cohort shares can " + f"be formed; missing {missing!r}. Call it as:\n" + " attgt_weights(gt_frame, data=panel, unit='id', " + "time='t', first_treat='g')" + ) + assert data is not None and unit is not None + assert time is not None and first_treat is not None + assert frame is not None + table, skipped = _attgt_from_frame(frame) + cohorts, periods, _ = _unit_cohorts_from_frame(data, unit, time, first_treat) + unit_weights = _resolve_frame_weights(weights, data, unit) + source = "DataFrame" + control_group = None + base_period = None + has_skip_reasons = "skip_reason" in frame.columns + else: + if supplied: + raise ValueError( + f"{sorted(supplied)!r} are only for the DataFrame fallback. A " + "fitted CallawaySantAnnaResults already carries the cohort " + "bookkeeping - drop them, or pass " + "result.to_dataframe('group_time') as the first argument." + ) + _guard_cs_design(results, aggregation) + table, skipped = _attgt_from_cs(results) + cohorts, survey_weights = _resolve_cs_inputs(results) + if survey_weights is not None and weights is not None: + raise ValueError( + "this fit already carries survey weights; passing weights= as " + "well is ambiguous. Drop weights= to use the fit's own." + ) + if weights is not None and not isinstance(weights, str): + unit_weights = np.asarray(weights, dtype=float) + elif isinstance(weights, str): + raise ValueError( + "weights= may only name a column on the DataFrame path; pass " + "an array of per-unit weights instead" + ) + else: + unit_weights = survey_weights + periods = np.asarray(results.time_periods) + source = "CallawaySantAnnaResults" + control_group = getattr(results, "control_group", None) + base_period = getattr(results, "base_period", None) + has_skip_reasons = True + + _validate_cohort_labels(cohorts, what="first_treat") + grid = _positional_grid(periods) + n_periods = len(grid) + first_period_pos = 1 + + # Positional mapping FIRST: the cohort universe the masses are formed over + # must be known before the masses are formed. + unit_g_pos = _to_positional_cohort(cohorts, grid) + if not (unit_g_pos != 0).any(): + raise ValueError( + "no ever-treated units found; cohort labels are all never-treated " + "sentinels (0 or inf)" + ) + g_pos = _to_positional_cohort(table["group"].to_numpy(), grid) + t_pos = np.array([grid[float(t)] for t in table["time"].to_numpy()]) + post_mask = t_pos >= g_pos + + # --- whole-cohort exclusion (R did drops units treated in the first period) + panel_cohorts = sorted({int(g) for g in unit_g_pos if g != 0}) + cohorts_with_post = {int(g) for g in g_pos[post_mask]} + excluded = [g for g in panel_cohorts if g not in cohorts_with_post] + if excluded: + if not has_skip_reasons: + not_structural = [g for g in excluded if g != first_period_pos] + if not_structural: + labels = [_label_for(grid, g) for g in not_structural] + raise ValueError( + f"cohort(s) {labels!r} are present in data= but have no " + "post-treatment cell in the ATT(g,t) frame. A bare frame " + "cannot say why; pass result.to_dataframe('group_time') " + "verbatim (it carries skip_reason) or the fitted result itself." + ) + n_units_excl = int(np.isin(unit_g_pos, excluded).sum()) + warnings.warn( + f"cohort(s) {[_label_for(grid, g) for g in excluded]!r} ({n_units_excl} " + "unit(s)) have no estimable post-treatment cell and were dropped from " + "the weight table and the cohort shares, matching R did's drop of units " + "already treated in the first observed period", + UserWarning, + stacklevel=2, + ) + keep_units = ~np.isin(unit_g_pos, excluded) + cohorts = cohorts[keep_units] + unit_g_pos = unit_g_pos[keep_units] + if unit_weights is not None: + unit_weights = np.asarray(unit_weights, dtype=float)[keep_units] + keep_rows = ~np.isin(g_pos, excluded) + table = table[keep_rows].reset_index(drop=True) + g_pos, t_pos, post_mask = g_pos[keep_rows], t_pos[keep_rows], post_mask[keep_rows] + skipped = {k: v for k, v in skipped.items() if _pos_of(grid, k[0]) not in excluded} + + p_all, p_treated, e_dt, mean_e_dt = _cohort_masses( + cohorts, grid, unit_weights, require_control_mass=(aggregation == "twfe") + ) + + # --- grid completeness + present = set(zip(g_pos.tolist(), t_pos.tolist())) + surviving = sorted(cohorts_with_post) + if aggregation == "twfe": + required = {(g, t) for g in surviving for t in range(1, n_periods + 1)} + else: + required = {(g, t) for g in surviving for t in range(g, n_periods + 1)} + missing_cells = sorted(required - present) + structurally_absent: List[Tuple[Any, Any]] = [] + if missing_cells: + carve_out_ok = aggregation != "twfe" and control_group == "not_yet_treated" + hard: List[Tuple[Tuple[Any, Any], Optional[str]]] = [] + for g, t in missing_cells: + label = (_label_for(grid, g), _label_for(grid, t)) + reason = skipped.get(label) + if carve_out_ok and reason == "zero_treated_control": + structurally_absent.append(label) + else: + hard.append((label, reason)) + if hard: + what = "cohort x period" if aggregation == "twfe" else "post-treatment" + detail = ", ".join( + f"{lab} [{reason or 'not in source table'}]" for lab, reason in hard[:6] + ) + raise ValueError( + f"aggregation={aggregation!r} needs the complete {what} grid, but " + f"{len(hard)} required cell(s) are missing: {detail}. A weight table " + "over a partial grid is not the named estimand. Fix the source fit " + "(or pass the complete to_dataframe('group_time') output)." + ) + warnings.warn( + f"{len(structurally_absent)} post-treatment cell(s) {structurally_absent[:6]!r} " + "have no not-yet-treated comparison units (skip_reason " + "'zero_treated_control') and are treated as structurally absent: " + f"aggregation={aggregation!r} averages over each cohort's AVAILABLE " + "post periods, as R aggte() does on a not-yet-treated fit", + UserWarning, + stacklevel=2, + ) + + # Non-estimable PRE cells of surviving cohorts are the only drops left; + # the CS estimands ignore pre cells, so they change nothing. + dropped = sum( + 1 + for (g_lab, t_lab) in skipped + if _pos_of(grid, g_lab) in cohorts_with_post and _pos_of(grid, t_lab) < _pos_of(grid, g_lab) + ) + if dropped and aggregation != "twfe": + warnings.warn( + f"{dropped} pre-treatment group-time cell(s) had no estimable ATT(g,t) " + f"and were excluded; aggregation={aggregation!r} places no weight on " + "pre-treatment cells, so the weights are unaffected", + UserWarning, + stacklevel=2, + ) + + if aggregation == "twfe": + weight_vec = _twfe_weight_vector(g_pos, t_pos, n_periods, p_all, e_dt, mean_e_dt) + elif aggregation == "overall": + n_post_available = None + if structurally_absent: + n_post_available = {g: int(((g_pos == g) & post_mask).sum()) for g in surviving} + weight_vec = _overall_weight_vector(g_pos, t_pos, n_periods, p_treated, n_post_available) + else: + weight_vec = _simple_weight_vector(g_pos, t_pos, p_treated) + + out = pd.DataFrame( + { + "group": table["group"].to_numpy(), + "time": table["time"].to_numpy(), + "post": post_mask.astype(int), + "weight": weight_vec, + "att": table["att"].to_numpy(), + } + ) + + negative = weight_vec < 0 + abs_total = float(np.abs(weight_vec).sum()) + negative_post = negative & post_mask + abs_post_total = float(np.abs(weight_vec[post_mask]).sum()) + return ATTGTWeightsResult( + weights=out, + aggregation=aggregation, + implied_att=float((weight_vec * table["att"].to_numpy()).sum()), + n_negative=int(negative.sum()), + negative_weight_share=( + float(np.abs(weight_vec[negative]).sum() / abs_total) if abs_total > 0 else 0.0 + ), + n_negative_post=int(negative_post.sum()), + negative_post_weight_share=( + float(np.abs(weight_vec[negative_post]).sum() / abs_post_total) + if abs_post_total > 0 + else 0.0 + ), + n_cells=len(out), + source=source, + control_group=control_group, + base_period=base_period, + n_dropped_cells=dropped, + ) + + +def _label_for(grid: Dict[float, int], pos: int) -> Any: + """Positional period -> original label (inverse of ``_positional_grid``).""" + for label, p in grid.items(): + if p == pos: + return int(label) if float(label).is_integer() else label + return pos + + +def _pos_of(grid: Dict[float, int], label: Any) -> int: + """Original label -> positional period; never-treated sentinel stays 0.""" + try: + value = float(label) + except (TypeError, ValueError): + return -1 + if value == 0 or value == np.inf: + return 0 + return grid.get(value, -1) + + +def _resolve_frame_weights( + weights: Optional[Union[str, np.ndarray]], + data: pd.DataFrame, + unit: str, +) -> Optional[np.ndarray]: + """Turn ``weights=`` into one value per unit, or None.""" + if weights is None: + return None + if isinstance(weights, str): + if weights not in data.columns: + raise ValueError(f"weights column {weights!r} not found in data") + per_unit = data.groupby(unit, sort=True)[weights].nunique(dropna=False) + if (per_unit > 1).any(): + offenders = per_unit[per_unit > 1].index.tolist()[:5] + raise ValueError( + f"weights column {weights!r} varies within unit(s) " + f"{offenders!r}; sampling weights must be time-invariant" + ) + return data.groupby(unit, sort=True)[weights].first().to_numpy(dtype=float) + return np.asarray(weights, dtype=float) + + +# --------------------------------------------------------------------------- +# Panel plumbing for the decomposition +# --------------------------------------------------------------------------- + + +def _weighted_mean(values: np.ndarray, weights: np.ndarray) -> float: + """``stats::weighted.mean`` on flat arrays.""" + total = weights.sum() + if total == 0: + return float("nan") + return float((values * weights).sum() / total) + + +def _effective_sample_size(est_weights: np.ndarray, sampling_weights: np.ndarray) -> float: + """``sum(w)^2 / sum(w^2)`` after normalizing both weight vectors.""" + sw = sampling_weights / sampling_weights.mean() + ew = est_weights / _weighted_mean(est_weights, sw) + denom = float((ew**2).sum()) + if denom == 0: + return float("nan") + return float(ew.sum() ** 2 / denom) + + +class _Panel: + """Balanced panel reshaped to ``(n_units, n_periods)`` with positional time. + + Sorting by ``(unit, period)`` and reshaping means every ``(g, t)`` slice + is a plain boolean row mask plus a column index, instead of repeated + boolean scans over the long frame. + """ + + def __init__( + self, + data: pd.DataFrame, + *, + outcome: str, + unit: str, + time: str, + first_treat: str, + covariates: Sequence[str], + weights: Optional[str], + ) -> None: + for col in (outcome, unit, time, first_treat, *covariates): + if col not in data.columns: + raise ValueError(f"column {col!r} not found in data") + if weights is not None and weights not in data.columns: + raise ValueError(f"weights column {weights!r} not found in data") + + _validate_time_labels(data[time].to_numpy(), what=time) + frame = data.sort_values([unit, time]).reset_index(drop=True) + units = frame[unit].to_numpy() + periods = frame[time].to_numpy() + self.unit_ids = np.asarray(sorted(pd.unique(units))) + self.period_labels = np.asarray(sorted(pd.unique(periods))) + n_units = len(self.unit_ids) + n_periods = len(self.period_labels) + if len(frame) != n_units * n_periods: + raise ValueError( + f"decompose_twfe_weights requires a balanced panel: got " + f"{len(frame)} rows for {n_units} units x {n_periods} periods. " + "Balance it first, e.g. diff_diff.balance_panel(data, unit=..., " + "time=...)." + ) + counts = frame.groupby(unit, sort=True)[time].nunique().to_numpy() + if not np.all(counts == n_periods): + raise ValueError( + "decompose_twfe_weights requires a balanced panel: some units " + "are missing periods" + ) + + self.grid = _positional_grid(self.period_labels) + self.n_units = n_units + self.n_periods = n_periods + + cohort_long = frame[first_treat].to_numpy() + # dropna=False: a NaN label in one period must fail invariance, not + # be skipped by nunique(). + per_unit = frame.groupby(unit, sort=True)[first_treat].nunique(dropna=False) + if (per_unit > 1).any(): + offenders = per_unit[per_unit > 1].index.tolist()[:5] + raise ValueError( + f"{first_treat!r} varies within unit(s) {offenders!r}; cohort " + "membership must be time-invariant" + ) + raw_cohorts = cohort_long.reshape(n_units, n_periods)[:, 0] + _validate_cohort_labels(raw_cohorts, unit_ids=self.unit_ids, what=first_treat) + self.cohorts = _to_positional_cohort(raw_cohorts, self.grid) + if not (self.cohorts == 0).any(): + raise ValueError( + "decompose_twfe_weights needs never-treated units as the " + "comparison group; none were found (matching R's twfeweights, " + "which supports only a never-treated comparison)" + ) + self.outcome = frame[outcome].to_numpy(dtype=float).reshape(n_units, n_periods) + if weights is None: + self.weights = np.ones((n_units, n_periods)) + else: + block = frame[weights].to_numpy(dtype=float).reshape(n_units, n_periods) + # Finite check FIRST: np.allclose is False on any NaN, which would + # otherwise be misreported as "varies within unit". + if not np.all(np.isfinite(block)): + raise ValueError( + f"weights column {weights!r} must be finite; got NaN or infinite weight(s)" + ) + if not np.allclose(block, block[:, :1]): + raise ValueError( + f"weights column {weights!r} varies within unit; sampling " + "weights must be time-invariant" + ) + _validate_unit_weights(block[:, 0], self.cohorts == 0, require_control_mass=True) + self.weights = block + self.covariates = tuple(covariates) + if covariates: + self.design = ( + frame[list(covariates)] + .to_numpy(dtype=float) + .reshape(n_units, n_periods, len(covariates)) + ) + else: + self.design = np.zeros((n_units, n_periods, 0)) + + periods_positional = np.arange(1, n_periods + 1) + self.treated = ( + (periods_positional[None, :] >= self.cohorts[:, None]) & (self.cohorts[:, None] != 0) + ).astype(float) + + # Two-way demeaning through the house helper (the same alternating + # projections fixest::demean runs), on the sorted long frame so the + # (unit, period) reshape afterwards is a plain view. The treatment + # indicator is DERIVED from cohorts x positional periods, not an input + # column, so it is synthesized here before the call. Both the RAW and + # the demeaned covariate blocks are kept: the annihilation filter in + # _fwl_residuals compares one against the other. + demean_frame = pd.DataFrame( + {"_unit": frame[unit].to_numpy(), "_time": frame[time].to_numpy()} + ) + demean_frame["_treated"] = self.treated.reshape(-1) + for j, name in enumerate(self.covariates): + demean_frame[f"_x{j}"] = self.design[:, :, j].reshape(-1) + row_weights = None if weights is None else self.weights.reshape(-1) + demeaned = within_transform( + demean_frame, + ["_treated", *(f"_x{j}" for j in range(len(self.covariates)))], + "_unit", + "_time", + weights=row_weights, + suffix="_dm", + tol=1e-12, + ) + self.treated_demeaned = ( + demeaned["_treated_dm"].to_numpy(dtype=float).reshape(n_units, n_periods) + ) + if self.covariates: + self.design_demeaned = np.stack( + [ + demeaned[f"_x{j}_dm"].to_numpy(dtype=float).reshape(n_units, n_periods) + for j in range(len(self.covariates)) + ], + axis=2, + ) + else: + self.design_demeaned = np.zeros((n_units, n_periods, 0)) + + def covariate_block( + self, names: Sequence[str], data: pd.DataFrame, unit: str, time: str + ) -> np.ndarray: + """Unit-mean-collapsed covariates, one column per name. + + R's ``twfe_cov_bal`` averages each balance covariate over ALL periods + within a unit before comparing groups, so a time-varying covariate is + summarized by its unit mean. + """ + frame = data.sort_values([unit, time]).reset_index(drop=True) + block = ( + frame[list(names)] + .to_numpy(dtype=float) + .reshape(self.n_units, self.n_periods, len(names)) + ) + return block.mean(axis=1) + + +def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float]: + """Frisch-Waugh-Lovell residual of treatment on covariates, plus its scale. + + Double-demeans ``D`` and ``X``, projects the demeaned treatment on the + demeaned covariates, and returns the residual. That residual IS the + implicit weight the regression applies to each observation; ``alpha_den`` + is the normalization ``E[resid * Ddot]`` from R's + ``combine_twfe_weights_gt``. + + With no covariates the projection is empty and the residual is just the + double-demeaned treatment - which is exactly the branch R cannot run, + because ``fixest::demean`` segfaults on the zero-column model matrix it + builds for ``xformula = ~1``. + """ + weights = panel.weights + d_dot = panel.treated_demeaned + x_dot = panel.design_demeaned + + flat_d = d_dot.reshape(-1) + flat_w = weights.reshape(-1) + # Explicit row count: with zero covariates the trailing axis is 0 and + # numpy cannot infer a -1 against it. This is the same no-covariate branch + # on which fixest::demean segfaults; here it simply has to be spelled out. + flat_x = x_dot.reshape(panel.n_units * panel.n_periods, x_dot.shape[2]) + + # Numerical hygiene: drop covariates that double-demeaning ANNIHILATED + # before anything is projected on them. A time-invariant regressor leaves a + # column of pure rounding noise (~1e-16 against a raw scale of ~1). Keeping + # it is not catastrophic - the column lies in the FE span and is orthogonal + # to the treatment residual, so on mpdta's `lpop` it moves the FWL residual + # by ~2e-18 - but regressing on an exactly-zero column is meaningless, and + # dropping it is what makes covariates=None and covariates=[] + # agree exactly. The test is scale-relative: a column counts as having no + # within-variation when its demeaned norm is negligible NEXT TO ITS OWN raw + # norm, which a rank test on the demeaned matrix alone cannot see (there, + # 1e-16 is simply the largest pivot). The 1e-10 relative threshold is a + # blunt instrument: a covariate with a large level and genuinely small + # within-variation can trip it, which is why the warning says so. + raw_scale = np.linalg.norm( + panel.design.reshape(panel.n_units * panel.n_periods, x_dot.shape[2]), + axis=0, + ) + demeaned_scale = np.linalg.norm(flat_x, axis=0) + annihilated = demeaned_scale <= 1e-10 * np.maximum(raw_scale, 1.0) + if annihilated.any(): + names = [panel.covariates[j] for j in np.flatnonzero(annihilated)] + warnings.warn( + f"covariate(s) {names!r} have no within-unit-and-period variation " + "(or within-variation below 1e-10 of their own level) and were " + "dropped: two-way demeaning annihilates them, so they cannot affect " + "a two-way fixed effects regression. If that is not intended, " + "centre or rescale the covariate so its within-variation is not " + "negligible next to its level", + UserWarning, + stacklevel=3, + ) + flat_x = flat_x[:, ~annihilated] + surviving = [name for name, drop in zip(panel.covariates, annihilated) if not drop] + + if flat_x.shape[1]: + # House solver: WLS through the origin (R's lm(y ~ -1 + X, w)). On a + # rank-deficient design it fits the maximal independent set, sets the + # dropped coefficients to NaN (R-style) and computes the residual from + # the identified ones - so the residual is the FWL residual we need + # and the NaN positions name the collinear columns. + gamma, resid, _ = solve_ols( + flat_x, + flat_d, + weights=flat_w, + return_vcov=False, + rank_deficient_action="silent", + column_names=list(surviving), + ) + dropped = np.flatnonzero(np.isnan(gamma)) + if dropped.size: + names = [surviving[j] for j in dropped] + warnings.warn( + f"dropped collinear covariate column(s) {names!r} after " + "double-demeaning; they carry no within-variation independent of " + "the others", + UserWarning, + stacklevel=3, + ) + resid = np.asarray(resid, dtype=float) + else: + resid = flat_d + alpha_den = _weighted_mean(resid * flat_d, flat_w) + if not np.isfinite(alpha_den) or alpha_den == 0: + raise ValueError( + "the treatment indicator has no within-variation left after " + "double-demeaning and covariate adjustment, so the TWFE " + "coefficient is not identified" + ) + return resid.reshape(panel.n_units, panel.n_periods), alpha_den + + +def _normalize_cell_weights( + resid: np.ndarray, sampling_weights: np.ndarray, scale: float +) -> Tuple[np.ndarray, bool]: + """Scale a cell's residuals to mean one, handling the 0/0 case. + + The implicit weights within a cell are ``resid / mean(resid)``. For the + never-treated comparison group the residual is CONSTANT within a period + (their treatment indicator is identically zero, so the double-demeaned + value is ``-E_t[D] + mean_t E_t[D]``, the same for every control unit) - + and for some cohort structures that constant is analytically ZERO. On + sim_staggered (three equal cohorts at g in {0,3,4}, T=5) it vanishes + exactly at t=3: ``-1/3 + 1/3``. + + That makes the ratio 0/0. The limit is unambiguous - a constant divided + by its own mean is one - so return exactly one rather than dividing two + rounding errors. R divides anyway, which is why its per-cell ATT(g,t) at + such a cell carries ~1e-4 of noise; the aggregate is unaffected because + the weights on the affected cells cancel exactly. + + Returns the weights and whether the degenerate branch was taken. + """ + mean = _weighted_mean(resid, sampling_weights) + spread = float(np.max(resid) - np.min(resid)) if resid.size else 0.0 + tol = 1e-12 * max(scale, 1.0) + if abs(mean) <= tol: + if spread <= tol: + return np.ones_like(resid), True + raise ValueError( + "a group-time cell has comparison-group implicit weights that " + "average to zero but are not constant, so the cell's ATT(g,t) is " + "not identified. This usually means the panel has too little " + "variation in treatment timing." + ) + return resid / mean, False + + +def _decompose_fwl( + panel: _Panel, + base_period: str, + balance_covariates: Sequence[str], + balance_block: Optional[np.ndarray], +) -> Dict[str, Any]: + """R ``implicit_twfe_weights``: TWFE as weighted ATT(g, t) + a remainder.""" + resid, alpha_den = _fwl_residuals(panel) + weights = panel.weights + flat_w = weights.reshape(-1) + cohorts = panel.cohorts + treated_cohorts = sorted({int(g) for g in cohorts if g != 0}) + if not treated_cohorts: + raise ValueError("no ever-treated units found; nothing to decompose") + control_mask = cohorts == 0 + if not control_mask.any(): + raise ValueError( + "decompose_twfe_weights needs never-treated units as the " + "comparison group; none were found (matching R's twfeweights, " + "which supports only a never-treated comparison)" + ) + if base_period == "gmin1" and 1 in treated_cohorts: + raise ValueError( + "base_period='gmin1' needs a period before each cohort's " + "treatment, but a cohort is treated in the first period. Use " + "base_period='first_period', or drop that cohort." + ) + + resid_scale = float(np.abs(resid).max()) + cells: List[Dict[str, Any]] = [] + balance_rows: List[Dict[str, Any]] = [] + degenerate_cells: List[Tuple[Any, Any]] = [] + for g in treated_cohorts: + treated_mask = cohorts == g + for t_pos in range(1, panel.n_periods + 1): + col = t_pos - 1 + w_treated = weights[treated_mask, col] + w_control = weights[control_mask, col] + + r_treated = resid[treated_mask, col] + r_control = resid[control_mask, col] + gpart_w, _ = _normalize_cell_weights(r_treated, w_treated, resid_scale) + upart_w, degenerate = _normalize_cell_weights(r_control, w_control, resid_scale) + if degenerate: + degenerate_cells.append((panel.period_labels[g - 1], panel.period_labels[col])) + + y_t = panel.outcome[:, col] + if base_period == "first_period": + base = panel.outcome[:, 0] + else: + base = panel.outcome[:, g - 2] + adjusted = y_t - base + + gpart = _weighted_mean(gpart_w * adjusted[treated_mask], w_treated) + upart = _weighted_mean(upart_w * adjusted[control_mask], w_control) + + p_g = _weighted_mean( + (cohorts == g).astype(float)[:, None].repeat(panel.n_periods, axis=1).reshape(-1), + flat_w, + ) + alpha_weight = ( + _weighted_mean(r_treated, w_treated) * p_g / (alpha_den * panel.n_periods) + ) + + remainder = 0.0 + if base_period == "gmin1": + y_gmin1 = panel.outcome[:, g - 2] + remainder = -_weighted_mean(upart_w * y_gmin1[control_mask], w_control) + + cells.append( + { + "group": panel.period_labels[g - 1], + "time": panel.period_labels[col], + "post": int(t_pos >= g), + "att": gpart - upart, + "weight": alpha_weight, + "ess": _effective_sample_size(upart_w, w_control), + "remainder": remainder, + } + ) + if balance_block is not None: + balance_rows.extend( + _balance_cell( + balance_block, + balance_covariates, + treated_mask, + control_mask, + gpart_w, + upart_w, + w_treated, + w_control, + group=panel.period_labels[g - 1], + time=panel.period_labels[col], + post=int(t_pos >= g), + ) + ) + + if degenerate_cells: + warnings.warn( + f"{len(degenerate_cells)} group-time cell(s) {degenerate_cells[:4]!r}" + " have comparison-group implicit weights that are constant and " + "average to zero, so their ATT(g,t) is a 0/0 limit (taken as the " + "unweighted contrast). The weights on these cells cancel in the " + "aggregate, so `estimate` is unaffected; read the individual " + "ATT(g,t) there with caution", + UserWarning, + stacklevel=3, + ) + + frame = pd.DataFrame(cells) + weight_vec = frame["weight"].to_numpy() + att_col = frame["att"].to_numpy() + post_col = frame["post"].to_numpy().astype(bool) + decomposition = float((weight_vec * att_col).sum()) + remainder_total = float((frame["remainder"].to_numpy() * weight_vec).sum()) + ess_col = frame["ess"].to_numpy() + return { + "cells": frame, + "estimate": decomposition + remainder_total, + "decomposition": decomposition, + "remainder": remainder_total, + "pretrend_bias": float((weight_vec[~post_col] * att_col[~post_col]).sum()), + "post_only": float((weight_vec[post_col] * att_col[post_col]).sum()), + # summary.decomposed_twfe: post cells only, on both factors + "effective_sample_size": float( + post_col.sum() * (weight_vec[post_col] * ess_col[post_col]).sum() + ), + "balance": pd.DataFrame(balance_rows) if balance_block is not None else None, + } + + +# --------------------------------------------------------------------------- +# Balance statistics (Imbens & Rubin 2015, as implemented upstream) +# --------------------------------------------------------------------------- + + +def _weighted_ecdf(values: np.ndarray, weights: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """``BMisc::weighted_ecdf``: knots and CDF heights. + + ``weights`` are normalized by their mean, the knots are the sorted unique + values, and ``F(knot_j) = mean(w * (y <= knot_j))``. + """ + w = weights / weights.mean() + # Sort once and read cumulative mass at each unique-value boundary, rather + # than rescanning the full vector per knot (the naive form is O(n * k), and + # this runs per covariate x cohort x period). `np.unique` returns the + # sorted knots, so a single searchsorted locates each boundary. + order = np.argsort(values, kind="stable") + sorted_values = values[order] + cumulative = np.cumsum(w[order]) + knots = np.unique(values) + last = np.searchsorted(sorted_values, knots, side="right") - 1 + heights = cumulative[last] / len(values) + return knots, heights + + +def _ecdf_eval(knots: np.ndarray, heights: np.ndarray, at: float) -> float: + """Evaluate the step function from ``BMisc::make_dist``. + + ``approxfun(method="constant", yleft=0, yright=1, f=0)``: the value on + ``[knot_i, knot_{i+1})`` is ``heights[i]``, zero below the first knot and + one above the last. + """ + if at < knots[0]: + return 0.0 + if at > knots[-1]: + return 1.0 + idx = int(np.searchsorted(knots, at, side="right") - 1) + return float(heights[idx]) + + +def _ecdf_quantile(knots: np.ndarray, heights: np.ndarray, prob: float) -> float: + """``stats:::quantile.ecdf``: type-7 quantile of a reconstructed sample. + + R does NOT invert the step function directly. It rebuilds a pseudo-sample + by repeating each knot ``diff(c(0, round(nobs * F)))`` times - where + ``nobs`` is the number of KNOTS, not the number of observations - and then + takes an ordinary type-7 quantile of that. Reproduced exactly, because the + rounding makes the result differ from a direct inversion. + """ + nobs = len(knots) + counts = np.diff(np.concatenate([[0.0], np.round(nobs * heights)])) + counts = np.maximum(counts, 0).astype(int) + sample = np.repeat(knots, counts) + if sample.size == 0: + return float("nan") + # R's default type-7 quantile. + sample = np.sort(sample) + h = (len(sample) - 1) * prob + lo = int(np.floor(h)) + hi = min(lo + 1, len(sample) - 1) + return float(sample[lo] + (h - lo) * (sample[hi] - sample[lo])) + + +def _pooled_sd(x: np.ndarray, treated: np.ndarray, sampling_weights: np.ndarray) -> float: + """Pooled standard deviation across the treated and comparison groups.""" + sw = sampling_weights / sampling_weights.mean() + + def wvar(values: np.ndarray, w: np.ndarray) -> float: + return _weighted_mean((values - _weighted_mean(values, w)) ** 2, w) + + var1 = wvar(x[treated == 1], sw[treated == 1]) + var0 = wvar(x[treated == 0], sw[treated == 0]) + n1 = sw[treated == 1].sum() + n0 = sw[treated == 0].sum() + if n1 + n0 - 2 <= 0: + return float("nan") + return float(np.sqrt(((n1 - 1) * var1 + (n0 - 1) * var0) / (n1 + n0 - 2))) + + +def _normalize_est_weights( + est_weights: np.ndarray, treated: np.ndarray, sw: np.ndarray +) -> np.ndarray: + """Scale estimation weights to mean one WITHIN each group, as R does.""" + out = np.array(est_weights, dtype=float, copy=True) + for group in (0, 1): + mask = treated == group + if mask.any(): + out[mask] = out[mask] / _weighted_mean(out[mask], sw[mask]) + return out + + +def _log_ratio_sd( + x: np.ndarray, + treated: np.ndarray, + est_weights: np.ndarray, + sampling_weights: np.ndarray, +) -> float: + """Log ratio of treated to comparison spread. + + Note: upstream scales each group's SD by ``sqrt(n - 1)`` before taking + the ratio, which is not a conventional standard deviation. Preserved + verbatim for parity - the quantity is only ever read as a relative + balance statistic, and the extra factor largely cancels in the ratio. + """ + sw = sampling_weights / sampling_weights.mean() + ew = _normalize_est_weights(est_weights, treated, sw) + + def wvar(values: np.ndarray, e: np.ndarray, w: np.ndarray) -> float: + scaled = values * e + return _weighted_mean((scaled - _weighted_mean(scaled, w)) ** 2, w) + + var1 = wvar(x[treated == 1], ew[treated == 1], sw[treated == 1]) + var0 = wvar(x[treated == 0], ew[treated == 0], sw[treated == 0]) + n1 = sw[treated == 1].sum() + n0 = sw[treated == 0].sum() + sd1 = np.sqrt(max(n1 - 1, 0)) * np.sqrt(var1) + sd0 = np.sqrt(max(n0 - 1, 0)) * np.sqrt(var0) + if sd1 <= 0 or sd0 <= 0: + return float("nan") + return float(np.log(sd1) - np.log(sd0)) + + +def _frac_treated_extreme( + x: np.ndarray, + treated: np.ndarray, + est_weights: np.ndarray, + sampling_weights: np.ndarray, + alpha: float = 0.05, +) -> float: + """Share of treated mass outside the comparison group's central range. + + A step function of a weighted empirical CDF, so a perturbation of order + 1e-12 can move one unit across a knot and shift the value by 1/n. Tests + gate it with an absolute tolerance of ``1 / n_control`` rather than a + relative one. + """ + if len(np.unique(x)) < 3: + return float("nan") + sw = sampling_weights / sampling_weights.mean() + ew = _normalize_est_weights(est_weights, treated, sw) + + control = treated == 0 + treat = treated == 1 + knots_u, heights_u = _weighted_ecdf(ew[control] * x[control], sw[control]) + upper = _ecdf_quantile(knots_u, heights_u, 1 - alpha / 2) + lower = _ecdf_quantile(knots_u, heights_u, alpha / 2) + knots_t, heights_t = _weighted_ecdf(ew[treat] * x[treat], sw[treat]) + return float( + 1.0 - _ecdf_eval(knots_t, heights_t, upper) + _ecdf_eval(knots_t, heights_t, lower) + ) + + +def _balance_cell( + block: np.ndarray, + names: Sequence[str], + treated_mask: np.ndarray, + control_mask: np.ndarray, + weights_treated: np.ndarray, + weights_control: np.ndarray, + sw_treated: np.ndarray, + sw_control: np.ndarray, + *, + group: Any, + time: Any, + post: int, +) -> List[Dict[str, Any]]: + """Per-covariate implicit-weight balance for one ``(g, t)`` cell.""" + both = treated_mask | control_mask + indicator = np.where(treated_mask[both], 1, 0) + est = np.empty(int(both.sum())) + est[indicator == 1] = weights_treated + est[indicator == 0] = weights_control + sw_both = np.empty_like(est) + sw_both[indicator == 1] = sw_treated + sw_both[indicator == 0] = sw_control + ones = np.ones_like(est) + + rows: List[Dict[str, Any]] = [] + for j, name in enumerate(names): + col = block[:, j] + x_t = col[treated_mask] + x_c = col[control_mask] + x_both = col[both] + unweighted_treated = _weighted_mean(x_t, sw_treated) + unweighted_control = _weighted_mean(x_c, sw_control) + weighted_treated = _weighted_mean(x_t * weights_treated, sw_treated) + weighted_control = _weighted_mean(x_c * weights_control, sw_control) + rows.append( + { + "group": group, + "time": time, + "post": post, + "covariate": name, + "unweighted_treated": unweighted_treated, + "unweighted_control": unweighted_control, + "unweighted_diff": unweighted_treated - unweighted_control, + "weighted_treated": weighted_treated, + "weighted_control": weighted_control, + "weighted_diff": weighted_treated - weighted_control, + "sd": _pooled_sd(x_both, indicator, sw_both), + "unweighted_log_ratio_sd": _log_ratio_sd(x_both, indicator, ones, sw_both), + "weighted_log_ratio_sd": _log_ratio_sd(x_both, indicator, est, sw_both), + "unweighted_frac_extreme": _frac_treated_extreme(x_both, indicator, ones, sw_both), + "weighted_frac_extreme": _frac_treated_extreme(x_both, indicator, est, sw_both), + } + ) + return rows + + +_METHODS = ("fwl",) +_BASE_PERIODS = ("first_period", "gmin1") + + +def decompose_twfe_weights( + data: pd.DataFrame, + *, + outcome: str, + unit: str, + time: str, + first_treat: str, + method: str = "fwl", + covariates: Optional[Sequence[str]] = None, + base_period: str = "first_period", + balance_covariates: Optional[Sequence[str]] = None, + weights: Optional[str] = None, +) -> TWFEDecompositionResult: + """Decompose a TWFE estimate into weighted group-time effects. + + Runs the regression, recovers the implicit weight it places on each + ATT(g, t), and separates the part of the estimate that comes from + PRE-treatment cells - i.e. from parallel-trends violations rather than + from treatment. + + Takes the raw panel rather than a fitted result, because it re-estimates: + it double-demeans treatment and covariates and forms its own group-time + contrasts, so there is no ATT(g, t) table it could consume. Its companion + :func:`attgt_weights` is the fitted-result surface, and the two are tied + by an identity that holds when the fit used ``base_period="universal"``, + ``control_group="never_treated"`` and no covariates:: + + sum(attgt_weights(cs, aggregation="twfe").weights.eval("weight * att")) + == decompose_twfe_weights(panel, ...).estimate + + Parameters + ---------- + data : pd.DataFrame + Balanced panel in long form. + outcome, unit, time, first_treat : str + Column names, matching :meth:`CallawaySantAnna.fit`. Never-treated + units carry ``first_treat`` of ``0`` (or ``inf``). + method : {"fwl"}, default "fwl" + ``"fwl"`` recovers the Frisch-Waugh-Lovell implicit weights from the + TWFE regression. + covariates : sequence of str, optional + Covariates the regression adjusts for. ``None`` runs the + no-covariate decomposition. + base_period : {"first_period", "gmin1"}, default "first_period" + Which pre-period each cell is measured against. ``"gmin1"`` (the + period before treatment) generates a non-zero ``remainder``. + balance_covariates : sequence of str, optional + Covariates to report implicit-weight balance for, readable afterwards + via :meth:`TWFEDecompositionResult.covariate_balance`. Each is + averaged over periods within unit before groups are compared, as + upstream does. + weights : str, optional + Time-invariant sampling-weight column. + + Returns + ------- + TWFEDecompositionResult + + Raises + ------ + ValueError + On an unknown ``method`` or ``base_period``; on an unbalanced panel, + a missing never-treated group, or time-varying cohort labels; or when + the treatment has no within-variation left after demeaning. + + Examples + -------- + >>> import diff_diff # doctest: +SKIP + >>> dec = diff_diff.decompose_twfe_weights( # doctest: +SKIP + ... panel, outcome="y", unit="id", time="t", first_treat="g", + ... covariates=["x"], balance_covariates=["x"], + ... ) + >>> dec.pretrend_bias # doctest: +SKIP + >>> dec.covariate_balance() # doctest: +SKIP + """ + if method not in _METHODS: + raise ValueError(f"method must be one of {list(_METHODS)!r}, got {method!r}") + if base_period not in _BASE_PERIODS: + raise ValueError( + f"base_period must be one of {list(_BASE_PERIODS)!r}, got " f"{base_period!r}" + ) + + covariate_names = tuple(covariates or ()) + balance_names = tuple(balance_covariates or ()) + panel = _Panel( + data, + outcome=outcome, + unit=unit, + time=time, + first_treat=first_treat, + covariates=covariate_names, + weights=weights, + ) + balance_block = ( + panel.covariate_block(balance_names, data, unit, time) if balance_names else None + ) + + payload = _decompose_fwl(panel, base_period, balance_names, balance_block) + return TWFEDecompositionResult( + cells=payload["cells"], + method=method, + estimate=payload["estimate"], + decomposition=payload["decomposition"], + remainder=payload["remainder"], + pretrend_bias=payload["pretrend_bias"], + post_only=payload["post_only"], + base_period=base_period, + covariates=covariate_names, + effective_sample_size=payload["effective_sample_size"], + n_units=panel.n_units, + n_periods=panel.n_periods, + balance=payload["balance"], + ) diff --git a/diff_diff/twfe_weights_results.py b/diff_diff/twfe_weights_results.py new file mode 100644 index 000000000..3f7299873 --- /dev/null +++ b/diff_diff/twfe_weights_results.py @@ -0,0 +1,459 @@ +"""Result containers for the TWFE implicit-weight diagnostics. + +See :mod:`diff_diff.twfe_weights` for the entry points that build these, and +for the upstream MIT attribution. + +Both containers subclass :class:`diff_diff.Diagnostic`: they assess a design +(what a regression implicitly weights) rather than estimating a causal effect, +so neither carries the estimator quintet ``att``/``se``/``t_stat``/``p_value``/ +``conf_int``. The headline scalars are named ``implied_att`` and ``estimate`` +precisely so they do not read as inference-bearing point estimates - the +decomposition is an algebraic identity, exactly like +:class:`~diff_diff.BaconDecompositionResults`. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional, Tuple + +import numpy as np +import pandas as pd + +from diff_diff.results_base import Diagnostic + +__all__ = ["ATTGTWeightsResult", "TWFEDecompositionResult"] + +_AGGREGATION_LABELS = { + "twfe": "TWFE regression", + "overall": "ATT^O (Callaway & Sant'Anna overall)", + "simple": "ATT^simple (Callaway & Sant'Anna simple)", +} + +# Per-cell balance columns, in report order. The three ``_`` -prefixed groups +# mirror R's ``cov_bal_df`` under diff-diff naming; see the mapping table in +# the REGISTRY entry. +_BALANCE_STATS = ( + "unweighted_treated", + "unweighted_control", + "unweighted_diff", + "weighted_treated", + "weighted_control", + "weighted_diff", + "sd", + "unweighted_log_ratio_sd", + "weighted_log_ratio_sd", + "unweighted_frac_extreme", + "weighted_frac_extreme", +) + + +def _fmt(value: float, width: int = 12, digits: int = 4) -> str: + """Right-aligned float that renders NaN without blowing up the layout.""" + if value is None or (isinstance(value, float) and not np.isfinite(value)): + return f"{'n/a':>{width}}" + return f"{value:>{width}.{digits}f}" + + +@dataclass +class ATTGTWeightsResult(Diagnostic): + """Weights that an estimand places on each group-time effect ATT(g, t). + + Returned by :func:`diff_diff.attgt_weights`. One row per ``(g, t)`` cell. + + Attributes + ---------- + weights : pd.DataFrame + Columns ``group``, ``time``, ``post``, ``weight``, ``att``. ``post`` + is ``1`` when ``time >= group`` (the cells the estimand targets), + ``0`` for pre-treatment cells. ``att`` is the ATT(g, t) the weight + multiplies, carried through from the source so that + ``(weight * att).sum()`` reproduces ``implied_att``. + aggregation : str + Which estimand's weights these are: ``"twfe"``, ``"overall"`` + (ATT^O), or ``"simple"`` (ATT^simple). + implied_att : float + ``sum(weight * att)`` - what the estimand delivers given these + ATT(g, t). For ``aggregation="twfe"`` this is the TWFE coefficient. + n_negative : int + Number of cells - PRE and post - receiving a negative weight. Under + ``aggregation="twfe"`` the weights over the full ``g != 0`` grid sum + to zero (post to +1, pre to -1), so this is non-zero in every + staggered design; read ``n_negative_post`` for the pathology. + negative_weight_share : float + ``sum(|w| : w < 0) / sum(|w|)`` over ALL cells - how much of the total + weight mass points the wrong way. Near 0.5 is normal under + ``"twfe"`` for the same reason. ``0.0`` when no weight is negative. + n_negative_post : int + Number of POST-treatment cells receiving a negative weight. This is + the classic staggered-adoption pathology: the regression subtracts + treatment effects it should be adding. Zero for ``"overall"`` and + ``"simple"`` by construction. + negative_post_weight_share : float + ``sum(|w| : w < 0, post) / sum(|w| : post)`` - the share of + post-treatment weight mass that is negative. No R counterpart; see + the methodology registry. + n_cells : int + Number of ``(g, t)`` cells contributing. + source : str or None + ``"CallawaySantAnnaResults"`` when built from a fitted result, + ``"DataFrame"`` on the fallback path. + control_group, base_period : str or None + Design metadata carried from the source fit, when available. + n_dropped_cells : int + Cells excluded because their ATT(g, t) was non-estimable (NaN). + """ + + weights: pd.DataFrame + aggregation: str + implied_att: float + n_negative: int + negative_weight_share: float + n_negative_post: int + negative_post_weight_share: float + n_cells: int + source: Optional[str] = None + control_group: Optional[str] = None + base_period: Optional[str] = None + n_dropped_cells: int = 0 + + def __repr__(self) -> str: + return ( + f"ATTGTWeightsResult(aggregation={self.aggregation!r}, " + f"implied_att={self.implied_att:.4f}, " + f"n_cells={self.n_cells}, n_negative={self.n_negative})" + ) + + def summary(self) -> str: + """Formatted per-cell weight table with the negative-weight roll-up.""" + width = 72 + label = _AGGREGATION_LABELS.get(self.aggregation, self.aggregation) + lines = [ + "=" * width, + "Implicit Weights on ATT(g, t)".center(width), + "=" * width, + "", + f"{'Estimand:':<28} {label}", + f"{'Group-time cells:':<28} {self.n_cells:>10}", + ] + if self.n_dropped_cells: + lines.append(f"{'Non-estimable cells dropped:':<28} {self.n_dropped_cells:>10}") + if self.source is not None: + lines.append(f"{'Source:':<28} {self.source}") + if self.control_group is not None: + lines.append(f"{'Control group:':<28} {self.control_group}") + if self.base_period is not None: + lines.append(f"{'Base period:':<28} {self.base_period}") + lines += [ + "", + "-" * width, + f"{'Group':>8} {'Time':>8} {'Post':>6} {'Weight':>14} {'ATT(g,t)':>14}", + "-" * width, + ] + for row in self.weights.itertuples(index=False): + lines.append( + f"{row.group:>8} {row.time:>8} {int(row.post):>6} " + f"{_fmt(row.weight, 14, 6)} {_fmt(row.att, 14, 6)}" + ) + lines += [ + "-" * width, + "", + f"{'Implied estimate:':<28} {_fmt(self.implied_att)}", + f"{'Negative POST-period cells:':<28} {self.n_negative_post:>12}", + f"{'Negative POST-weight share:':<28} {_fmt(self.negative_post_weight_share)}", + f"{'Negative cells (all):':<28} {self.n_negative:>12}", + f"{'Negative share (all):':<28} {_fmt(self.negative_weight_share)}", + "", + ] + if self.n_negative_post: + lines += [ + "Note: negative POST-period weights mean this estimand subtracts some", + " treatment-period ATT(g, t). Under heterogeneous effects the", + " estimate need not lie in the convex hull of those effects.", + "", + ] + elif self.n_negative: + lines += [ + "Note: the negative weights fall on PRE-treatment cells only, which is", + " how the TWFE weights sum to zero over the full grid; no", + " treatment-period effect is being subtracted.", + "", + ] + lines.append("=" * width) + return "\n".join(lines) + + def print_summary(self) -> None: + """Print :meth:`summary` to stdout.""" + print(self.summary()) + + def to_dataframe(self) -> pd.DataFrame: + """Per-cell weight table (a copy).""" + return self.weights.copy() + + def to_dict(self) -> Dict[str, Any]: + """Serializable view of the result.""" + return { + "aggregation": self.aggregation, + "implied_att": self.implied_att, + "n_cells": self.n_cells, + "n_negative": self.n_negative, + "negative_weight_share": self.negative_weight_share, + "n_negative_post": self.n_negative_post, + "negative_post_weight_share": self.negative_post_weight_share, + "n_dropped_cells": self.n_dropped_cells, + "source": self.source, + "control_group": self.control_group, + "base_period": self.base_period, + "weights": self.weights.to_dict(orient="list"), + } + + +@dataclass +class TWFEDecompositionResult(Diagnostic): + """Decomposition of a TWFE estimate into weighted ATT(g, t). + + Returned by :func:`diff_diff.decompose_twfe_weights`. + + Attributes + ---------- + cells : pd.DataFrame + Columns ``group``, ``time``, ``post``, ``att``, ``weight``, ``ess`` + and ``remainder``. ``weight`` is the implicit weight the regression + places on that cell's ATT(g, t) - R's ``alpha_weight``. + method : str + ``"fwl"`` - Frisch-Waugh-Lovell residual weights from the TWFE + regression. (The only method currently implemented; upstream's AIPW + decomposition is a documented follow-up.) + estimate : float + The estimate being decomposed - ``decomposition + remainder``. + decomposition : float + ``sum(weight * att)`` over all cells, pre and post. + remainder : float + Part of ``estimate`` not attributable to any ATT(g, t) cell. + Identically ``0.0`` except under ``base_period="gmin1"``. + pretrend_bias : float + ``sum(weight * att)`` over PRE-treatment cells only. Under parallel + trends every pre-treatment ATT(g, t) is zero and this vanishes; a + non-zero value is the contribution of parallel-trends violations to + ``estimate``. + post_only : float + ``sum(weight * att)`` over post-treatment cells only. + base_period : str or None + ``"first_period"`` or ``"gmin1"``. + covariates : tuple of str + Covariates the regression adjusted for. Empty tuple when none. + effective_sample_size : float + Weight-concentration roll-up. Small values relative to ``n_units`` + mean the estimate leans on few observations. + n_units, n_periods : int + Panel dimensions. + balance : pd.DataFrame or None + Per-cell implicit covariate balance, populated when + ``balance_covariates=`` was requested. Read it via + :meth:`covariate_balance`. + """ + + cells: pd.DataFrame + method: str + estimate: float + decomposition: float + remainder: float + pretrend_bias: float + post_only: float + base_period: Optional[str] + covariates: Tuple[str, ...] + effective_sample_size: float + n_units: int + n_periods: int + balance: Optional[pd.DataFrame] = field(default=None) + + def __repr__(self) -> str: + return ( + f"TWFEDecompositionResult(method={self.method!r}, " + f"estimate={self.estimate:.4f}, " + f"pretrend_bias={self.pretrend_bias:.4f}, " + f"n_cells={len(self.cells)})" + ) + + def summary(self) -> str: + """Formatted decomposition table with the pre-trend contribution.""" + width = 78 + method_label = { + "fwl": "TWFE regression (Frisch-Waugh-Lovell implicit weights)", + }.get(self.method, self.method) + covs = ", ".join(self.covariates) if self.covariates else "(none)" + lines = [ + "=" * width, + "Decomposition into Group-Time Effects".center(width), + "=" * width, + "", + f"{'Method:':<30} {method_label}", + f"{'Covariates:':<30} {covs}", + ] + if self.base_period is not None: + lines.append(f"{'Base period:':<30} {self.base_period}") + lines += [ + f"{'Units / periods:':<30} {self.n_units} / {self.n_periods}", + f"{'Group-time cells:':<30} {len(self.cells)}", + "", + "-" * width, + f"{'Group':>8} {'Time':>8} {'Post':>6} {'Weight':>14} " + f"{'ATT(g,t)':>14} {'Contribution':>14}", + "-" * width, + ] + for row in self.cells.itertuples(index=False): + lines.append( + f"{row.group:>8} {row.time:>8} {int(row.post):>6} " + f"{_fmt(row.weight, 14, 6)} {_fmt(row.att, 14, 6)} " + f"{_fmt(row.weight * row.att, 14, 6)}" + ) + lines += [ + "-" * width, + "", + f"{'Estimate:':<30} {_fmt(self.estimate)}", + f"{' from ATT(g,t) cells:':<30} {_fmt(self.decomposition)}", + f"{' post-treatment only:':<30} {_fmt(self.post_only)}", + f"{' pre-trend violations:':<30} {_fmt(self.pretrend_bias)}", + f"{' remainder:':<30} {_fmt(self.remainder)}", + "", + f"{'Effective sample size:':<30} {_fmt(self.effective_sample_size)}", + "", + ] + if abs(self.pretrend_bias) > 1e-10: + lines += [ + "Note: a non-zero pre-trend contribution means pre-treatment", + " ATT(g, t) are not zero, so part of the estimate reflects", + " parallel-trends violations rather than treatment effects.", + "", + ] + if self.balance is not None: + lines += [ + "Covariate balance available via .covariate_balance().", + "", + ] + lines.append("=" * width) + return "\n".join(lines) + + def print_summary(self) -> None: + """Print :meth:`summary` to stdout.""" + print(self.summary()) + + def to_dataframe(self) -> pd.DataFrame: + """Per-cell decomposition table (a copy).""" + return self.cells.copy() + + def to_dict(self) -> Dict[str, Any]: + """Serializable view of the result.""" + out: Dict[str, Any] = { + "method": self.method, + "estimate": self.estimate, + "decomposition": self.decomposition, + "remainder": self.remainder, + "pretrend_bias": self.pretrend_bias, + "post_only": self.post_only, + "base_period": self.base_period, + "covariates": list(self.covariates), + "effective_sample_size": self.effective_sample_size, + "n_units": self.n_units, + "n_periods": self.n_periods, + "cells": self.cells.to_dict(orient="list"), + } + if self.balance is not None: + out["balance"] = self.balance.to_dict(orient="list") + return out + + def covariate_balance( + self, + *, + level: str = "summary", + standardize: bool = True, + post_only: bool = True, + ) -> pd.DataFrame: + """Implicit-weight covariate balance. + + Asks whether the weights the regression implicitly applies actually + balance the covariates across the treated and comparison groups. If + ``weighted_diff`` is no closer to zero than ``unweighted_diff``, the + covariate adjustment is not buying what it appears to. + + Parameters + ---------- + level : {"summary", "cell"}, default "summary" + ``"summary"`` aggregates across ``(g, t)`` cells to one row per + covariate, weighting each cell by its implicit weight. + ``"cell"`` returns the unaggregated per-``(g, t)`` rows. + standardize : bool, default True + Append ``unweighted_std_diff`` / ``weighted_std_diff``, the + differences divided by the pooled standard deviation. These are + a diff-diff addition; R reports the raw differences only. + post_only : bool, default True + Restrict the summary roll-up to post-treatment cells, matching + R's ``mp_covariate_bal_summary_helper``. Ignored when + ``level="cell"``. + + Returns + ------- + pd.DataFrame + One row per covariate (``level="summary"``) or per + ``(group, time, covariate)`` (``level="cell"``). + + Raises + ------ + ValueError + If balance was not requested at compute time, or ``level`` is + not one of the two accepted values. + """ + if self.balance is None: + raise ValueError( + "Covariate balance was not computed for this decomposition. " + "Re-run with balance_covariates=, e.g.\n" + " decompose_twfe_weights(..., balance_covariates=['x1', 'x2'])" + ) + if level not in ("summary", "cell"): + raise ValueError(f"level must be 'summary' or 'cell', got {level!r}") + + table = self.balance.copy() + if level == "cell": + if standardize: + table = _append_standardized(table) + return table + + weights = self.cells.set_index(["group", "time"])["weight"] + keys = pd.MultiIndex.from_arrays([table["group"], table["time"]]) + table["_w"] = weights.reindex(keys).to_numpy() + if post_only: + # Mask on the `post` COLUMN, not on a zero roll-up weight: R's + # post-only helper never touches the pre cells, but a post cell + # whose implicit weight happens to be exactly zero still + # contributes - and still propagates its NA. + table = table[table["post"].to_numpy().astype(bool)] + + # R propagates NA: if ANY contributing cell of a (covariate, statistic) + # is NA, the summary is NA. pandas' sum() skips NaN, which would turn + # the documented NA return of frac_treated_extreme (fewer than three + # distinct values) into a spurious 0.0. + stats = table[list(_BALANCE_STATS)].mul(table["_w"], axis=0) + groups = table["covariate"].to_numpy() + rolled = stats.groupby(groups, sort=False).sum(min_count=1) + any_nan = table[list(_BALANCE_STATS)].isna().groupby(groups, sort=False).any() + rolled = rolled.mask(any_nan) + rolled.index.name = "covariate" + out = rolled.reset_index() + if standardize: + out = _append_standardized(out) + return out + + +def _append_standardized(table: pd.DataFrame) -> pd.DataFrame: + """Add ``*_std_diff`` columns (difference / pooled SD). + + A diff-diff addition on top of R's columns - additive, so parity is + asserted on the R columns only. ``sd == 0`` yields NaN rather than an + infinity, so a degenerate covariate does not poison a summary table. + """ + out = table.copy() + sd = out["sd"].to_numpy(dtype=float) + safe = np.where(sd == 0, np.nan, sd) + out["unweighted_std_diff"] = out["unweighted_diff"].to_numpy(dtype=float) / safe + out["weighted_std_diff"] = out["weighted_diff"].to_numpy(dtype=float) / safe + return out diff --git a/diff_diff/visualization/__init__.py b/diff_diff/visualization/__init__.py index 4c06bb751..8d255f67c 100644 --- a/diff_diff/visualization/__init__.py +++ b/diff_diff/visualization/__init__.py @@ -15,6 +15,7 @@ from diff_diff.visualization._diagnostic import ( plot_bacon, plot_sensitivity, + plot_twfe_weights, ) from diff_diff.visualization._event_study import ( PlottableResults, @@ -48,6 +49,7 @@ "plot_group_effects", "plot_sensitivity", "plot_bacon", + "plot_twfe_weights", "plot_power_curve", "plot_pretrends_power", # New public functions diff --git a/diff_diff/visualization/_diagnostic.py b/diff_diff/visualization/_diagnostic.py index ef2dd05b2..ddb99e224 100644 --- a/diff_diff/visualization/_diagnostic.py +++ b/diff_diff/visualization/_diagnostic.py @@ -1,12 +1,20 @@ -"""Diagnostic visualization functions (sensitivity, Bacon decomposition).""" +"""Diagnostic visualization functions (sensitivity, Bacon decomposition). -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +``plot_twfe_weights`` renders the implicit ATT(g, t) weights (or their covariate +balance) from :func:`diff_diff.attgt_weights` / :func:`diff_diff.decompose_twfe_weights`. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import numpy as np if TYPE_CHECKING: from diff_diff.bacon import BaconDecompositionResults from diff_diff.honest_did import SensitivityResults + from diff_diff.twfe_weights_results import ( + ATTGTWeightsResult, + TWFEDecompositionResult, + ) def plot_sensitivity( @@ -817,3 +825,333 @@ def _render_bacon_plotly( fig.show() return fig + + +def plot_twfe_weights( + results: "Union[ATTGTWeightsResult, TWFEDecompositionResult]", + *, + kind: str = "auto", + standardize: bool = True, + absolute_value: bool = True, + figsize: Tuple[float, float] = (10, 6), + title: Optional[str] = None, + xlabel: Optional[str] = None, + ylabel: Optional[str] = None, + post_color: str = "#2563eb", + pre_color: str = "#dc2626", + markersize: int = 80, + alpha: float = 0.8, + annotate: bool = False, + ax: Optional[Any] = None, + show: bool = True, + backend: str = "matplotlib", +) -> Any: + """Visualize implicit TWFE weights on ATT(g, t), or their covariate balance. + + Two views, matching upstream's ``ggtwfeweights`` methods: + + - ``kind="weights"`` plots weight against ATT(g, t), one point per + group-time cell, coloured by pre/post. Points to the LEFT of the + vertical zero line carry negative weight - the staggered-TWFE + pathology. + - ``kind="balance"`` plots unweighted against implicitly-weighted + covariate differences. Points near zero on the vertical axis are + covariates the implicit weights balance. + + Parameters + ---------- + results : ATTGTWeightsResult or TWFEDecompositionResult + Output of :func:`diff_diff.attgt_weights` or + :func:`diff_diff.decompose_twfe_weights`. + kind : {"auto", "weights", "balance"}, default "auto" + ``"auto"`` picks ``"balance"`` when the result carries a balance + table and ``"weights"`` otherwise. + standardize : bool, default True + Balance view: divide differences by the pooled standard deviation. + absolute_value : bool, default True + Balance view: plot absolute differences, so "closer to zero is + better" reads the same for every covariate. + figsize : tuple, default (10, 6) + Figure size in inches. Ignored when ``ax`` is supplied. + title, xlabel, ylabel : str, optional + Overrides for the defaults chosen per ``kind``. + post_color, pre_color : str + Colors for post- and pre-treatment cells (weights view). + markersize : int, default 80 + Scatter marker area. + alpha : float, default 0.8 + Marker opacity. + annotate : bool, default False + Label each point with its ``(group, time)`` or covariate name. + ax : matplotlib Axes, optional + Axes to draw on. A new figure is created when omitted. Matplotlib only. + show : bool, default True + Call ``plt.show()`` / ``fig.show()`` before returning. + backend : str, default "matplotlib" + Plotting backend: ``"matplotlib"`` or ``"plotly"``. + + Returns + ------- + matplotlib.axes.Axes or plotly.graph_objects.Figure + + Raises + ------ + ValueError + On an unknown ``kind`` or ``backend``; when ``kind="balance"`` is + requested for a result that carries no balance table; or when the + balance table has no finite differences to plot. + + Examples + -------- + >>> import diff_diff # doctest: +SKIP + >>> w = diff_diff.attgt_weights(cs_result) # doctest: +SKIP + >>> diff_diff.plot_twfe_weights(w) # doctest: +SKIP + >>> diff_diff.plot_twfe_weights(w, backend="plotly") # doctest: +SKIP + """ + if kind not in ("auto", "weights", "balance"): + raise ValueError(f"kind must be one of ['auto', 'weights', 'balance'], got {kind!r}") + if backend not in ("matplotlib", "plotly"): + raise ValueError(f"backend must be 'matplotlib' or 'plotly', got {backend!r}") + has_balance = getattr(results, "balance", None) is not None + if kind == "auto": + kind = "balance" if has_balance else "weights" + if kind == "balance" and not has_balance: + raise ValueError( + "this result carries no covariate balance table, so kind='balance' " + "has nothing to plot. Recompute with " + "decompose_twfe_weights(..., balance_covariates=[...])." + ) + + payload = _twfe_weights_payload(results, kind, standardize, absolute_value) + render = _render_twfe_weights_plotly if backend == "plotly" else _render_twfe_weights_mpl + return render( + payload, + kind=kind, + standardize=standardize, + figsize=figsize, + title=title, + xlabel=xlabel, + ylabel=ylabel, + post_color=post_color, + pre_color=pre_color, + markersize=markersize, + alpha=alpha, + annotate=annotate, + ax=ax, + show=show, + ) + + +def _twfe_weights_payload( + results: Any, kind: str, standardize: bool, absolute_value: bool +) -> Dict[str, Any]: + """Backend-agnostic data for either view (shared by the two renderers).""" + if kind == "weights": + table = getattr(results, "weights", None) + if table is None: + table = results.cells + weight = table["weight"].to_numpy(dtype=float) + return { + "post": table["post"].to_numpy().astype(bool), + "weight": weight, + "att": table["att"].to_numpy(dtype=float), + "labels": [f"({g}, {t})" for g, t in zip(table["group"], table["time"])], + "n_negative": int((weight < 0).sum()), + } + balance = results.covariate_balance(level="summary", standardize=standardize) + suffix = "_std_diff" if standardize else "_diff" + unweighted = balance["unweighted" + suffix].to_numpy(dtype=float) + weighted = balance["weighted" + suffix].to_numpy(dtype=float) + if absolute_value: + unweighted, weighted = np.abs(unweighted), np.abs(weighted) + finite = np.isfinite(unweighted) & np.isfinite(weighted) + if not finite.any(): + raise ValueError( + "the covariate balance table has no finite differences to plot " + "(every covariate is degenerate - e.g. constant or binary with a zero " + "pooled SD under standardize=True); try standardize=False or check the " + "balance_covariates" + ) + limit = float(np.max(np.abs(np.concatenate([unweighted[finite], weighted[finite]]))) or 1.0) + return { + "unweighted": unweighted, + "weighted": weighted, + "labels": [str(c) for c in balance["covariate"]], + "limit": limit, + } + + +def _render_twfe_weights_mpl( + payload: Dict[str, Any], + *, + kind: str, + standardize: bool, + figsize: Tuple[float, float], + title: Optional[str], + xlabel: Optional[str], + ylabel: Optional[str], + post_color: str, + pre_color: str, + markersize: int, + alpha: float, + annotate: bool, + ax: Optional[Any], + show: bool, +) -> Any: + from diff_diff.visualization._common import _require_matplotlib + + plt = _require_matplotlib() + if ax is None: + _, ax = plt.subplots(figsize=figsize) + + if kind == "weights": + post, weight, att = payload["post"], payload["weight"], payload["att"] + ax.axhline(0, color="0.4", linewidth=1.2, zorder=1) + ax.axvline(0, color="0.4", linewidth=1.2, zorder=1) + for mask, color, label in ( + (post, post_color, "post-treatment"), + (~post, pre_color, "pre-treatment"), + ): + if mask.any(): + ax.scatter( + weight[mask], + att[mask], + s=markersize, + alpha=alpha, + color=color, + label=label, + zorder=3, + ) + if annotate: + for w, a, lab in zip(weight, att, payload["labels"]): + ax.annotate(lab, (w, a), fontsize=8, xytext=(4, 4), textcoords="offset points") + ax.set_xlabel(xlabel or "Implicit weight") + ax.set_ylabel(ylabel or "ATT(g, t)") + default_title = "Implicit weights on group-time effects" + if payload["n_negative"]: + default_title += f" ({payload['n_negative']} negative)" + ax.set_title(title or default_title) + ax.legend(frameon=False) + else: + unweighted, weighted, limit = payload["unweighted"], payload["weighted"], payload["limit"] + ax.axhline(0, color="0.4", linewidth=1.2, zorder=1) + ax.scatter(unweighted, weighted, s=markersize, alpha=alpha, color=post_color, zorder=3) + ax.plot( + [0, limit], + [0, limit], + color="0.6", + linestyle="--", + linewidth=1.0, + zorder=2, + label="no improvement", + ) + if annotate: + for x, y, name in zip(unweighted, weighted, payload["labels"]): + ax.annotate(name, (x, y), fontsize=8, xytext=(4, 4), textcoords="offset points") + kindword = "standardized " if standardize else "" + ax.set_xlabel(xlabel or f"Unweighted {kindword}difference") + ax.set_ylabel(ylabel or f"Implicitly-weighted {kindword}difference") + ax.set_title(title or "Covariate balance under the implicit weights") + ax.legend(frameon=False) + + if show: + plt.show() + return ax + + +def _render_twfe_weights_plotly( + payload: Dict[str, Any], + *, + kind: str, + standardize: bool, + figsize: Tuple[float, float], + title: Optional[str], + xlabel: Optional[str], + ylabel: Optional[str], + post_color: str, + pre_color: str, + markersize: int, + alpha: float, + annotate: bool, + ax: Optional[Any], + show: bool, +) -> Any: + from diff_diff.visualization._common import _require_plotly + + go = _require_plotly() + fig = go.Figure() + marker_px = max(4.0, float(np.sqrt(markersize))) # matplotlib area -> plotly diameter + mode = "markers+text" if annotate else "markers" + + if kind == "weights": + post, weight, att = payload["post"], payload["weight"], payload["att"] + labels = np.asarray(payload["labels"], dtype=object) + for mask, color, name in ( + (post, post_color, "post-treatment"), + (~post, pre_color, "pre-treatment"), + ): + if mask.any(): + fig.add_trace( + go.Scatter( + x=weight[mask], + y=att[mask], + mode=mode, + name=name, + text=labels[mask] if annotate else None, + textposition="top right", + marker={"size": marker_px, "color": color, "opacity": alpha}, + hovertemplate="%{text}
weight=%{x:.4f}
ATT(g,t)=%{y:.4f}", + customdata=None, + ) + ) + fig.data[-1].text = labels[mask] + fig.add_hline(y=0, line={"color": "gray", "width": 1.2}) + fig.add_vline(x=0, line={"color": "gray", "width": 1.2}) + default_title = "Implicit weights on group-time effects" + if payload["n_negative"]: + default_title += f" ({payload['n_negative']} negative)" + fig.update_layout( + title=title or default_title, + xaxis_title=xlabel or "Implicit weight", + yaxis_title=ylabel or "ATT(g, t)", + ) + else: + unweighted, weighted, limit = payload["unweighted"], payload["weighted"], payload["limit"] + fig.add_trace( + go.Scatter( + x=unweighted, + y=weighted, + mode=mode, + name="covariates", + text=payload["labels"], + textposition="top right", + marker={"size": marker_px, "color": post_color, "opacity": alpha}, + hovertemplate="%{text}
unweighted=%{x:.4f}
weighted=%{y:.4f}", + ) + ) + fig.add_trace( + go.Scatter( + x=[0, limit], + y=[0, limit], + mode="lines", + name="no improvement", + line={"color": "gray", "dash": "dash", "width": 1.0}, + ) + ) + fig.add_hline(y=0, line={"color": "gray", "width": 1.2}) + kindword = "standardized " if standardize else "" + fig.update_layout( + title=title or "Covariate balance under the implicit weights", + xaxis_title=xlabel or f"Unweighted {kindword}difference", + yaxis_title=ylabel or f"Implicitly-weighted {kindword}difference", + ) + + fig.update_layout( + width=int(figsize[0] * 100), + height=int(figsize[1] * 100), + template="plotly_white", + legend={"orientation": "h", "y": -0.15}, + ) + if show: + fig.show() + return fig diff --git a/docs/api/index.rst b/docs/api/index.rst index dd2917402..ae593daf2 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -76,6 +76,8 @@ Result containers returned by estimators: diff_diff.TwoStageBootstrapResults diff_diff.SpilloverDiDResults diff_diff.BaconDecompositionResults + diff_diff.ATTGTWeightsResult + diff_diff.TWFEDecompositionResult diff_diff.wooldridge_results.WooldridgeDiDResults diff_diff.lpdid_results.LPDiDResults diff_diff.changes_in_changes_results.ChangesInChangesResults @@ -119,6 +121,7 @@ Plotting functions and plot builders: diff_diff.plot_honest_event_study diff_diff.RDPlot diff_diff.plot_bacon + diff_diff.plot_twfe_weights diff_diff.plot_power_curve diff_diff.plot_pretrends_power @@ -138,6 +141,8 @@ Placebo tests and model diagnostics: diff_diff.leave_one_out_test diff_diff.run_all_placebo_tests diff_diff.PlaceboTestResults + diff_diff.attgt_weights + diff_diff.decompose_twfe_weights diff_diff.RDDensityTest Panel Profiling @@ -400,6 +405,7 @@ Diagnostics & Inference honest_did power pretrends + twfe_weights Reporting ~~~~~~~~~ diff --git a/docs/api/twfe_weights.rst b/docs/api/twfe_weights.rst new file mode 100644 index 000000000..ebf809ef2 --- /dev/null +++ b/docs/api/twfe_weights.rst @@ -0,0 +1,159 @@ +TWFE Weight Diagnostics (Callaway ``twfeweights``) +=================================================== + +What a two-way fixed effects regression *implicitly* weights. + +Run on staggered-adoption data, a TWFE regression does not estimate a simple +average of the underlying group-time effects ATT(g, t). It estimates a +weighted average, and some of those weights can be **negative** -- so the +coefficient need not lie in the convex hull of the effects it summarizes. +This module reports those weights, next to the weights the target estimands +ATT\ :sup:`O` and ATT\ :sup:`simple` would use, and decomposes the regression +back into its building blocks. + +**When to use these diagnostics:** + +- You have a staggered design and want to see, cell by cell, what your TWFE + specification is actually averaging +- You want to quantify how much of a TWFE estimate comes from *pre-treatment* + cells -- i.e. from parallel-trends violations rather than from treatment +- Your TWFE and :class:`~diff_diff.CallawaySantAnna` estimates disagree and + you want to see which cells drive the gap +- You adjusted for covariates and want to check whether the regression's + implicit weights actually *balance* them + +**How this differs from the neighbouring surfaces:** + +- :func:`diff_diff.twowayfeweights` implements de Chaisemartin & + D'Haultfoeuille (2020) Theorem 1 and weights **(unit, time) cells**. The + functions here weight **ATT(g, t) parameters**. +- :class:`diff_diff.BaconDecomposition` decomposes TWFE into **2x2 DiD + comparisons**. :func:`diff_diff.decompose_twfe_weights` decomposes it into + **group-time effects**, plus a pre-trend-violation term. + +**Reference:** Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & +Sant'Anna, P. H. C. (2025). Difference-in-Differences Designs: A +Practitioner's Guide. arXiv:2503.13323. Callaway, B., & Sant'Anna, P. H. C. +(2021) for the ATT\ :sup:`O` / ATT\ :sup:`simple` weights. + +Ported from the ``twfeweights`` R package (v0.9.0) by Brantly Callaway, MIT +License, Copyright (c) 2023 Brantly Callaway. + +.. module:: diff_diff.twfe_weights + +attgt_weights +------------- + +Weights an estimand places on each group-time effect. + +.. autofunction:: diff_diff.attgt_weights + +decompose_twfe_weights +---------------------- + +Decomposition of a TWFE estimate into weighted group-time effects. + +.. autofunction:: diff_diff.decompose_twfe_weights + +plot_twfe_weights +----------------- + +.. autofunction:: diff_diff.plot_twfe_weights + +Result Objects +-------------- + +.. autoclass:: diff_diff.ATTGTWeightsResult + :members: + :undoc-members: + :show-inheritance: + :no-index: + +.. autoclass:: diff_diff.TWFEDecompositionResult + :members: + :undoc-members: + :show-inheritance: + :no-index: + +Example Usage +------------- + +Inspecting what a TWFE regression weights +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import diff_diff + + panel = diff_diff.load_mpdta() + + cs = diff_diff.CallawaySantAnna( + control_group="never_treated", + base_period="universal", # required for aggregation="twfe" + ).fit( + panel, outcome="lemp", unit="countyreal", time="year", + first_treat="first_treat", + ) + + weights = diff_diff.attgt_weights(cs, aggregation="twfe") + print(weights.summary()) + print(weights.n_negative, "cells carry negative weight") + +Comparing against the estimand you meant to report +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``aggregation="overall"`` and ``"simple"`` give the Callaway & Sant'Anna +target-parameter weights, which are non-negative and sum to one. The gap +between ``implied_att`` values is the cost of the TWFE specification: + +.. code-block:: python + + import diff_diff + + panel = diff_diff.load_mpdta() + cs = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal", + ).fit( + panel, outcome="lemp", unit="countyreal", time="year", + first_treat="first_treat", + ) + + for aggregation in ("twfe", "overall", "simple"): + w = diff_diff.attgt_weights(cs, aggregation=aggregation) + print(f"{aggregation:8s} {w.implied_att: .4f} " + f"({w.n_negative} negative weights)") + +Separating treatment effects from pre-trend violations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import diff_diff + + panel = diff_diff.load_mpdta() + + decomposition = diff_diff.decompose_twfe_weights( + panel, + outcome="lemp", unit="countyreal", time="year", + first_treat="first_treat", + covariates=["lpop"], + balance_covariates=["lpop"], + ) + + print(decomposition.summary()) + print("from pre-treatment cells:", decomposition.pretrend_bias) + + # Do the implicit weights balance the covariates? + print(decomposition.covariate_balance()) + + diff_diff.plot_twfe_weights(decomposition) + +Validation +---------- + +Validated against R ``twfeweights`` 0.9.0 output on three fixtures (``mpdta`` +plus two simulated panels). Goldens live at +``benchmarks/data/twfeweights_golden.json`` and are regenerated with +``Rscript benchmarks/R/generate_twfeweights_golden.R``; R is never needed to +run the test suite. Tolerances and their rationale are in +``docs/methodology/REGISTRY.md`` under "TWFE Weight Diagnostics". diff --git a/docs/api/visualization.rst b/docs/api/visualization.rst index b143d2d20..2b03605a0 100644 --- a/docs/api/visualization.rst +++ b/docs/api/visualization.rst @@ -194,6 +194,12 @@ plot_bacon Visualize Goodman-Bacon decomposition results. +.. seealso:: + + :func:`diff_diff.plot_twfe_weights` renders the implicit ATT(g, t) weights + and their covariate balance. It is documented on its own page, + :doc:`twfe_weights`, and supports the same ``backend=`` options. + .. autofunction:: diff_diff.plot_bacon Example diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index 7beb3e6c3..fcb56740b 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -21,6 +21,9 @@ # Members resolve to the first entry (the primary module) for doc lookup. # ────────────���───────────────────────────────────────────────────────── groups: + twfe_weights: + - diff_diff/twfe_weights.py + - diff_diff/twfe_weights_results.py staggered: - diff_diff/staggered.py - diff_diff/staggered_aggregation.py @@ -1122,6 +1125,36 @@ sources: - path: docs/r_comparison.rst type: user_guide + # ── TWFE Weight Diagnostics ───���───────────────────────────────────────── + + diff_diff/twfe_weights.py: + drift_risk: low + docs: + - path: docs/methodology/REGISTRY.md + section: "TWFE Weight Diagnostics" + type: methodology + - path: docs/api/twfe_weights.rst + type: api_reference + - path: README.md + section: "Diagnostics & Sensitivity (one-line catalog entry)" + type: user_guide + - path: docs/references.rst + type: user_guide + - path: diff_diff/guides/llms.txt + section: "Diagnostics and Sensitivity Analysis" + type: user_guide + - path: diff_diff/guides/llms-full.txt + section: "TWFE Weight Diagnostics" + type: user_guide + diff_diff/twfe_weights_results.py: + drift_risk: low + docs: + - path: docs/api/twfe_weights.rst + type: api_reference + - path: docs/methodology/REGISTRY.md + section: "TWFE Weight Diagnostics" + type: methodology + # ── BaconDecomposition ───���───────────────────────────────────────── diff_diff/bacon.py: diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index e19c8601c..fdc19ea34 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -38,6 +38,7 @@ This document provides the academic foundations and key implementation requireme 5. [Diagnostics and Sensitivity](#diagnostics-and-sensitivity) - [PlaceboTests](#placebotests) - [BaconDecomposition](#bacondecomposition) + - [TWFE Weight Diagnostics](#twfe-weight-diagnostics) - [HonestDiD](#honestdid) - [PreTrendsPower](#pretrendspower) - [PowerAnalysis](#poweranalysis) @@ -5698,6 +5699,132 @@ Where `n_k` is the sample share of timing group `k`, `n_{kℓ} = n_k / (n_k + n_ --- +## TWFE Weight Diagnostics + +**Primary source:** [Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & Sant'Anna, P. H. C. (2025). "Difference-in-Differences Designs: A Practitioner's Guide." arXiv:2503.13323](https://arxiv.org/abs/2503.13323) + +**Secondary source:** [Callaway, B., & Sant'Anna, P. H. C. (2021). Difference-in-Differences with multiple time periods. *Journal of Econometrics*, 225(2), 200-230.](https://doi.org/10.1016/j.jeconom.2020.12.001) — for the ATT^O / ATT^simple target-parameter weights. + +**Reference implementation:** the `twfeweights` R package (v0.9.0) by Brantly Callaway, MIT License, Copyright (c) 2023 Brantly Callaway. The upstream notice is reproduced in the module docstring of `diff_diff/twfe_weights.py`, as its terms require. + +**Scope:** these are DIAGNOSTICS, not estimators. Both result containers subclass `Diagnostic` and carry no inference quintet — the decomposition is an algebraic identity, so there is nothing to attach a standard error to. The headline scalars are named `implied_att` and `estimate` rather than `att` for the same reason. + +### Relationship to neighbouring surfaces + +- **vs `twowayfeweights` (de Chaisemartin & D'Haultfoeuille 2020, Theorem 1):** that surface weights **(unit, time) cells**; these functions weight **ATT(g,t) parameters** — the cohort-by-period building blocks. Both detect negative weighting in staggered TWFE, but they decompose along different axes and their weight tables are not comparable row-for-row. The names are deliberately disjoint (`attgt_weights` / `ATTGTWeightsResult` vs `twowayfeweights` / `TWFEWeightsResult`). +- **vs `BaconDecomposition` (Goodman-Bacon 2021):** Bacon decomposes TWFE into **2x2 DiD comparisons** and asks which comparisons drive the estimate. `decompose_twfe_weights` decomposes it into **group-time effects** and additionally isolates a pre-trend-violation term. Use Bacon to see the forbidden comparisons; use this to see the per-`(g,t)` weights and how much of the estimate is not a treatment effect at all. + +### Estimator equations (as implemented) + +All expressions are evaluated in POSITIONAL time (periods mapped to `1..T`, cohorts to their period position, never-treated staying `0`), so `maxT == T`. + +*ATT(g,t) weights — `attgt_weights(aggregation=...)`:* + +`aggregation="twfe"` (R `twfe_weights`), with `p_g` the share of ALL units in cohort `g` and `E_t[D]` the share of units treated by `t`: + +``` +h(g,t) = 1[t >= g] - (maxT - g + 1)/T - E_t[D] + mean_t E_t[D] +num(g,t) = h(g,t) * p_g +w(g,t) = num(g,t) / sum over {t >= g, g != 0} of num(g,t) +``` + +`aggregation="overall"` (ATT^O, R `attO_weights`), with `pbar_g` the share of EVER-TREATED units in cohort `g`: + +``` +w(g,t) = 1[t >= g] * pbar_g / (maxT - g + 1) +``` + +`aggregation="simple"` (ATT^simple, R `att_simple_weights`): + +``` +w(g,t) = 1[t >= g] * pbar_g, then normalized to sum to one +``` + +*FWL decomposition — `decompose_twfe_weights(method="fwl")` (R `implicit_twfe_weights`):* + +Double-demean the treatment indicator `D` and the covariates `X` over unit and period, project the demeaned treatment on the demeaned covariates, and take the residual: + +``` +gamma = argmin_b || Ddot - Xdot b ||_w +resid = Ddot - Xdot gamma +alpha_den = E_w[resid * Ddot] +``` + +The residual IS the implicit weight the regression applies to each observation. Per `(g,t)` cell, with the treated and comparison weights each normalized to mean one: + +``` +alpha_weight(g,t) = E_w[resid | G=g, T=t] * p_g / (alpha_den * T) +ATT(g,t) = E_w[wtreated * Ytilde | G=g] - E_w[wcontrol * Ytilde | G=0] +``` + +where `Ytilde` is the outcome measured against the base period (`Y_t - Y_1` under `base_period="first_period"`, `Y_t - Y_{g-1}` under `"gmin1"`). Roll-ups: + +``` +decomposition = sum over all cells of alpha_weight * ATT +remainder = sum of alpha_weight * cell remainder (0 unless base_period="gmin1") +estimate = decomposition + remainder +pretrend_bias = sum over PRE cells (t < g) of alpha_weight * ATT +``` + +Under parallel trends every pre-treatment ATT(g,t) is zero and `pretrend_bias` vanishes; a non-zero value is the contribution of parallel-trends violations to the TWFE coefficient. + +*Cross-surface identity (pinned by `tests/test_twfe_weights_parity.py::TestCrossSurfaceIdentity`):* when the CS fit used `base_period="universal"`, `control_group="never_treated"` and no covariates, + +``` +attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate +``` + +Verified on `mpdta` at `-0.03654894` from both directions. + +### Edge cases + +- **Note (grid completeness is a hard error):** `attgt_weights` fails closed on an incomplete group-time grid: a weight table over a partial grid is not the named estimand. `aggregation="twfe"` requires every cohort x period cell (pre cells enter `h(g,t)`); `"overall"` / `"simple"` require every post-treatment cell. The error names the missing cells and each cell's CS `skip_reason`. Only the two structural gaps below are exempt. A non-estimable PRE cell of a surviving cohort under `"overall"` / `"simple"` is still merely dropped-and-warned and counted in `n_dropped_cells`: those estimands place no weight on pre cells, so nothing renormalizes and no number moves. +- **Note (matches R `did`'s first-period drop):** a cohort with NO estimable post-treatment cell, canonically one treated in the first observed period (no base period), is excluded from the weight table AND from the cohort masses (`p_g`, `pbar_g`, `E_t[D]`) with a `UserWarning`, exactly as `did::pre_process_did` drops units already treated in the first period. The criterion is *post* cells, not all cells: a cohort can have an estimable universal-base pre cell and still no usable post cell. On a bare ATT(g,t) frame with no `skip_reason` column, only a cohort whose label equals the first observed period is excludable this way; any other absent cohort raises, being indistinguishable from user truncation. +- **Note (matches R `aggte`'s available-period averaging under not-yet-treated controls):** on a `control_group="not_yet_treated"` fit the last cohorts run out of comparison units and CS marks those post cells `skip_reason="zero_treated_control"`. For `"overall"` / `"simple"` they are treated as structurally absent: `"overall"` divides each cohort by its number of AVAILABLE post periods instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells, which is what R `aggte(type="group")` / `aggte(type="simple")` compute on such a fit. A `UserWarning` names the cells. `"twfe"` requires a never-treated comparison group and never reaches this branch; a bare frame without `skip_reason` stays strict. +- **Note (cohort labels are validated):** never-treated is exactly `0` or `+inf`. Any other non-finite label (NaN, `-inf`) raises rather than being silently absorbed into cohort 0; before this check a single NaN label moved `decompose_twfe_weights(...).estimate` by ~1.4% with no warning. Within-unit invariance uses `nunique(dropna=False)` so a label that is NaN in one period fails, and non-finite period labels are rejected up front. +- **Note (`aggregation="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping warns instead; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. +- **Note (sampling weights are validated):** unit weights must be finite, non-negative, with positive total and positive treated mass. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `aggregation="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. +- `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. +- `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. +- `attgt_weights` rejects repeated-cross-section fits and unbalanced-panel fallbacks: `E_t[D]` and the cohort shares average over a fixed unit set. + +### Notes and deviations + +- **Note (upstream `fixest::demean` segfault on the no-covariate branch):** `twfeweights::implicit_twfe_weights(xformula = ~1)` builds `model.matrix(~-1, data)`, an `nT x 0` matrix, and `fixest::demean()` SEGFAULTS on a zero-column matrix (reproduced in isolation on R 4.6.1 / fixest 0.14.2: `fixest::demean(matrix(numeric(0), 10, 0), ids)` → `*** caught segfault *** memory not mapped`). This is a zero-column bug, not a property of any fixture. The no-covariate golden is therefore generated with a TIME-INVARIANT covariate, which double-demeaning annihilates exactly, making the call numerically the `~1` branch; the parity test asserts BOTH `covariates=None` and `covariates=[]` against that single golden, so the equivalence is proven rather than assumed. Verified on `mpdta`: `twfe_weights(att_gt(...))` aggregate and `implicit_twfe_weights(xformula = ~lpop)$est` both equal `-0.03654894`. +- **Note (annihilated covariates are dropped before the projection, as numerical hygiene):** a covariate with no within-unit-and-period variation leaves a column of pure rounding noise after double-demeaning (~1e-16 against a raw scale of ~1). Keeping it is not catastrophic: the column lies in the fixed-effect span and is orthogonal to the treatment residual, so on mpdta's `lpop` it moves the FWL residual by ~2e-18. But regressing on an exactly-zero column is meaningless, and dropping it is what makes `covariates=None` and `covariates=[]` agree exactly rather than approximately. diff-diff judges each column against its own PRE-demeaning norm; a rank test on the demeaned matrix alone cannot see this, because there 1e-16 is simply the largest pivot. **Limitation:** the 1e-10 relative threshold is blunt, so a covariate with a large level and genuinely small within-variation can trip it. The `UserWarning` says so and suggests centring or rescaling. +- **Deviation from R (0/0 cells report the limit, not the rounding noise):** for the never-treated comparison group the double-demeaned treatment is CONSTANT within a period (`-E_t[D] + mean_t E_t[D]`), and for some cohort structures that constant is analytically ZERO — on the `sim_staggered` fixture (three equal cohorts at `g in {0,3,4}`, `T=5`) it vanishes exactly at `t=3`, where `-1/3 + 1/3 = 0`. The cell's implicit weights are then `0/0`. diff-diff returns the limit (a constant divided by its own mean is one), giving the plain unweighted contrast; R divides the two rounding errors and lands ~3e-4 away. Verified against a hand-computed contrast that uses none of this module's machinery: diff-diff is exact to 4.4e-16. A `UserWarning` names the affected cells. **`estimate` is unaffected either way** — the weights on such cells cancel exactly (on `sim_staggered`, `w(3,3) + w(4,3) = 0`), so it matches R to 1e-15. Other user-visible fields DO move, because the cancelling cells straddle the pre/post split. Measured against the pinned R values on `sim_staggered`: `pretrend_bias` and `post_only` each by ~1.2e-4 (equal and opposite, so their sum stays exact), `effective_sample_size` by ~0.99, per-cell `ess` by up to ~0.53, and per-cell `remainder` (under `base_period="gmin1"`) correspondingly. All print in `summary()`. The parity suite asserts every one: tight wherever the degeneracy is not detected, and at the degenerate cells against R's own weights with our limit value substituted only where R's number is 0/0 noise. +- **Deviation from R (positional time rescaling in `attgt_weights`):** R evaluates `(maxT - g + 1) / length(tlist)` on the RAW period labels, which is only correct when those labels are consecutive integers. diff-diff maps periods to `1..T` first (mirroring `BMisc::orig2t`, which R already applies inside `implicit_twfe_weights` but not inside `twfe_weights`). Bit-identical on consecutive grids — `mpdta`'s 2003..2007 maps to 1..5 and both give `4/5` at `g = 2004` — and correct on gapped ones. Pinned by a test that remaps periods to 10, 20, 30, 40, 50. +- **Deviation from R (`keep_untreated` not exposed):** R's `keep_untreated=TRUE` synthesizes `G = 0` rows with `attgt = 0` to mirror an internal vector layout. Those rows are excluded from every normalization (`cond <- .t >= .group & .group != 0`) and contribute exactly zero, so the argument is numerically inert. +- **Deviation from R (consolidated API):** upstream exports 21 symbols in a flat namespace. diff-diff exposes five: `attgt_weights` (folding `twfe_weights` / `attO_weights` / `att_simple_weights` behind `aggregation=`), `decompose_twfe_weights` (folding `implicit_twfe_weights` behind `method=`), the two result classes, and `plot_twfe_weights` (replacing `ggtwfeweights`). The per-cell helpers and the eleven balance statistics are private and pinned through the public surfaces that expose them. The two two-period kernels (`two_period_reg_weights` / `two_period_aipw_weights`) and the AIPW blocks are captured in the golden but read by no test: they are labelled **reserved** in the JSON `meta` and the generator header, pinned so the `method="aipw"` follow-up needs no R re-run. The AIPW golden is covariate-adjusted, since a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score. +- **Deviation from R (post-lasso block out of scope):** `did_post_lasso` / `did_post_lasso_ra` are not ported. The upstream source is unfinished — `R/did_post_lasso.R:69` contains a leftover `browser()` call and references undefined variables — so there is no runnable reference to validate against, and it would add an sklearn dependency. +- **Deviation from R (`method="aipw"` not yet implemented):** upstream's `implicit_aipw_weights` is out of scope for the initial port; `method=` currently accepts `"fwl"` only and raises listing the accepted values. +- **Note (`log_ratio_sd` scaling preserved verbatim):** upstream scales each group's standard deviation by `sqrt(n - 1)` before taking the log ratio, which is not a conventional standard deviation. Preserved as-is for parity; the quantity is only read as a relative balance statistic and the factor largely cancels in the ratio. +- **Note (`frac_treated_extreme` is a step function):** upstream routes through `BMisc::weighted_ecdf` → `make_dist` (an `approxfun(method="constant")` classed as `ecdf`) → `stats:::quantile.ecdf`, which does NOT invert the step function but rebuilds a pseudo-sample by repeating each knot `diff(c(0, round(nobs * F)))` times and takes an ordinary type-7 quantile of that. diff-diff reproduces this exactly, including the `NA` return when the covariate has fewer than three distinct values, and that `NA` survives the summary roll-up. R propagates NA if any contributing cell is NA, whereas a plain pandas `.sum()` skips it and would report a spurious `0.0` for a binary or constant covariate; the roll-up therefore masks on the `post` column (not on a zero roll-up weight, since a post cell whose implicit weight is exactly zero still contributes) and returns NaN whenever any contributing post cell is NaN. Because the statistic is a step function of a weighted ECDF, a perturbation of order 1e-12 can move one unit across a knot and shift the value by `1/n`; parity is gated accordingly. +- **Note (negative-weight statistics; no R counterpart):** over the `g != 0` grid the TWFE weights sum to zero, post cells to +1 and pre cells to -1, so `n_negative` is non-zero and `negative_weight_share` sits near 0.5 in EVERY staggered design, including one with no negative post-period weight. The pathology the literature describes is negative weight on POST cells, so `ATTGTWeightsResult` reports both, labelled: `n_negative` / `negative_weight_share` over all cells, and `n_negative_post` / `negative_post_weight_share` restricted to post cells (the share of post-period weight MASS that is negative). `summary()` leads with the post-only figures. R reports neither statistic. +- **Note (weighted `aggregation="twfe"` is a diff-diff extension):** R's `twfe_weights` takes no `w=`, so there is no upstream reference for a weighted TWFE weight table. The equations above define `p_g` and `E_t[D]` as unweighted shares; with `weights=` they become the corresponding weighted shares (each unit's mass is its sampling weight rather than one), the same algebra a weighted TWFE regression implies on a balanced panel. Pinned by asserting that `attgt_weights(fit, aggregation="twfe", weights=w)` and `decompose_twfe_weights(panel, weights="w")` produce identical weight vectors to 1e-12, plus a frozen-numbers regression test on a synthetic weighted panel (every parity fixture is unweighted, so R cannot gate this path). +- **Note (linear algebra runs through the house helpers):** the two-way demeaning is `diff_diff.utils.within_transform` (the same alternating projections `fixest::demean` runs) applied to the sorted long frame before the `(unit, period)` reshape, with the treatment indicator synthesized as a column since it is derived from cohorts x positional periods rather than supplied. The Frisch-Waugh-Lovell solve is `diff_diff.linalg.solve_ols(..., weights=, rank_deficient_action="silent")`: on a rank-deficient design it fits the maximal independent set, sets the aliased coefficients to `NaN` (R-style) and returns the residual computed from the identified ones, so the module reads the dropped column names off the `NaN` positions and uses the returned residual directly. This replaced a bespoke pivoted QR whose docstring claimed to drop "later columns first"; it did not, being the same norm-pivoted QR `solve_ols` uses, and dropping the same column. +- **Note (diff-diff adds standardized differences):** `covariate_balance(standardize=True)` appends `unweighted_std_diff` / `weighted_std_diff` (difference divided by the pooled SD). R does not emit these; they are additive, so parity is asserted on the R columns only. A zero pooled SD yields NaN rather than an infinity. +- **Note (balance is requested up front, not bolted on):** R mutates a `decomposed_twfe` object in a second pass (`twfe_cov_bal`). diff-diff computes the table at construction when `balance_covariates=` is supplied and exposes it via `covariate_balance()`, so the result never retains the raw panel — consistent with the `AggregationKit` data-minimization contract. Calling `covariate_balance()` without having requested it raises with the fix inlined. + +### R output parity + +Goldens: `benchmarks/data/twfeweights_golden.json`, plus two simulated sibling panel CSVs. The `mpdta` fixture reads the shared `benchmarks/data/mpdta_stata_panel.csv` and derives `lpop_t` from a `derived_columns` expression in the golden, rather than committing a renamed copy; the generator asserts the two sources agree to CSV round-trip precision. Every cells block carries ORIGINAL period labels: `implicit_*` run in positional time internally, and the generator maps them back so one convention holds throughout and the tests assert labels rather than array position. Regenerated by `benchmarks/R/generate_twfeweights_golden.R`; R is needed only to regenerate, never to run the tests. Tests: `tests/test_twfe_weights_parity.py`. + +Three fixtures: `mpdta` (real; non-`1..T` period labels; provenance `data(mpdta, package="did")`), `sim_staggered` (three equal cohorts of 100, which is exactly what makes the comparison-group normalizer vanish at `t=3`, so this fixture deliberately exercises the degenerate cells above; its `pretrend_bias` is non-zero but the ~0.093 is sampling noise, since `x1` is iid and cohorts are assigned by unit index, so no differential pre-trend is designed in), and `unbalanced_cohorts` (120/70/60, which breaks the `p_g == 1/3` degeneracy that would let a cohort-share bug pass silently on the equal-cohort fixture). + +| Surface | Gate | Rationale | +|---------|------|-----------| +| ATT(g,t) weights, all three aggregations | `atol=1e-12` | Closed-form rational expression in cohort masses; only double-precision representation error separates the two sides. Observed max deviation 4.7e-16. | +| `implied_att` (R's own ATT(g,t) fed back in) | `atol=1e-12` | Isolates the weight arithmetic from CallawaySantAnna-vs-`did` parity. | +| End-to-end from a CS fit | `rtol=1e-6` | COMPOSED check — carries the pre-existing CS parity band, not this module's. | +| FWL decomposition scalars and cell weights | `atol=1e-10` | R double-demeans with `fixest::demean`, iterative alternating projections at a 1e-8 fixed-point tolerance; ours is the exact closed form on a balanced panel. The gap is fixest's convergence slack. | +| FWL with covariates | `atol=1e-8` | The demeaning slack propagates through the OLS projection of `Ddot` on `Xdot`. | +| Covariate balance (11 statistics) | `atol=1e-9` | Smooth functions of the weights above. Observed max deviation 7.3e-11. | +| Per-cell ATT, `pretrend_bias` / `post_only`, and (under `gmin1`) the decomposition/remainder split at DEGENERATE cells | `atol=5e-2` | R reports 0/0 rounding noise there; we report the exact limit. Degeneracy is DETECTED from the weight structure, never hard-coded to a fixture or period, and `estimate` stays on the tight gate everywhere. Under `base_period="first_period"` the remainder is identically zero, so the split is gated tight even where the mask fires. | +| `effective_sample_size` and per-cell `ess` at DEGENERATE cells | expected value rebuilt from R's own cells | R's `ess` at a 0/0 cell is a ratio of rounding errors (scalar gap ~0.99, per-cell up to ~0.53). The expectation uses R's weights and R's `ess` wherever the degeneracy is not detected, substituting our limit value only at the detected cells, so the assertion is anchored to R rather than to our own implementation. | + +--- + ## HonestDiD **Primary source:** [Rambachan, A., & Roth, J. (2023). A More Credible Approach to Parallel Trends. *Review of Economic Studies*, 90(5), 2555-2591.](https://doi.org/10.1093/restud/rdad018) diff --git a/docs/references.rst b/docs/references.rst index 5f7f7347d..7ea0dc912 100644 --- a/docs/references.rst +++ b/docs/references.rst @@ -292,6 +292,11 @@ Multi-Period and Staggered Adoption - **Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & Sant'Anna, P. H. C. (2025).** "Difference-in-Differences Designs: A Practitioner's Guide." *arXiv preprint* arXiv:2503.13323. https://arxiv.org/abs/2503.13323 + Primary source for the implicit-TWFE-weight diagnostics + (:func:`diff_diff.attgt_weights`, :func:`diff_diff.decompose_twfe_weights`). + Reference implementation: the ``twfeweights`` R package (v0.9.0) by Brantly + Callaway, MIT License, Copyright (c) 2023 Brantly Callaway. + Source for the 8-step practitioner workflow surfaced via ``diff_diff.get_llm_guide("practitioner")`` and the README ``## Practitioner Workflow`` section. See ``docs/methodology/REGISTRY.md`` for the diff-diff renumbering and per-step deviations. Double/Debiased Machine Learning diff --git a/tests/helpers/results_foundation.py b/tests/helpers/results_foundation.py index c7168bb81..0f229dde0 100644 --- a/tests/helpers/results_foundation.py +++ b/tests/helpers/results_foundation.py @@ -43,6 +43,48 @@ def make_constructed_diagnostics() -> Dict[str, Any]: poly = pd.DataFrame({"rdplot_x": [-1.0, 0.0, 1.0], "rdplot_y": [0.9, 1.4, 2.1]}) coef = pd.DataFrame({"side": ["left", "right"], "coef_0": [1.0, 2.0]}) + # TWFE weight diagnostics: a 2-cohort x 2-period grid with one negative + # weight, so summary() exercises the negative-weight branch. + attgt_weight_cells = pd.DataFrame( + { + "group": [2, 2, 3, 3], + "time": [2, 3, 2, 3], + "post": [1, 1, 0, 1], + "weight": [0.6, 0.5, -0.2, 0.1], + "att": [1.0, 1.2, 0.0, 0.8], + } + ) + decomposition_cells = pd.DataFrame( + { + "group": [2, 2, 3, 3], + "time": [2, 3, 2, 3], + "post": [1, 1, 0, 1], + "att": [1.0, 1.2, 0.0, 0.8], + "weight": [0.4, 0.3, 0.1, 0.2], + "ess": [8.0, 8.0, 6.0, 6.0], + "remainder": [0.0, 0.0, 0.0, 0.0], + } + ) + decomposition_balance = pd.DataFrame( + { + "group": [2, 2, 3, 3], + "time": [2, 3, 2, 3], + "post": [1, 1, 0, 1], + "covariate": ["x1", "x1", "x1", "x1"], + "unweighted_treated": [0.5, 0.5, 0.4, 0.4], + "unweighted_control": [0.3, 0.3, 0.2, 0.2], + "unweighted_diff": [0.2, 0.2, 0.2, 0.2], + "weighted_treated": [0.5, 0.5, 0.4, 0.4], + "weighted_control": [0.45, 0.45, 0.38, 0.38], + "weighted_diff": [0.05, 0.05, 0.02, 0.02], + "sd": [1.0, 1.0, 1.0, 1.0], + "unweighted_log_ratio_sd": [0.01, 0.01, 0.02, 0.02], + "weighted_log_ratio_sd": [0.005, 0.005, 0.01, 0.01], + "unweighted_frac_extreme": [0.05, 0.05, 0.06, 0.06], + "weighted_frac_extreme": [0.04, 0.04, 0.05, 0.05], + } + ) + qug = diff_diff.QUGTestResults( t_stat=1.2, p_value=0.23, @@ -271,5 +313,60 @@ def make_constructed_diagnostics() -> Dict[str, Any]: interpretation="All applicable checks passed.", applicable_checks=("parallel_trends",), ), + # Every count / share is DERIVED from the cells so the fixture cannot + # drift from the object it imitates. + "ATTGTWeightsResult": diff_diff.ATTGTWeightsResult( + weights=attgt_weight_cells, + aggregation="twfe", + implied_att=float((attgt_weight_cells["weight"] * attgt_weight_cells["att"]).sum()), + n_negative=int((attgt_weight_cells["weight"] < 0).sum()), + negative_weight_share=float( + attgt_weight_cells["weight"].clip(upper=0).abs().sum() + / attgt_weight_cells["weight"].abs().sum() + ), + n_negative_post=int( + ((attgt_weight_cells["weight"] < 0) & (attgt_weight_cells["post"] == 1)).sum() + ), + negative_post_weight_share=float( + attgt_weight_cells.loc[attgt_weight_cells["post"] == 1, "weight"] + .clip(upper=0) + .abs() + .sum() + / attgt_weight_cells.loc[attgt_weight_cells["post"] == 1, "weight"].abs().sum() + ), + n_cells=len(attgt_weight_cells), + source="CallawaySantAnnaResults", + control_group="never_treated", + base_period="universal", + ), + "TWFEDecompositionResult": diff_diff.TWFEDecompositionResult( + cells=decomposition_cells, + method="fwl", + estimate=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), + decomposition=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), + remainder=0.0, + pretrend_bias=float( + (decomposition_cells["weight"] * decomposition_cells["att"])[ + decomposition_cells["post"] == 0 + ].sum() + ), + post_only=float( + (decomposition_cells["weight"] * decomposition_cells["att"])[ + decomposition_cells["post"] == 1 + ].sum() + ), + base_period="first_period", + covariates=("x1",), + # Module identity: post_count * sum_post(weight * ess). + effective_sample_size=float( + (decomposition_cells["post"] == 1).sum() + * (decomposition_cells["weight"] * decomposition_cells["ess"])[ + decomposition_cells["post"] == 1 + ].sum() + ), + n_units=12, + n_periods=4, + balance=decomposition_balance, + ), } return instances diff --git a/tests/test_diagnostic_marker.py b/tests/test_diagnostic_marker.py index 1b43397cc..55c5f0369 100644 --- a/tests/test_diagnostic_marker.py +++ b/tests/test_diagnostic_marker.py @@ -48,6 +48,8 @@ "StuteJointResult", "HADPretestReport", "DiagnosticReportResults", + "ATTGTWeightsResult", + "TWFEDecompositionResult", ] # Representative ESTIMATOR results: marked with BaseResults, never Diagnostic. diff --git a/tests/test_doc_snippets.py b/tests/test_doc_snippets.py index da2d06cc1..e957eedaf 100644 --- a/tests/test_doc_snippets.py +++ b/tests/test_doc_snippets.py @@ -46,6 +46,7 @@ "api/dml_did.rst", "api/mmm.rst", "api/triple_diff.rst", + "api/twfe_weights.rst", "practitioner_decision_tree.rst", "practitioner_getting_started.rst", "python_comparison.rst", diff --git a/tests/test_naming_guard.py b/tests/test_naming_guard.py index 82444be4e..0adc1bc5d 100644 --- a/tests/test_naming_guard.py +++ b/tests/test_naming_guard.py @@ -557,6 +557,31 @@ def _build_rowed_index(): "TripleDifference.fit[group]": ( "rule-3 reserved treated-group 0/1 indicator (v4-design section 8 rule 3)" ), + # The TWFE weight diagnostics share vocabulary with three rename families + # without reading any of them. `time=` here is the panel PERIOD COLUMN + # NAME (the same role as CallawaySantAnna.fit[time], which no row touches), + # not the two-period 0/1 post dummy M-030/M-031/M-082/M-137/M-138 rename to + # `post`: both functions are staggered-only. `aggregation=` selects an + # ESTIMAND ("twfe" / "overall" / "simple"), not the Wooldridge output + # granularity M-044 renames to `level` and M-087 removes. + **{ + f"{fn}[time]": ( + "panel PERIOD COLUMN NAME (as in CallawaySantAnna.fit[time]), not " + "the two-period 0/1 post dummy renamed to `post` by " + "M-030/M-031/M-082/M-137/M-138; survives 4.0" + ) + for fn in ("attgt_weights", "decompose_twfe_weights") + }, + "attgt_weights[aggregation]": ( + "ESTIMAND selector ('twfe' / 'overall' / 'simple'), not the " + "WooldridgeDiDResults output granularity M-044 renames to `level` and " + "M-087 removes; survives 4.0" + ), + "ATTGTWeightsResult.aggregation": ( + "records which ESTIMAND's weights the result holds - the " + "attgt_weights[aggregation] value, not a Wooldridge output granularity " + "(M-044 / M-087); survives 4.0" + ), "run_placebo_test[time]": ( "OVERLOADED pass-through, redesign pending (TODO.md): forwarded as " "the calendar column to placebo_timing_test/placebo_group_test AND " @@ -981,6 +1006,23 @@ def _token_family_code_refs(tok): # field. The one file that DID name it - diff_diff/guides/llms-full.txt - # was migrated in this same diff (migrate-first rule) and remains a lane # hit only through its unrelated backticked schema key. + # The TWFE weight diagnostics document their own `time=` (panel period + # COLUMN) and `aggregation=` (estimand selector) on these two surfaces; + # neither reads a renamed name. See the SURFACE_ALLOWLIST entries for + # attgt_weights / decompose_twfe_weights. + ("time", "diff_diff/guides/llms.txt"): ( + "attgt_weights / decompose_twfe_weights document a panel PERIOD COLUMN " + "named `time`, not the two-period 0/1 post dummy renamed by " + "M-030/M-031/M-082/M-137/M-138" + ), + ("aggregation", "diff_diff/guides/llms.txt"): ( + "attgt_weights' ESTIMAND selector, not WooldridgeDiDResults' output " + "granularity (M-044 / M-087)" + ), + ("aggregation", "docs/methodology/REGISTRY.md"): ( + "the TWFE Weight Diagnostics section documents attgt_weights' ESTIMAND " + "selector, not WooldridgeDiDResults' output granularity (M-044 / M-087)" + ), ("estimator", "diff_diff/aggregation.py"): ( "AggregationResult.estimator - independent field holding a CLASS NAME" ), diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py new file mode 100644 index 000000000..3c93df678 --- /dev/null +++ b/tests/test_twfe_weights.py @@ -0,0 +1,1084 @@ +"""Contract, guard and edge-case tests for the TWFE weight diagnostics. + +R output parity lives in ``tests/test_twfe_weights_parity.py``; this module +covers the behaviour that is ours rather than R's - the input guards, the +result-object surface, and the design restrictions we enforce as errors. +""" + +import numpy as np +import pandas as pd +import pytest + +import diff_diff +from diff_diff.twfe_weights import attgt_weights + + +def _panel(seed=11, n_per_cohort=40, n_periods=5, cohorts=(0, 3, 4)): + """Balanced staggered panel with a never-treated group.""" + rng = np.random.RandomState(seed) + first_treat = np.repeat(np.array(cohorts), n_per_cohort) + n_units = len(first_treat) + unit_fe = rng.normal(size=n_units) + rows = [] + for t in range(1, n_periods + 1): + treated = (first_treat != 0) & (t >= first_treat) + rows.append( + pd.DataFrame( + { + "unit": np.arange(n_units), + "period": t, + "first_treat": first_treat, + "outcome": ( + unit_fe + + 0.5 * t + + 1.0 * treated * (t - first_treat + 1) + + rng.normal(scale=0.3, size=n_units) + ), + } + ) + ) + return pd.concat(rows, ignore_index=True).sort_values(["unit", "period"]) + + +def _fit(df, **kwargs): + params = {"control_group": "never_treated", "base_period": "universal"} + params.update(kwargs) + return diff_diff.CallawaySantAnna(**params).fit( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + +@pytest.fixture(scope="module") +def panel(): + return _panel() + + +@pytest.fixture(scope="module") +def fitted(panel): + return _fit(panel) + + +class TestPublicSurface: + def test_exported_from_package_root(self): + assert diff_diff.attgt_weights is attgt_weights + for name in ("attgt_weights", "ATTGTWeightsResult", "TWFEDecompositionResult"): + assert name in diff_diff.__all__ + + def test_name_is_distinct_from_the_dcdh_surface(self): + """The two weight surfaces must stay separately addressable.""" + assert diff_diff.attgt_weights is not diff_diff.twowayfeweights + assert diff_diff.ATTGTWeightsResult is not diff_diff.TWFEWeightsResult + + def test_result_is_a_diagnostic_without_the_quintet(self, fitted): + result = attgt_weights(fitted) + assert isinstance(result, diff_diff.Diagnostic) + for banned in ("att", "se", "t_stat", "p_value", "conf_int"): + assert not hasattr(result, banned) + + def test_result_renders(self, fitted): + result = attgt_weights(fitted) + text = result.summary() + assert "Implicit Weights on ATT(g, t)" in text + assert "TWFE regression" in text + frame = result.to_dataframe() + assert list(frame.columns) == ["group", "time", "post", "weight", "att"] + # to_dataframe hands back a copy, not the live table + frame.loc[0, "weight"] = 999.0 + assert result.weights.loc[0, "weight"] != 999.0 + assert set(result.to_dict()) >= {"aggregation", "implied_att", "weights"} + assert "aggregation='twfe'" in repr(result) + + +class TestAggregationBehaviour: + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_implied_att_is_the_weighted_sum(self, fitted, aggregation): + result = attgt_weights(fitted, aggregation=aggregation) + expected = (result.weights["weight"] * result.weights["att"]).sum() + assert result.implied_att == pytest.approx(expected, abs=1e-14) + + @pytest.mark.parametrize("aggregation", ["overall", "simple"]) + def test_target_estimands_are_convex(self, fitted, aggregation): + """ATT^O / ATT^simple weights are non-negative and sum to one.""" + weights = attgt_weights(fitted, aggregation=aggregation).weights["weight"] + assert (weights >= 0).all() + assert weights.sum() == pytest.approx(1.0, abs=1e-12) + + def test_twfe_weights_can_be_negative(self, fitted): + """The whole point of the diagnostic: staggered TWFE is not convex.""" + result = attgt_weights(fitted, aggregation="twfe") + assert result.n_negative > 0 + assert 0.0 < result.negative_weight_share < 1.0 + assert "Negative POST-period cells:" in result.summary() + assert result.n_negative_post <= result.n_negative + + def test_pre_treatment_cells_carry_weight_under_twfe(self, fitted): + """TWFE loads on pre-treatment cells; the CS estimands do not.""" + twfe = attgt_weights(fitted, aggregation="twfe").weights + assert (twfe.loc[twfe["post"] == 0, "weight"].abs() > 0).any() + for aggregation in ("overall", "simple"): + benign = attgt_weights(fitted, aggregation=aggregation).weights + assert (benign.loc[benign["post"] == 0, "weight"] == 0).all() + + def test_rejects_unknown_aggregation(self, fitted): + with pytest.raises(ValueError, match="aggregation must be one of"): + attgt_weights(fitted, aggregation="everything") + + +class TestDesignGuards: + def test_rejects_non_universal_base_period_for_twfe(self, panel): + fit = _fit(panel, base_period="varying") + with pytest.raises(ValueError, match="base_period='universal'"): + attgt_weights(fit, aggregation="twfe") + + def test_varying_base_is_fine_for_the_cs_estimands(self, panel): + """Only the TWFE formula needs the complete grid.""" + fit = _fit(panel, base_period="varying") + for aggregation in ("overall", "simple"): + result = attgt_weights(fit, aggregation=aggregation) + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + + def test_rejects_not_yet_treated_control_for_twfe(self, panel): + fit = _fit(panel, control_group="not_yet_treated") + with pytest.raises(ValueError, match="control_group='never_treated'"): + attgt_weights(fit, aggregation="twfe") + + def test_rejects_repeated_cross_sections(self, panel): + # A true RCS needs one observation per unit id, so re-key the rows + # rather than just flipping the flag (panel=False rejects duplicates). + rcs = panel.copy().reset_index(drop=True) + rcs["unit"] = np.arange(len(rcs)) + fit = _fit(rcs, panel=False) + with pytest.raises(ValueError, match="requires a panel fit"): + attgt_weights(fit) + + +class TestDataFrameFallback: + def test_requires_the_full_panel_spec(self, fitted, panel): + frame = fitted.to_dataframe("group_time") + with pytest.raises(ValueError, match="missing"): + attgt_weights(frame, data=panel, unit="unit") + + def test_rejects_panel_args_alongside_a_fitted_result(self, fitted, panel): + with pytest.raises(ValueError, match="only for the DataFrame fallback"): + attgt_weights( + fitted, + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_accepts_an_att_column_as_well_as_effect(self, fitted, panel): + frame = fitted.to_dataframe("group_time")[["group", "time", "effect"]] + via_effect = attgt_weights( + frame, data=panel, unit="unit", time="period", first_treat="first_treat" + ) + via_att = attgt_weights( + frame.rename(columns={"effect": "att"}), + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + np.testing.assert_allclose( + via_effect.weights["weight"], via_att.weights["weight"], atol=1e-15 + ) + + def test_rejects_a_frame_without_an_effect_column(self, panel): + frame = pd.DataFrame({"group": [3], "time": [3]}) + with pytest.raises(ValueError, match="'effect' or 'att'"): + attgt_weights( + frame, + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_rejects_time_varying_cohort_labels(self, fitted, panel): + broken = panel.copy() + broken.loc[broken.index[0], "first_treat"] = 99 + with pytest.raises(ValueError, match="varies within unit"): + attgt_weights( + fitted.to_dataframe("group_time"), + data=broken, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_source_is_recorded(self, fitted, panel): + assert attgt_weights(fitted).source == "CallawaySantAnnaResults" + from_frame = attgt_weights( + fitted.to_dataframe("group_time"), + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + assert from_frame.source == "DataFrame" + + +class TestNonConsecutiveTimeLabels: + """Positional rescaling: gapped period labels must not change the weights.""" + + def test_gapped_periods_match_consecutive_ones(self, panel): + consecutive = attgt_weights(_fit(panel), aggregation="twfe") + + gapped = panel.copy() + remap = {1: 10, 2: 20, 3: 30, 4: 40, 5: 50} + gapped["period"] = gapped["period"].map(remap) + gapped["first_treat"] = gapped["first_treat"].map(lambda g: remap.get(g, 0)) + result = attgt_weights(_fit(gapped), aggregation="twfe") + + np.testing.assert_allclose( + result.weights["weight"].to_numpy(), + consecutive.weights["weight"].to_numpy(), + atol=1e-14, + ) + assert result.implied_att == pytest.approx(consecutive.implied_att, abs=1e-14) + + +class TestSamplingWeights: + def test_uniform_weights_are_a_no_op(self, fitted, panel): + baseline = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + weighted_panel = panel.assign(w=1.0) + weighted = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=weighted_panel, + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + np.testing.assert_allclose( + baseline.weights["weight"], weighted.weights["weight"], atol=1e-15 + ) + + def test_reweighting_a_cohort_shifts_its_weight(self, fitted, panel): + """Doubling a cohort's sampling weight raises its share of ATT^O.""" + baseline = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + tilted_panel = panel.assign(w=np.where(panel["first_treat"] == 3, 2.0, 1.0)) + tilted = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=tilted_panel, + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + mass_3_before = baseline.weights.query("group == 3")["weight"].sum() + mass_3_after = tilted.weights.query("group == 3")["weight"].sum() + assert mass_3_after > mass_3_before + assert tilted.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + + def test_rejects_time_varying_sampling_weights(self, fitted, panel): + broken = panel.copy() + broken["w"] = np.arange(len(broken), dtype=float) + with pytest.raises(ValueError, match="must be time-invariant"): + attgt_weights( + fitted.to_dataframe("group_time"), + data=broken, + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + + def test_rejects_a_weights_column_name_on_the_fitted_path(self, fitted): + with pytest.raises(ValueError, match="only name a column"): + attgt_weights(fitted, weights="w") + + +class TestDegenerateInputs: + def test_rejects_a_panel_with_no_treated_units(self, panel): + frame = pd.DataFrame({"group": [3.0], "time": [3.0], "effect": [1.0]}) + never = panel.assign(first_treat=0) + with pytest.raises(ValueError, match="no ever-treated units"): + attgt_weights( + frame, + data=never, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_rejects_a_frame_with_no_finite_effects(self, panel): + frame = pd.DataFrame({"group": [3.0, 3.0], "time": [3.0, 4.0], "effect": [np.nan, np.nan]}) + with pytest.raises(ValueError, match="no finite effects"): + attgt_weights( + frame, + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_warns_and_renormalizes_when_cells_are_dropped(self, fitted, panel): + frame = fitted.to_dataframe("group_time").copy() + frame.loc[frame.index[0], "effect"] = np.nan + with pytest.warns(UserWarning, match="had no estimable ATT"): + result = attgt_weights( + frame, + aggregation="overall", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + assert result.n_dropped_cells == 1 + assert len(result.weights) == len(frame) - 1 + assert "Non-estimable cells dropped:" in result.summary() + + def test_rejects_a_cohort_label_off_the_period_grid(self, panel): + frame = pd.DataFrame({"group": [3.0], "time": [3.0], "effect": [1.0]}) + broken = panel.copy() + broken.loc[broken["first_treat"] == 4, "first_treat"] = 99 + with pytest.raises(ValueError, match="not one of the observed time periods"): + attgt_weights( + frame, + data=broken, + unit="unit", + time="period", + first_treat="first_treat", + ) + + +class TestWeightedRegressionPin: + """Frozen-numbers pin for the WEIGHTED branches. + + No parity fixture passes ``weights=`` and R ``twfe_weights`` has no ``w=``, + so the weighted code paths (weighted two-way demeaning, weighted FWL solve, + weighted cohort masses) have no external oracle. These literals were + captured from the implementation BEFORE the linear algebra was routed + through ``diff_diff.linalg.solve_ols`` / ``diff_diff.utils.within_transform`` + and pin that behaviour: any refactor must leave them green at 1e-12. + """ + + @staticmethod + def _weighted_panel(): + rng = np.random.default_rng(20260907) + n_per, n_periods = 12, 5 + cohorts = [0] * n_per + [3] * n_per + [4] * n_per + rows = [] + for i, g in enumerate(cohorts): + w = float(rng.choice([0.5, 1.0, 1.5, 2.5])) + alpha = rng.normal() + for t in range(1, n_periods + 1): + x = rng.normal() + 0.3 * t + effect = 1.0 * (t - g + 1) if (g and t >= g) else 0.0 + y = alpha + 0.2 * t + 0.5 * x + effect + rng.normal(scale=0.3) + rows.append({"id": i, "t": t, "g": g, "y": y, "x": x, "w": w}) + return pd.DataFrame(rows) + + _DEC = { + "nocov": dict( + kwargs={}, + estimate=1.4221735897240102, + pretrend_bias=0.5127724996498023, + post_only=0.9094010900742079, + ess=59.999999999999986, + weight=[ + -0.26383763837638374, + -0.26383763837638374, + 0.3726937269372693, + 0.07749077490774903, + 0.07749077490774903, + -0.059040590405904064, + -0.059040590405904064, + -0.3542435424354244, + 0.23616236162361626, + 0.23616236162361626, + ], + att=[ + 0.0, + -1.4436560985156102, + 0.2605332041282682, + 1.5423748763069138, + 2.020457826078349, + 0.0, + -0.8817595960499504, + -0.22533107108782402, + 1.0544057969164409, + 1.2161310006314463, + ], + ), + "cov": dict( + kwargs={"covariates": ["x"]}, + estimate=1.3930847792561663, + pretrend_bias=0.5397671816632773, + post_only=0.8533175975928889, + ess=58.816020983744, + weight=[ + -0.26642873140338846, + -0.26217358219812914, + 0.3731356281678529, + 0.07703988099301252, + 0.07842680444065203, + -0.05851961726129342, + -0.05862391113360511, + -0.35425415800358373, + 0.23433448557874187, + 0.23706320081974044, + ], + att=[ + 0.0, + -1.4333600946359863, + 0.1470326297519402, + 1.502553090163623, + 1.9646825183908976, + 0.0, + -0.8405505765908995, + -0.32378354968013, + 1.0394622134176261, + 1.2023475510925157, + ], + ), + "gmin1": dict( + kwargs={"base_period": "gmin1"}, + estimate=1.4221735897240106, + pretrend_bias=-0.3554385674608017, + post_only=1.777612157184812, + ess=59.999999999999986, + weight=None, # identical to nocov (weights do not depend on the base period) + att=[ + 1.4436560985156102, + 0.0, + 1.7041893026438784, + 2.986030974822525, + 3.4641139245939594, + 0.2253310710878238, + -0.6564285249621264, + 0.0, + 1.279736868004265, + 1.4414620717192705, + ], + ), + } + + @pytest.mark.parametrize("key", ["nocov", "cov", "gmin1"]) + def test_decomposition_weighted_branches(self, key): + spec = self._DEC[key] + df = self._weighted_panel() + result = diff_diff.decompose_twfe_weights( + df, outcome="y", unit="id", time="t", first_treat="g", weights="w", **spec["kwargs"] + ) + assert result.estimate == pytest.approx(spec["estimate"], abs=1e-12) + assert result.pretrend_bias == pytest.approx(spec["pretrend_bias"], abs=1e-12) + assert result.post_only == pytest.approx(spec["post_only"], abs=1e-12) + assert result.effective_sample_size == pytest.approx(spec["ess"], abs=1e-9) + expected_w = spec["weight"] if spec["weight"] is not None else self._DEC["nocov"]["weight"] + np.testing.assert_allclose(result.cells["weight"].to_numpy(), expected_w, atol=1e-12) + np.testing.assert_allclose(result.cells["att"].to_numpy(), spec["att"], atol=1e-12) + + _AGG = { + "twfe": ( + 1.4348838554104435, + [ + -0.2638376383763837, + -0.2638376383763837, + 0.3726937269372694, + 0.0774907749077491, + 0.0774907749077491, + -0.059040590405904085, + -0.059040590405904085, + -0.3542435424354242, + 0.2361623616236162, + 0.2361623616236162, + ], + ), + "overall": ( + 2.101822892918353, + [ + 0, + 0, + 0.17874396135265702, + 0.17874396135265702, + 0.17874396135265702, + 0, + 0, + 0, + 0.2318840579710145, + 0.2318840579710145, + ], + ), + "simple": ( + 2.239531384413449, + [ + 0, + 0, + 0.21142857142857147, + 0.21142857142857147, + 0.21142857142857147, + 0, + 0, + 0, + 0.18285714285714288, + 0.18285714285714288, + ], + ), + } + + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_attgt_weighted_branches(self, aggregation): + df = self._weighted_panel() + cs = diff_diff.CallawaySantAnna(base_period="universal", control_group="never_treated").fit( + df, outcome="y", unit="id", time="t", first_treat="g" + ) + unit_w = df.groupby("id", sort=True)["w"].first().to_numpy() + result = attgt_weights(cs, aggregation=aggregation, weights=unit_w) + implied, weight = self._AGG[aggregation] + assert result.implied_att == pytest.approx(implied, abs=1e-12) + np.testing.assert_allclose(result.weights["weight"].to_numpy(), weight, atol=1e-12) + assert list(zip(result.weights["group"], result.weights["time"])) == [ + (3, 1), + (3, 2), + (3, 3), + (3, 4), + (3, 5), + (4, 1), + (4, 2), + (4, 3), + (4, 4), + (4, 5), + ] + + +# --------------------------------------------------------------------------- +# Review-response regression tests (PR #812 items 1-5, 9, 17, 19) +# --------------------------------------------------------------------------- + + +def _gt_frame(fitted): + return fitted.to_dataframe("group_time") + + +def _frame_call(frame, panel, **kw): + return attgt_weights( + frame, data=panel, unit="unit", time="period", first_treat="first_treat", **kw + ) + + +class TestCohortLabelValidation: + """Item 1: NaN / -inf labels are errors, never a silent never-treated unit.""" + + @pytest.mark.parametrize("bad", [np.nan, -np.inf]) + def test_decompose_rejects_non_finite_labels(self, panel, bad): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[df["unit"] == 45, "first_treat"] = bad + with pytest.raises(ValueError, match="NaN or -inf cohort label"): + diff_diff.decompose_twfe_weights( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + @pytest.mark.parametrize("bad", [np.nan, -np.inf]) + def test_frame_path_rejects_non_finite_labels(self, fitted, panel, bad): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[df["unit"] == 45, "first_treat"] = bad + with pytest.raises(ValueError, match="NaN or -inf cohort label"): + _frame_call(_gt_frame(fitted), df) + + def test_plus_inf_is_never_treated(self, fitted, panel): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[df["first_treat"] == 0, "first_treat"] = np.inf + with_inf = _frame_call(_gt_frame(fitted), df) + with_zero = _frame_call(_gt_frame(fitted), panel) + np.testing.assert_allclose( + with_inf.weights["weight"].to_numpy(), + with_zero.weights["weight"].to_numpy(), + atol=1e-15, + ) + + def test_nan_in_one_period_fails_invariance(self, fitted, panel): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[(df["unit"] == 45) & (df["period"] == 2), "first_treat"] = np.nan + with pytest.raises(ValueError, match="varies within unit"): + _frame_call(_gt_frame(fitted), df) + with pytest.raises(ValueError, match="varies within unit"): + diff_diff.decompose_twfe_weights( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + def test_nan_period_label_is_rejected_up_front(self, fitted, panel): + df = panel.copy() + df["period"] = df["period"].astype(float) + df.loc[(df["unit"] == 45) & (df["period"] == 2), "period"] = np.nan + with pytest.raises(ValueError, match="non-finite or non-numeric period"): + _frame_call(_gt_frame(fitted), df) + with pytest.raises(ValueError, match="non-finite or non-numeric period"): + diff_diff.decompose_twfe_weights( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + +class TestBalanceNaNPropagation: + """Item 2: frac_extreme's NA for <3 distinct values survives the summary roll-up.""" + + @pytest.fixture(scope="class") + def decomposed(self): + df = _panel() + rng = np.random.RandomState(3) + df["binary"] = rng.binomial(1, 0.4, size=len(df)).astype(float) + df["const"] = 1.0 + df["cont"] = rng.normal(size=len(df)) + # Make the binary / constant columns unit-invariant so the unit mean + # keeps them at <3 distinct values. + df["binary"] = df.groupby("unit")["binary"].transform("first") + return diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["cont"], + balance_covariates=["binary", "const", "cont"], + ) + + def test_cell_level_is_nan_for_degenerate_covariates(self, decomposed): + cells = decomposed.covariate_balance(level="cell") + for cov in ("binary", "const"): + sub = cells[cells["covariate"] == cov] + assert sub["unweighted_frac_extreme"].isna().all() + assert sub["weighted_frac_extreme"].isna().all() + cont = cells[cells["covariate"] == "cont"] + assert np.isfinite(cont["unweighted_frac_extreme"]).all() + + def test_summary_level_propagates_nan_not_zero(self, decomposed): + summary = decomposed.covariate_balance(level="summary").set_index("covariate") + for cov in ("binary", "const"): + assert np.isnan(summary.loc[cov, "unweighted_frac_extreme"]) + assert np.isnan(summary.loc[cov, "weighted_frac_extreme"]) + # The other statistics are ordinary sums and stay finite. + assert np.isfinite(summary.loc[cov, "unweighted_diff"]) + assert np.isfinite(summary.loc["cont", "weighted_frac_extreme"]) + + +class TestCovariateGuard: + """Item 3: a covariate-adjusted CS fit is not a TWFE regression.""" + + def test_covariate_adjusted_fit_is_rejected_under_twfe(self, panel): + df = panel.copy() + df["x"] = np.random.RandomState(5).normal(size=len(df)) + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + ) + assert fit._aggregation_kit.bookkeeping["covariates"] == ("x",) + with pytest.raises(ValueError, match="requires a fit without covariates"): + attgt_weights(fit, aggregation="twfe") + # The CS estimands do not depend on the regression specification. + assert attgt_weights(fit, aggregation="overall").n_negative_post == 0 + + def test_unadjusted_fit_records_empty_covariates(self, fitted): + assert fitted._aggregation_kit.bookkeeping["covariates"] == () + + def test_legacy_kit_without_the_key_warns(self, fitted): + kit = fitted._aggregation_kit + saved = kit.bookkeeping.pop("covariates") + try: + with pytest.warns(UserWarning, match="predates covariate bookkeeping"): + attgt_weights(fitted, aggregation="twfe") + finally: + kit.bookkeeping["covariates"] = saved + + def test_wrong_result_type_is_a_type_error(self, panel): + dec = diff_diff.decompose_twfe_weights( + panel, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + with pytest.raises(TypeError, match="CallawaySantAnna"): + attgt_weights(dec) # type: ignore[arg-type] + + +class TestFrameValidationAndGrid: + """Item 4: duplicates, non-finite cells, and incomplete grids fail closed.""" + + def test_duplicate_cells_are_rejected(self, fitted, panel): + frame = _gt_frame(fitted) + dup = pd.concat([frame, frame.iloc[[2]]], ignore_index=True) + with pytest.raises(ValueError, match="duplicated \\(group, time\\)"): + _frame_call(dup, panel) + + def test_non_finite_group_label_is_rejected(self, fitted, panel): + frame = _gt_frame(fitted) + frame.loc[0, "group"] = np.nan + with pytest.raises(ValueError, match="NaN or -inf cohort label"): + _frame_call(frame, panel) + + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_inf_att_on_a_post_cell_is_an_incomplete_grid(self, fitted, panel, aggregation): + frame = _gt_frame(fitted) + idx = frame.index[(frame["group"] == 3) & (frame["time"] == 3)][0] + frame.loc[idx, "effect"] = np.inf + with pytest.raises(ValueError, match="complete .* grid"): + _frame_call(frame, panel, aggregation=aggregation) + + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_missing_post_cell_raises_for_every_aggregation(self, fitted, panel, aggregation): + frame = _gt_frame(fitted) + frame = frame[~((frame["group"] == 3) & (frame["time"] == 3))] + with pytest.raises(ValueError, match="required cell\\(s\\) are missing"): + _frame_call(frame, panel, aggregation=aggregation) + + def test_missing_pre_cell_raises_for_twfe_but_warns_for_cs_estimands(self, fitted, panel): + frame = _gt_frame(fitted) + frame = frame[~((frame["group"] == 4) & (frame["time"] == 1))] + with pytest.raises(ValueError, match="complete cohort x period grid"): + _frame_call(frame, panel, aggregation="twfe") + complete = _frame_call(_gt_frame(fitted), panel, aggregation="overall") + for aggregation in ("overall", "simple"): + # An ABSENT pre row is not a drop: nothing to warn about, weights unchanged. + partial = _frame_call(frame, panel, aggregation=aggregation) + assert partial.n_dropped_cells == 0 + assert len(partial.weights) == len(frame) + ref = _frame_call(_gt_frame(fitted), panel, aggregation=aggregation) + assert partial.implied_att == pytest.approx(ref.implied_att, abs=1e-15) + # A NaN pre cell (present but non-estimable) is what n_dropped_cells counts. + frame = _gt_frame(fitted) + frame.loc[frame.index[(frame["group"] == 4) & (frame["time"] == 1)][0], "effect"] = np.nan + with pytest.warns(UserWarning, match="pre-treatment group-time cell"): + dropped = _frame_call(frame, panel, aggregation="overall") + assert dropped.n_dropped_cells == 1 + assert dropped.implied_att == pytest.approx(complete.implied_att, abs=1e-15) + + def test_first_period_cohort_is_dropped_like_r_did(self): + df = _panel(cohorts=(0, 1, 3, 4), n_periods=5) + fit = _fit(df) + with pytest.warns(UserWarning, match="no estimable post-treatment cell"): + result = attgt_weights(fit, aggregation="overall") + assert 1 not in set(result.weights["group"]) + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + # Same numbers as fitting on the panel with those units removed up front. + pre_filtered = _fit(df[df["first_treat"] != 1]) + reference = attgt_weights(pre_filtered, aggregation="overall") + np.testing.assert_allclose( + result.weights["weight"].to_numpy(), reference.weights["weight"].to_numpy(), atol=1e-12 + ) + + def test_bare_frame_missing_a_whole_non_first_cohort_raises(self, fitted, panel): + frame = _gt_frame(fitted).drop(columns=["skip_reason"], errors="ignore") + frame = frame[frame["group"] != 4] + with pytest.raises(ValueError, match="no post-treatment cell in the ATT"): + _frame_call(frame, panel, aggregation="overall") + + def test_not_yet_treated_carve_out_mirrors_aggte(self): + df = _panel(cohorts=(3, 4, 5), n_periods=6) + fit = _fit(df, control_group="not_yet_treated") + with pytest.warns(UserWarning) as record: + result = attgt_weights(fit, aggregation="overall") + messages = " | ".join(str(w.message) for w in record) + assert "structurally absent" in messages # (3,5),(3,6),(4,5),(4,6) + assert "no estimable post-treatment cell" in messages # cohort 5 + assert set(result.weights["group"]) == {3, 4} + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + post = result.weights[result.weights["post"] == 1] + # cohort 3 keeps (3,3),(3,4): divisor 2; cohort 4 keeps (4,4): divisor 1 + assert post[post["group"] == 3]["time"].tolist() == [3, 4] + assert post[post["group"] == 4]["time"].tolist() == [4] + w3 = post[post["group"] == 3]["weight"].to_numpy() + w4 = post[post["group"] == 4]["weight"].to_numpy() + assert w3[0] == pytest.approx(w3[1]) + assert w4[0] == pytest.approx(2 * w3[0]) # equal cohorts: pbar_3 == pbar_4 + expected = float((post["weight"] * post["att"]).sum()) + assert result.implied_att == pytest.approx(expected, abs=1e-12) + with pytest.raises(ValueError, match="control_group='never_treated'"): + attgt_weights(fit, aggregation="twfe") + + +class TestWeightValidation: + """Item 5: finite, non-negative, positive treated (and control) mass.""" + + @pytest.mark.parametrize( + "mutate, match", + [ + (lambda w: np.where(np.arange(len(w)) < 50, -1.0, w), "non-negative"), + (lambda w: np.where(np.arange(len(w)) == 0, np.nan, w), "must be finite"), + (lambda w: np.where(np.arange(len(w)) == 0, np.inf, w), "must be finite"), + (lambda w: np.zeros_like(w), "sum to zero"), + ], + ) + def test_bad_unit_weights_are_rejected(self, fitted, mutate, match): + w = mutate(np.ones(len(fitted._aggregation_kit.bookkeeping["unit_cohorts"]))) + with pytest.raises(ValueError, match=match): + attgt_weights(fitted, aggregation="overall", weights=w) + + def test_zero_control_mass_only_matters_where_controls_enter(self, fitted, panel): + cohorts = np.asarray(fitted._aggregation_kit.bookkeeping["unit_cohorts"], dtype=float) + w = np.where(cohorts == 0, 0.0, 1.0) + with pytest.raises(ValueError, match="never-treated comparison group carries zero"): + attgt_weights(fitted, aggregation="twfe", weights=w) + for aggregation in ("overall", "simple"): + assert attgt_weights(fitted, aggregation=aggregation, weights=w).n_cells > 0 + + def test_decompose_reports_a_nan_weight_as_non_finite(self, panel): + df = panel.copy() + df["w"] = 1.0 + df.loc[(df["unit"] == 3) & (df["period"] == 2), "w"] = np.nan + with pytest.raises(ValueError, match="must be finite"): + diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + + +class TestNegativePostWeights: + """Item 9: the pathology is negative weight on POST cells.""" + + @pytest.mark.parametrize("aggregation", ["overall", "simple"]) + def test_cs_estimands_have_no_negative_post_weight(self, fitted, aggregation): + result = attgt_weights(fitted, aggregation=aggregation) + assert result.n_negative_post == 0 + assert result.negative_post_weight_share == 0.0 + + def test_twfe_reports_both_labelled(self, fitted): + result = attgt_weights(fitted, aggregation="twfe") + assert result.n_negative_post <= result.n_negative + assert 0.0 <= result.negative_post_weight_share <= 1.0 + d = result.to_dict() + assert {"n_negative_post", "negative_post_weight_share"} <= set(d) + assert "Negative POST-period cells:" in result.summary() + + +class TestWeightedTwfeExtension: + """Item 17: weighted aggregation="twfe" has no R counterpart; tie it to the decomposition.""" + + def test_weighted_twfe_weights_match_the_weighted_decomposition(self, panel): + df = panel.copy() + rng = np.random.RandomState(9) + unit_w = pd.Series( + rng.choice([0.5, 1.0, 2.0], size=df["unit"].nunique()), + index=sorted(df["unit"].unique()), + ) + df["w"] = df["unit"].map(unit_w) + fit = _fit(df) + weighted = attgt_weights(fit, aggregation="twfe", weights=unit_w.to_numpy()) + decomposed = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + np.testing.assert_allclose( + weighted.weights["weight"].to_numpy(), decomposed.cells["weight"].to_numpy(), atol=1e-12 + ) + + +class TestHandComputedWeights: + """Item 19: assert against numbers computed OUTSIDE the module.""" + + def test_implied_att_equals_hand_computed_overall(self): + # 2 cohorts (g=2,3) x 3 periods, 10 + 30 treated units + 20 never-treated. + frame = pd.DataFrame( + { + "group": [2, 2, 2, 3, 3, 3], + "time": [1, 2, 3, 1, 2, 3], + "effect": [0.0, 1.0, 2.0, 0.0, 0.0, 4.0], + } + ) + rows = [] + for u, g in enumerate([0] * 20 + [2] * 10 + [3] * 30): + for t in (1, 2, 3): + rows.append({"unit": u, "period": t, "first_treat": g, "outcome": 0.0}) + panel = pd.DataFrame(rows) + result = _frame_call(frame, panel, aggregation="overall") + # pbar_2 = 10/40, pbar_3 = 30/40; cohort 2 has 2 post periods, cohort 3 has 1. + expected = (10 / 40) / 2 * (1.0 + 2.0) + (30 / 40) / 1 * 4.0 + assert result.implied_att == pytest.approx(expected, abs=1e-15) + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-15) + + def test_all_five_public_names_are_exported(self): + for name in ( + "attgt_weights", + "decompose_twfe_weights", + "ATTGTWeightsResult", + "TWFEDecompositionResult", + "plot_twfe_weights", + ): + assert name in diff_diff.__all__, name + assert hasattr(diff_diff, name) + + +class TestCollinearCovariates: + """Item 7: rank-deficient designs go through solve_ols' R-style NaN handling.""" + + def test_exactly_collinear_pair_warns_once_and_leaves_the_estimate_unchanged(self, panel): + df = panel.copy() + rng = np.random.RandomState(21) + df["x1"] = rng.normal(size=len(df)) + df["x2"] = 2.0 * df["x1"] # exactly collinear twin + common = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") + with pytest.warns(UserWarning, match="dropped collinear covariate") as record: + both = diff_diff.decompose_twfe_weights(df, covariates=["x1", "x2"], **common) + collinear = [w for w in record if "dropped collinear" in str(w.message)] + assert len(collinear) == 1 + message = str(collinear[0].message) + assert ("'x1'" in message) != ("'x2'" in message) # exactly one of the pair + alone = diff_diff.decompose_twfe_weights(df, covariates=["x1"], **common) + assert both.estimate == pytest.approx(alone.estimate, abs=1e-12) + np.testing.assert_allclose( + both.cells["weight"].to_numpy(), alone.cells["weight"].to_numpy(), atol=1e-12 + ) + + +class TestDecompositionEdgeCases: + """Item 8: the REGISTRY edge cases for decompose_twfe_weights, asserted.""" + + COMMON = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") + + def test_unbalanced_panel_is_rejected(self, panel): + df = panel.drop(panel.index[(panel["unit"] == 7) & (panel["period"] == 3)]) + with pytest.raises(ValueError, match="balanced panel"): + diff_diff.decompose_twfe_weights(df, **self.COMMON) + + def test_no_never_treated_group_is_rejected(self): + df = _panel(cohorts=(3, 4)) + with pytest.raises(ValueError, match="never-treated units"): + diff_diff.decompose_twfe_weights(df, **self.COMMON) + + def test_time_varying_cohort_is_rejected(self, panel): + df = panel.copy() + df.loc[(df["unit"] == 50) & (df["period"] == 5), "first_treat"] = 4 + with pytest.raises(ValueError, match="varies within unit"): + diff_diff.decompose_twfe_weights(df, **self.COMMON) + + def test_gmin1_with_a_first_period_cohort_is_rejected(self): + df = _panel(cohorts=(0, 1, 3)) + with pytest.raises(ValueError, match="gmin1"): + diff_diff.decompose_twfe_weights(df, base_period="gmin1", **self.COMMON) + + def test_balance_without_request_and_bad_level(self, panel): + result = diff_diff.decompose_twfe_weights(panel, **self.COMMON) + with pytest.raises(ValueError, match="balance_covariates="): + result.covariate_balance() + df = panel.copy() + df["x"] = np.random.RandomState(4).normal(size=len(df)) + with_balance = diff_diff.decompose_twfe_weights(df, balance_covariates=["x"], **self.COMMON) + with pytest.raises(ValueError, match="level must be"): + with_balance.covariate_balance(level="cohort") + + def test_bad_method_and_base_period(self, panel): + with pytest.raises(ValueError, match="method must be"): + diff_diff.decompose_twfe_weights(panel, method="aipw", **self.COMMON) + with_bad = dict(self.COMMON) + with pytest.raises(ValueError, match="base_period must be"): + diff_diff.decompose_twfe_weights(panel, base_period="universal", **with_bad) + + def test_identities_hold(self, panel, fitted): + result = diff_diff.decompose_twfe_weights(panel, **self.COMMON) + assert result.estimate == pytest.approx(result.decomposition + result.remainder, abs=1e-12) + assert result.pretrend_bias + result.post_only == pytest.approx( + result.decomposition, abs=1e-12 + ) + assert result.remainder == 0.0 + assert attgt_weights(fitted, aggregation="twfe").implied_att == pytest.approx( + result.estimate, abs=1e-6 + ) + gmin1 = diff_diff.decompose_twfe_weights(panel, base_period="gmin1", **self.COMMON) + assert gmin1.estimate == pytest.approx(result.estimate, abs=1e-10) + assert gmin1.estimate == pytest.approx(gmin1.decomposition + gmin1.remainder, abs=1e-12) + + +class TestPlotTWFEWeights: + """Item 8 / 13: matplotlib behaviour of plot_twfe_weights.""" + + @pytest.fixture(autouse=True) + def _agg_backend(self): + matplotlib = pytest.importorskip("matplotlib") + matplotlib.use("Agg") + yield + import matplotlib.pyplot as plt + + plt.close("all") + + def test_weights_view(self, fitted): + result = attgt_weights(fitted, aggregation="twfe") + ax = diff_diff.plot_twfe_weights(result, show=False) + assert ax.get_xlabel() == "Implicit weight" + assert "negative" in ax.get_title() + assert len(ax.collections) == 2 # post + pre scatters + + def test_ax_reuse_and_annotate(self, fitted): + import matplotlib.pyplot as plt + + _, ax = plt.subplots() + out = diff_diff.plot_twfe_weights( + attgt_weights(fitted, aggregation="overall"), ax=ax, annotate=True, show=False + ) + assert out is ax + assert len(ax.texts) == 10 + + def test_balance_view_and_auto(self, panel): + df = panel.copy() + df["x"] = np.random.RandomState(8).normal(size=len(df)) + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + ax = diff_diff.plot_twfe_weights(dec, show=False) # auto -> balance + assert "balance" in ax.get_title().lower() + ax2 = diff_diff.plot_twfe_weights(dec, kind="weights", show=False) + assert ax2.get_ylabel() == "ATT(g, t)" + + def test_balance_requested_without_table_raises(self, panel): + dec = diff_diff.decompose_twfe_weights( + panel, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + with pytest.raises(ValueError, match="no covariate balance table"): + diff_diff.plot_twfe_weights(dec, kind="balance", show=False) + + def test_all_nan_balance_table_is_a_clear_error(self, panel): + df = panel.copy() + df["const"] = 1.0 # zero pooled SD -> standardized diffs are all NaN + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + balance_covariates=["const"], + ) + with pytest.raises(ValueError, match="no finite differences"): + diff_diff.plot_twfe_weights(dec, kind="balance", show=False) + + def test_bad_kind_and_backend(self, fitted): + result = attgt_weights(fitted) + with pytest.raises(ValueError, match="kind must be"): + diff_diff.plot_twfe_weights(result, kind="heat", show=False) + with pytest.raises(ValueError, match="backend must be"): + diff_diff.plot_twfe_weights(result, backend="bokeh", show=False) diff --git a/tests/test_twfe_weights_parity.py b/tests/test_twfe_weights_parity.py new file mode 100644 index 000000000..c0c35f31c --- /dev/null +++ b/tests/test_twfe_weights_parity.py @@ -0,0 +1,589 @@ +"""R ``twfeweights`` output-parity tests for the TWFE weight diagnostics. + +Loads pre-computed golden values from +``benchmarks/data/twfeweights_golden.json`` (generated by +``benchmarks/R/generate_twfeweights_golden.R``) and asserts that the Python +implementation matches R ``twfeweights`` 0.9.0. + +**R is only needed to regenerate the JSON**, never to run these tests. The +committed JSON plus the panel CSVs it names (two simulated siblings, plus +the shared ``mpdta_stata_panel.csv``) are the source of truth and the +assertions run on any Python-only environment. Tests skip ONLY if a fixture +file is absent. + +Tolerances are module constants with a stated rationale; see the tolerance +table in ``docs/methodology/REGISTRY.md`` under "TWFE Weight Diagnostics". +""" + +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import diff_diff +from diff_diff.twfe_weights import attgt_weights + +DATA_DIR = Path(__file__).parents[1] / "benchmarks" / "data" +GOLDEN_PATH = DATA_DIR / "twfeweights_golden.json" +REGENERATE = "Rscript benchmarks/R/generate_twfeweights_golden.R" + +FIXTURES = ("mpdta", "sim_staggered", "unbalanced_cohorts") +AGGREGATIONS = ("twfe", "overall", "simple") + +# Closed-form weights: both sides evaluate the same rational expression in +# cohort masses in double precision, so only representation error separates +# them. Observed max deviation across all 3 fixtures x 3 aggregations is +# 4.7e-16 - two orders of margin below this gate. +WEIGHT_ATOL = 1e-12 +WEIGHT_RTOL = 0.0 + +# Composed check: our CallawaySantAnna ATT(g,t) vs R did::att_gt, then the +# weights on top. Bounded by the pre-existing CS parity band, not by anything +# this module introduces. +CS_COMPOSED_RTOL = 1e-6 + + +@pytest.fixture(scope="module") +def golden(): + """Load the committed R goldens; skip when absent.""" + if not GOLDEN_PATH.exists(): + pytest.skip(f"golden file not found at {GOLDEN_PATH}; run: {REGENERATE}") + with open(GOLDEN_PATH) as fh: + return json.load(fh) + + +def _fixture(golden, name): + """Panel for a fixture, with any golden-declared derived columns applied. + + ``fixtures.mpdta`` points at the SHARED ``mpdta_stata_panel.csv`` rather + than a renamed copy of it, and declares its one derived column + (``lpop_t``) as an expression evaluated against that file's own names. + Fixtures without a ``derived_columns`` block are unaffected. + """ + payload = golden["fixtures"][name] + path = DATA_DIR / payload["data_file"] + if not path.exists(): + pytest.skip(f"panel {path} not found; run: {REGENERATE}") + df = pd.read_csv(path) + for column, expression in (payload.get("derived_columns") or {}).items(): + df[column] = df.eval(expression) + return payload, df + + +def _sorted_golden_weights(block): + """Golden weight table, sorted to the same key order the API emits.""" + return ( + pd.DataFrame( + { + "group": block["group"], + "time": block["time"], + "post": block["post"], + "weight": block["weight"], + "att": block["att"], + } + ) + .sort_values(["group", "time"]) + .reset_index(drop=True) + ) + + +def _fit_cs(df, cols): + return diff_diff.CallawaySantAnna(control_group="never_treated", base_period="universal").fit( + df, + outcome=cols["outcome"], + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + + +class TestATTGTWeightsParity: + """Weights asserted against R using R's OWN ATT(g,t) values. + + Feeding the golden ``att`` column back in isolates the weight arithmetic + from CallawaySantAnna-vs-``did`` parity, which is covered separately by + ``csdid_golden_values.json``. A regression here is a regression in THIS + module. + """ + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_weight_column(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + expected = _sorted_golden_weights(payload["attgt_weights"][aggregation]) + + gt_frame = expected[["group", "time", "att"]].rename(columns={"att": "effect"}) + cols = payload["columns"] + result = attgt_weights( + gt_frame, + aggregation=aggregation, + data=df, + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + actual = result.weights.sort_values(["group", "time"]).reset_index(drop=True) + + assert len(actual) == len(expected) + np.testing.assert_array_equal(actual["group"].to_numpy(), expected["group"].to_numpy()) + np.testing.assert_array_equal(actual["time"].to_numpy(), expected["time"].to_numpy()) + np.testing.assert_array_equal(actual["post"].to_numpy(), expected["post"].to_numpy()) + np.testing.assert_allclose( + actual["weight"].to_numpy(), + expected["weight"].to_numpy(), + atol=WEIGHT_ATOL, + rtol=WEIGHT_RTOL, + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_implied_att(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + block = payload["attgt_weights"][aggregation] + expected = _sorted_golden_weights(block) + cols = payload["columns"] + + result = attgt_weights( + expected[["group", "time", "att"]].rename(columns={"att": "effect"}), + aggregation=aggregation, + data=df, + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + np.testing.assert_allclose( + result.implied_att, + block["implied_att"], + atol=WEIGHT_ATOL, + rtol=WEIGHT_RTOL, + ) + + +class TestATTGTWeightsFromCSFit: + """End-to-end: fit CallawaySantAnna, then weight its own ATT(g,t). + + This is a COMPOSED check - it multiplies this module's parity by + CallawaySantAnna-vs-``did`` parity. It is deliberately looser than + :class:`TestATTGTWeightsParity`, and a failure here with that class green + points at CS, not at the weights. + """ + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_end_to_end(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + block = payload["attgt_weights"][aggregation] + result = attgt_weights(_fit_cs(df, payload["columns"]), aggregation=aggregation) + actual = result.weights.sort_values(["group", "time"]).reset_index(drop=True) + expected = _sorted_golden_weights(block) + + np.testing.assert_allclose( + actual["weight"].to_numpy(), + expected["weight"].to_numpy(), + atol=WEIGHT_ATOL, + rtol=WEIGHT_RTOL, + ) + # ATT(g,t) come from our own fit here, so this leg carries the CS band. + np.testing.assert_allclose( + actual["att"].to_numpy(), + expected["att"].to_numpy(), + rtol=CS_COMPOSED_RTOL, + atol=1e-9, + ) + np.testing.assert_allclose( + result.implied_att, + block["implied_att"], + rtol=CS_COMPOSED_RTOL, + atol=1e-9, + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_negative_weights_are_a_twfe_phenomenon(self, golden, fixture): + """TWFE puts negative weight on some cells; the CS estimands never do.""" + payload, df = _fixture(golden, fixture) + fit = _fit_cs(df, payload["columns"]) + + twfe = attgt_weights(fit, aggregation="twfe") + assert twfe.n_negative > 0 + assert twfe.negative_weight_share > 0 + + for aggregation in ("overall", "simple"): + benign = attgt_weights(fit, aggregation=aggregation) + assert benign.n_negative == 0 + assert benign.negative_weight_share == 0.0 + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_target_estimand_weights_sum_to_one(self, golden, fixture): + """ATT^O and ATT^simple are proper averages of the post cells.""" + payload, df = _fixture(golden, fixture) + fit = _fit_cs(df, payload["columns"]) + for aggregation in ("overall", "simple"): + weights = attgt_weights(fit, aggregation=aggregation).weights + np.testing.assert_allclose(weights["weight"].sum(), 1.0, atol=1e-12) + + +class TestCSFitAndFrameAgree: + """The DataFrame fallback reproduces the fitted-result path exactly.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_paths_agree(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + fit = _fit_cs(df, cols) + + from_fit = attgt_weights(fit, aggregation=aggregation) + from_frame = attgt_weights( + fit.to_dataframe("group_time"), + aggregation=aggregation, + data=df, + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + left = from_fit.weights.sort_values(["group", "time"]).reset_index(drop=True) + right = from_frame.weights.sort_values(["group", "time"]).reset_index(drop=True) + np.testing.assert_allclose( + left["weight"].to_numpy(), right["weight"].to_numpy(), atol=1e-15 + ) + np.testing.assert_allclose(from_fit.implied_att, from_frame.implied_att, atol=1e-15) + + +# FWL decomposition. R double-demeans with `fixest::demean`, an iterative +# alternating-projections solver with a 1e-8 fixed-point tolerance; ours is +# the exact closed form on a balanced panel. The gap is fixest's convergence +# slack, which then propagates through the OLS projection of Ddot on Xdot. +DEMEAN_ATOL = 1e-10 +DEMEAN_COV_ATOL = 1e-8 +BALANCE_ATOL = 1e-9 + +# Cells whose comparison-group implicit weights are constant AND average to +# zero: ATT(g,t) there is a 0/0 limit. We return the limit (the unweighted +# contrast, exact); R divides the rounding errors and lands ~1e-4 away. The +# weights on such cells cancel exactly in the aggregate, so `estimate` is +# unaffected - which is why the scalar assertions below stay at 1e-10 while +# the per-cell gate is relaxed only where the degeneracy is DETECTED, never +# by hard-coding a fixture or period. +DEGENERATE_CELL_ATOL = 5e-2 + +# R names balance rows `mean_` (it averages each covariate over +# periods within unit first); we keep the covariate's own name. +R_BALANCE_COLUMNS = { + "unweighted_covs_treated": "unweighted_treated", + "unweighted_covs_comparison": "unweighted_control", + "unweighted_diff": "unweighted_diff", + "weighted_covs_treated": "weighted_treated", + "weighted_covs_comparison": "weighted_control", + "weighted_diff": "weighted_diff", + "sd": "sd", + "unweighted_log_ratio_sd_diff": "unweighted_log_ratio_sd", + "weighted_log_ratio_sd_diff": "weighted_log_ratio_sd", + "unweighted_frac_treated_extreme": "unweighted_frac_extreme", + "weighted_frac_treated_extreme": "weighted_frac_extreme", +} + + +def _decompose(df, cols, **kwargs): + return diff_diff.decompose_twfe_weights( + df, + outcome=cols["outcome"], + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + **kwargs, + ) + + +def _assert_cell_labels(cells, golden_cells): + """Row-for-row (group, time, post) agreement before any value comparison.""" + for column in ("group", "time", "post"): + np.testing.assert_array_equal( + cells[column].to_numpy(dtype=float), + np.asarray(golden_cells[column], dtype=float), + err_msg=f"cell {column!r} labels differ from the golden", + ) + + +def _degenerate_mask(cells, golden_cells): + """Rows where R's ATT(g,t) is a 0/0 artifact rather than a disagreement. + + Detected from the DATA: a degenerate cell is one whose weight is exactly + offset by another cell in the same period (they cancel in the aggregate), + which is the signature of a vanishing comparison-group normalizer. + """ + weights = np.asarray(golden_cells["weight"], dtype=float) + times = np.asarray(golden_cells["time"], dtype=float) + mask = np.zeros(len(weights), dtype=bool) + for t in np.unique(times): + in_period = times == t + if in_period.sum() > 1 and abs(weights[in_period].sum()) < 1e-12: + mask |= in_period + return mask + + +class TestDecompositionParityFWL: + """R ``implicit_twfe_weights`` parity, including the no-covariate branch.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_no_covariate_branch_two_ways(self, golden, fixture): + """``covariates=None`` and a time-invariant covariate must agree. + + The golden was generated with ``xformula = ~`` + because upstream cannot run ``~1`` (``fixest::demean`` segfaults on + the zero-column model matrix). Asserting BOTH Python calls against + that single golden proves the equivalence instead of assuming it. + """ + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = payload["decompose"]["fwl_nocov"] + + without = _decompose(df, cols, covariates=None) + with pytest.warns(UserWarning, match="no within-unit-and-period variation"): + with_invariant = _decompose(df, cols, covariates=[cols["invariant_cov"]]) + + np.testing.assert_allclose( + without.cells["weight"].to_numpy(), + with_invariant.cells["weight"].to_numpy(), + atol=1e-15, + ) + assert without.estimate == pytest.approx(with_invariant.estimate, abs=1e-15) + + for result in (without, with_invariant): + _assert_cell_labels(result.cells, expected["cells"]) + np.testing.assert_allclose(result.estimate, expected["estimate"], atol=DEMEAN_ATOL) + np.testing.assert_allclose( + result.cells["weight"].to_numpy(), + np.asarray(expected["cells"]["weight"]), + atol=DEMEAN_ATOL, + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("key", ["fwl_nocov", "fwl_cov", "fwl_gmin1"]) + def test_scalars(self, golden, fixture, key): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = payload["decompose"][key] + kwargs = { + "fwl_nocov": {"covariates": None}, + "fwl_cov": {"covariates": [cols["varying_cov"]]}, + "fwl_gmin1": {"covariates": None, "base_period": "gmin1"}, + }[key] + atol = DEMEAN_COV_ATOL if key == "fwl_cov" else DEMEAN_ATOL + + result = _decompose(df, cols, **kwargs) + + # `estimate` is invariant to the 0/0 cells - the weights on them + # cancel - so it is gated tightly on EVERY fixture. The + # decomposition/remainder SPLIT is not invariant: under gmin1 the + # remainder is itself built from the degenerate comparison-group + # weights, so R's noise moves mass between the two halves while + # leaving their sum exact. + np.testing.assert_allclose(result.estimate, expected["estimate"], atol=atol) + + _assert_cell_labels(result.cells, expected["cells"]) + degenerate = _degenerate_mask(result.cells, expected["cells"]) + # The decomposition/remainder SPLIT only moves under gmin1 (the remainder + # is built from the degenerate comparison-group weights). Under + # first_period the remainder is identically zero, so the split is gated + # tight even where the mask fires (observed gap on sim_staggered/fwl_nocov + # is 3e-15). + split_atol = DEGENERATE_CELL_ATOL if (degenerate.any() and key == "fwl_gmin1") else atol + for field in ("decomposition", "remainder"): + np.testing.assert_allclose(getattr(result, field), expected[field], atol=split_atol) + + # pretrend_bias / post_only straddle the pre/post split, so R's 0/0 noise + # at the degenerate cells (which sit on opposite sides of it on + # sim_staggered) moves each by ~1.2e-4 while their sum stays exact. + pp_atol = DEGENERATE_CELL_ATOL if degenerate.any() else atol + for field in ("pretrend_bias", "post_only"): + np.testing.assert_allclose(getattr(result, field), expected[field], atol=pp_atol) + + # effective_sample_size: post_count * sum_post(weight * ess). At the + # degenerate cells R's ess is a ratio of rounding errors (gap ~0.99 on + # sim_staggered), so the expected value is rebuilt from R's OWN cells, + # substituting our limit ess only where R's is noise - independent of + # our implementation, and not a tautology. + g_cells = expected["cells"] + r_w = np.asarray(g_cells["weight"], dtype=float) + r_ess = np.asarray(g_cells["ess"], dtype=float) + r_post = np.asarray(g_cells["post"], dtype=bool) + our_ess = result.cells["ess"].to_numpy(dtype=float) + ess_ref = np.where(degenerate, our_ess, r_ess) + expected_ess = r_post.sum() * float((r_w[r_post] * ess_ref[r_post]).sum()) + np.testing.assert_allclose(result.effective_sample_size, expected_ess, atol=1e-6) + if not degenerate.any(): + np.testing.assert_allclose( + result.effective_sample_size, expected["effective_sample_size"], atol=1e-6 + ) + + # estimate == decomposition + remainder is an identity, not a fit + assert result.estimate == pytest.approx(result.decomposition + result.remainder, abs=1e-12) + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("key", ["fwl_nocov", "fwl_cov", "fwl_gmin1"]) + def test_cells(self, golden, fixture, key): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = payload["decompose"][key] + kwargs = { + "fwl_nocov": {"covariates": None}, + "fwl_cov": {"covariates": [cols["varying_cov"]]}, + "fwl_gmin1": {"covariates": None, "base_period": "gmin1"}, + }[key] + atol = DEMEAN_COV_ATOL if key == "fwl_cov" else DEMEAN_ATOL + + result = _decompose(df, cols, **kwargs) + # Label alignment: the golden now carries ORIGINAL period labels in + # every block, so the row-for-row comparisons below are anchored rather + # than merely positional. + _assert_cell_labels(result.cells, expected["cells"]) + np.testing.assert_allclose( + result.cells["weight"].to_numpy(), + np.asarray(expected["cells"]["weight"]), + atol=atol, + ) + + actual_att = result.cells["att"].to_numpy() + golden_att = np.asarray(expected["cells"]["att"]) + degenerate = _degenerate_mask(result.cells, expected["cells"]) + np.testing.assert_allclose(actual_att[~degenerate], golden_att[~degenerate], atol=atol) + if degenerate.any(): + np.testing.assert_allclose( + actual_att[degenerate], + golden_att[degenerate], + atol=DEGENERATE_CELL_ATOL, + ) + + # Cell ess / remainder: tight where R is a valid reference; at the + # degenerate cells R divides rounding errors (cell-ess gap up to ~0.53 + # on sim_staggered), so only finiteness is asserted there. + for field in ("ess", "remainder"): + ours = result.cells[field].to_numpy(dtype=float) + theirs = np.asarray(expected["cells"][field], dtype=float) + np.testing.assert_allclose(ours[~degenerate], theirs[~degenerate], atol=atol) + assert np.isfinite(ours[degenerate]).all() + + +class TestDecompositionIsExactAtDegenerateCells: + """Where R reports 0/0 noise, we report the analytic limit.""" + + def test_limit_equals_the_unweighted_contrast(self, golden): + """sim_staggered has three equal cohorts, so E_3[D] == mean_t E_t[D]. + + The comparison-group implicit weights are then constant and average to + zero. The limit of ``resid / mean(resid)`` for a constant vector is + one, so ATT(g, 3) is the plain difference of mean outcome changes - + computable here without any of the module's machinery. + """ + payload, df = _fixture(golden, "sim_staggered") + cols = payload["columns"] + result = _decompose(df, cols, covariates=None) + + wide = df.pivot(index=cols["unit"], columns=cols["time"], values=cols["outcome"]).to_numpy() + cohorts = df.groupby(cols["unit"])[cols["first_treat"]].first().to_numpy() + change = wide[:, 2] - wide[:, 0] # base_period="first_period" + control_mean = change[cohorts == 0].mean() + + for cohort in (3, 4): + expected = change[cohorts == cohort].mean() - control_mean + actual = result.cells.query("group == @cohort and time == 3")["att"] + assert actual.iloc[0] == pytest.approx(expected, abs=1e-12) + + def test_warns_about_the_degenerate_cells(self, golden): + payload, df = _fixture(golden, "sim_staggered") + with pytest.warns(UserWarning, match="0/0 limit"): + _decompose(df, payload["columns"], covariates=None) + + +class TestBalanceParity: + """R ``twfe_cov_bal`` + ``mp_covariate_bal_summary_helper`` parity.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_cell_level(self, golden, fixture): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = pd.DataFrame(payload["balance"]["fwl"]["cells"]) + expected["covariate"] = expected["covariate"].str.replace("^mean_", "", regex=True) + + result = _decompose( + df, + cols, + covariates=[cols["varying_cov"]], + balance_covariates=[cols["invariant_cov"], cols["varying_cov"]], + ) + actual = result.covariate_balance(level="cell", standardize=False) + + key = ["group", "time", "covariate"] + expected = expected.sort_values(key).reset_index(drop=True) + actual = actual.sort_values(key).reset_index(drop=True) + assert actual["covariate"].tolist() == expected["covariate"].tolist() + for column in ("group", "time", "post"): + np.testing.assert_array_equal( + actual[column].to_numpy(dtype=float), + expected[column].to_numpy(dtype=float), + err_msg=f"balance cell {column!r} labels differ from the golden", + ) + + for r_name, our_name in R_BALANCE_COLUMNS.items(): + np.testing.assert_allclose( + actual[our_name].to_numpy(dtype=float), + expected[r_name].to_numpy(dtype=float), + atol=BALANCE_ATOL, + err_msg=f"balance column {our_name!r} ({fixture})", + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_summary_level(self, golden, fixture): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = pd.DataFrame(payload["balance"]["fwl"]["summary"]) + expected["covariate"] = expected["covariate"].str.replace("^mean_", "", regex=True) + + result = _decompose( + df, + cols, + covariates=[cols["varying_cov"]], + balance_covariates=[cols["invariant_cov"], cols["varying_cov"]], + ) + actual = result.covariate_balance(level="summary", standardize=False) + + expected = expected.sort_values("covariate").reset_index(drop=True) + actual = actual.sort_values("covariate").reset_index(drop=True) + assert actual["covariate"].tolist() == expected["covariate"].tolist() + + r_summary = { + "unweighted_treat": "unweighted_treated", + "unweighted_untreat": "unweighted_control", + "unweighted_diff": "unweighted_diff", + "weighted_treat": "weighted_treated", + "weighted_untreat": "weighted_control", + "weighted_diff": "weighted_diff", + "sd": "sd", + "unweighted_log_ratio_sd_diff": "unweighted_log_ratio_sd", + "weighted_log_ratio_sd_diff": "weighted_log_ratio_sd", + "unweighted_frac_treated_extreme": "unweighted_frac_extreme", + "weighted_frac_treated_extreme": "weighted_frac_extreme", + } + for r_name, our_name in r_summary.items(): + np.testing.assert_allclose( + actual[our_name].to_numpy(dtype=float), + expected[r_name].to_numpy(dtype=float), + atol=BALANCE_ATOL, + err_msg=f"balance summary {our_name!r} ({fixture})", + ) + + +class TestCrossSurfaceIdentity: + """attgt_weights and decompose_twfe_weights describe the same regression.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_twfe_weights_reproduce_the_decomposition(self, golden, fixture): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + + weighted = attgt_weights(_fit_cs(df, cols), aggregation="twfe") + decomposed = _decompose(df, cols, covariates=None) + + assert weighted.implied_att == pytest.approx(decomposed.estimate, abs=1e-6) diff --git a/tests/test_visualization_plotly.py b/tests/test_visualization_plotly.py index d3c4d774b..f8fb915f4 100644 --- a/tests/test_visualization_plotly.py +++ b/tests/test_visualization_plotly.py @@ -662,3 +662,63 @@ def test_band_labels(self): results.alpha = 0.025 fig_r_frac = plot_dose_response(results, backend="plotly", show=False) assert self._band_traces(fig_r_frac)[0].name == "97.5% CI" + + +class TestPlotlyTWFEWeights: + """Plotly backend for plot_twfe_weights (both views).""" + + @staticmethod + def _panel_and_fit(): + import pandas as pd + + import diff_diff + + rng = np.random.RandomState(11) + first_treat = np.repeat(np.array([0, 3, 4]), 30) + rows = [] + for t in range(1, 6): + treated = (first_treat != 0) & (t >= first_treat) + rows.append( + pd.DataFrame( + { + "unit": np.arange(len(first_treat)), + "period": t, + "first_treat": first_treat, + "outcome": rng.normal(size=len(first_treat)) + treated * 1.0, + "x": rng.normal(size=len(first_treat)), + } + ) + ) + df = pd.concat(rows, ignore_index=True) + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit(df, outcome="outcome", unit="unit", time="period", first_treat="first_treat") + return df, fit + + def test_weights_view(self): + import diff_diff + + _, fit = self._panel_and_fit() + fig = diff_diff.plot_twfe_weights( + diff_diff.attgt_weights(fit), backend="plotly", show=False + ) + assert isinstance(fig, go.Figure) + assert len(fig.data) == 2 # post + pre traces + assert len(fig.layout.shapes) >= 2 # zero lines + + def test_balance_view(self): + import diff_diff + + df, _ = self._panel_and_fit() + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + fig = diff_diff.plot_twfe_weights(dec, backend="plotly", show=False) + assert isinstance(fig, go.Figure) + assert any(trace.name == "no improvement" for trace in fig.data)