From 384a61da5a57cb65dc12c52ea84f8f31cbcdb3e4 Mon Sep 17 00:00:00 2001 From: Ryan James Date: Mon, 26 Oct 2026 01:12:32 -0600 Subject: [PATCH 1/5] Accept dit's discrete entropy estimators for layers and selection infer_multiplex(estimator=...) and the functions below it accept any of dit.inference.estimators.ENTROPY_ESTIMATORS (intrinsic flow evaluated at the hardened channel, since they need integer counts), and SkeletonSettings(entropy_estimator=...) swaps the entropies in the plug-in selection statistic. Defaults are unchanged. dit is moved to e3611818 for the new estimators. Validation scripts and report: Chao-Wang-Jost lowers layer RMSE 25-30% against Miller-Madow at 300-1000 samples and spurious synergy on iEEG 2.5x; Miller-Madow selection raises recall by 0.05 at no cost. --- docs/layers.rst | 15 +++ docs/references.bib | 33 ++++++ docs/skeleton.rst | 12 ++ infoflow/crossfit.py | 9 +- infoflow/layers.py | 13 ++- infoflow/measures.py | 57 ++++++++-- infoflow/network.py | 6 +- infoflow/selection.py | 39 +++++-- reports/dit_estimators.md | 111 ++++++++++++++++++ tests/test_measures.py | 23 ++++ tests/test_selection.py | 16 +++ uv.lock | 4 +- validation/dit_estimators.py | 115 +++++++++++++++++++ validation/dit_estimators_catfr.py | 150 +++++++++++++++++++++++++ validation/dit_estimators_selection.py | 52 +++++++++ 15 files changed, 628 insertions(+), 27 deletions(-) create mode 100644 reports/dit_estimators.md create mode 100644 validation/dit_estimators.py create mode 100644 validation/dit_estimators_catfr.py create mode 100644 validation/dit_estimators_selection.py diff --git a/docs/layers.rst b/docs/layers.rst index e3e7d72..ae9543c 100644 --- a/docs/layers.rst +++ b/docs/layers.rst @@ -26,6 +26,21 @@ otherwise, as CSSR merges histories :cite:`Shalizi2004`), the channel is fit on one fold and evaluated with the Miller--Madow estimator :cite:`Miller1955` on the other, and the folds are swapped and averaged. +``infer_multiplex(estimator=...)`` (and :func:`~infoflow.crossfit.crossfit_flows`) +also accepts dit's other discrete entropy estimators, applied to every entropy term +of the evaluation table; the intrinsic flow is then evaluated at the channel hardened +to its most likely output per context, since these estimators need integer counts. +On the sofic processes with exact flows (``validation/dit_estimators.py``; copy, XOR, +shared, and mixed edges with up to 54 contexts, 20 replicates), the Chao--Wang--Jost +estimator :cite:`Chao2013` lowered the cross-fitted layer RMSE against Miller--Madow +from 0.047 to 0.035 bits at 300 samples and from 0.019 to 0.014 at 1000 (0.007 for +both at 3000), and the synergistic layer's bias at 54 contexts from 0.081 to 0.033 +bits; NSB :cite:`Nemenman2002` was slightly more accurate still (0.030, 0.013) but +costs thousands of times more per entropy. On real iEEG with the source shuffled +(every true layer zero), Miller--Madow left 0.0010 bits of spurious synergy per edge, +Chao--Wang--Jost 0.0004 and NSB 0.0002. Shrinkage, Laplace, Bonachela and the +asymptotic NSB estimators were worse than Miller--Madow. + Statistics ---------- diff --git a/docs/references.bib b/docs/references.bib index dbe5788..0268bb0 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -1070,3 +1070,36 @@ @article{Grigorescu2024 year = {2024}, doi = {10.1109/TIT.2024.3411919} } + +@article{Chao2013, + author = {Chao, Anne and Wang, Y. T. and Jost, Lou}, + title = {Entropy and the species accumulation curve: a novel entropy estimator via discovery rates of new species}, + journal = {Methods in Ecology and Evolution}, + volume = {4}, + number = {11}, + pages = {1091--1100}, + year = {2013}, + doi = {10.1111/2041-210X.12108} +} + +@inproceedings{Nemenman2002, + author = {Nemenman, Ilya and Shafee, Fariel and Bialek, William}, + title = {{Entropy and inference, revisited}}, + booktitle = {Advances in Neural Information Processing Systems 14}, + pages = {471--478}, + publisher = {MIT Press}, + year = {2002}, + eprint = {physics/0108025}, + archivePrefix = {arXiv} +} + +@article{Zhang2012, + author = {Zhang, Zhiyi}, + title = {Entropy estimation in {T}uring's perspective}, + journal = {Neural Computation}, + volume = {24}, + number = {5}, + pages = {1368--1389}, + year = {2012}, + doi = {10.1162/NECO_a_00266} +} diff --git a/docs/skeleton.rst b/docs/skeleton.rst index 602b5d7..30386db 100644 --- a/docs/skeleton.rst +++ b/docs/skeleton.rst @@ -25,6 +25,18 @@ shuffles when there are trials, otherwise circular, block, or local permutations Selection and estimation can use disjoint halves of the data (``holdout``) to avoid post-selection bias in the layer estimates. +``SkeletonSettings(entropy_estimator=...)`` computes the plug-in estimator's +conditional mutual information with any of dit's discrete entropy estimators instead. +The permutation tests stay valid; only their power changes. On random VAR and +coupled-logistic networks (``validation/dit_estimators_selection.py``; 10 nodes, 500 +and 2000 samples, five networks each, the same networks for every estimator), +Miller--Madow raised recall by 0.05 over the plug-in on both kinds (better on 7 of +10 networks, worse on none) at the same cost and without more false positives on +VAR; Chao--Wang--Jost :cite:`Chao2013` raised it most (0.06 on VAR, 0.13 on +logistic maps) at two to nine times the cost, with 0.4 more false positives per +logistic network and a lower edge-score AUROC; Zhang :cite:`Zhang2012` matched +Miller--Madow at two to eight times the cost. + For continuous data, ``SkeletonSettings(estimator="ksg")`` selects parents with the KSG conditional mutual information on the raw values :cite:`Kraskov2004,Frenzel2007`, as IDTxl does. Conditioning on a parent continuously removes its influence where bins diff --git a/infoflow/crossfit.py b/infoflow/crossfit.py index b7c4a76..4e4e645 100644 --- a/infoflow/crossfit.py +++ b/infoflow/crossfit.py @@ -16,9 +16,9 @@ from .context import apply_merge, merge_contexts_statistical from .measures import ( EdgeFlow, - _degrade, cmi_from_joint, dense_codes, + evaluation_table, intrinsic_flow, joint_table, layers_from_values, @@ -42,7 +42,7 @@ def _evaluate(table_b, mapping, Q, estimator): n = merged_b.sum() te = cmi_from_joint(table_b, estimator) tdmi = cmi_from_joint(table_b.sum(axis=2, keepdims=True), estimator) - iif = cmi_from_joint(_degrade(merged_b, Q), estimator, n=n) + iif = cmi_from_joint(evaluation_table(merged_b, Q, estimator), estimator, n=n) return te, tdmi, min(iif, te, tdmi) @@ -66,8 +66,9 @@ def crossfit_flows( y, x, w : array_like The target present, source past, and context of each aligned realization (1D codes or 2D rows of joint values); `w` may be None. - estimator : {'miller_madow', 'plugin'} - The evaluation estimator on the held-out fold. + estimator : str + The evaluation estimator on the held-out fold (see + :func:`~infoflow.measures.cmi_from_joint`). merge : bool Merge statistically indistinguishable context values before fitting. merge_alpha : float diff --git a/infoflow/layers.py b/infoflow/layers.py index 15fd8c6..6a45f73 100644 --- a/infoflow/layers.py +++ b/infoflow/layers.py @@ -33,7 +33,16 @@ from .context import apply_merge, merge_contexts_statistical from .crossfit import crossfit_flows -from .measures import EdgeFlow, _degrade, cmi_from_joint, dense_codes, intrinsic_flow, joint_table, layers_from_values +from .measures import ( + EdgeFlow, + _degrade, + cmi_from_joint, + dense_codes, + evaluation_table, + intrinsic_flow, + joint_table, + layers_from_values, +) __all__ = ( "BAYES_OPTIONS", @@ -90,7 +99,7 @@ def _layers_on_table(table, mapping, n_clusters, estimator, initial, prng): _, Q = intrinsic_flow(merged, initial=initial, prng=prng) te = cmi_from_joint(table, estimator) tdmi = cmi_from_joint(table.sum(axis=2, keepdims=True), estimator) - iif = min(cmi_from_joint(_degrade(merged, Q), estimator, n=n), te, tdmi) + iif = min(cmi_from_joint(evaluation_table(merged, Q, estimator), estimator, n=n), te, tdmi) te, tdmi, iif, syn, shared, _ = layers_from_values(te, tdmi, iif) return {"te": te, "tdmi": tdmi, "intrinsic": iif, "synergistic": syn, "shared": shared} diff --git a/infoflow/measures.py b/infoflow/measures.py index fd6699d..05fce6f 100644 --- a/infoflow/measures.py +++ b/infoflow/measures.py @@ -68,6 +68,19 @@ def _entropy_terms(q): return float(-np.sum(q * np.log(q))) +def _dit_entropy(counts, estimator, K): + """ + Entropy in bits of a count array with a dit estimator and `K` possible outcomes. + """ + from dit.inference.estimators import ENTROPY_ESTIMATORS + + if estimator not in ENTROPY_ESTIMATORS: + raise ValueError(f"Unknown estimator {estimator!r}; choose from {sorted(ENTROPY_ESTIMATORS)}.") + counts = counts[counts > 0] + n = counts.sum() + return float(ENTROPY_ESTIMATORS[estimator](counts, n, K)) / _LOG2 if n > 0 else 0.0 + + def cmi_from_joint(p, estimator="plugin", n=None): """ :math:`I[Y : X \\mid W]` in bits from a table ``p[y, x, w]`` (counts or probabilities). @@ -77,10 +90,14 @@ def cmi_from_joint(p, estimator="plugin", n=None): p : np.ndarray Joint table over (y, x, w). With a single context value this is :math:`I[Y : X]`. - estimator : {'plugin', 'miller_madow'} - ``'miller_madow'`` adds each entropy term's :math:`(K - 1) / 2N` bias - correction :cite:`Miller1955`, with :math:`K` the number of occupied - cells. It needs the sample size `n` (taken from ``p.sum()`` for counts). + estimator : str + ``'plugin'``; ``'miller_madow'``, which adds each entropy term's + :math:`(K - 1) / 2N` bias correction :cite:`Miller1955`, with :math:`K` the + number of occupied cells (it needs the sample size `n`, taken from ``p.sum()`` + for counts); or any other of dit's discrete entropy estimators + (:data:`dit.inference.estimators.ENTROPY_ESTIMATORS`), applied to each entropy + term of the count table with the product of its axes' sizes as the alphabet. + Those need integer counts: pass counts, or probabilities with `n`. n : float, None The number of samples behind a probability table. """ @@ -88,17 +105,37 @@ def cmi_from_joint(p, estimator="plugin", n=None): if total <= 0: return 0.0 n = total if n is None else n + if estimator not in ("plugin", "miller_madow"): + c = np.rint(p * (n / total)) + Ky, Kx, Kw = p.shape + return ( + _dit_entropy(c.sum(axis=0).ravel(), estimator, Kx * Kw) + + _dit_entropy(c.sum(axis=1).ravel(), estimator, Ky * Kw) + - _dit_entropy(c.ravel(), estimator, Ky * Kx * Kw) + - _dit_entropy(c.sum(axis=(0, 1)), estimator, Kw) + ) q = p / total q_xw, q_yw, q_w = q.sum(axis=0), q.sum(axis=1), q.sum(axis=(0, 1)) value = _entropy_terms(q_xw) + _entropy_terms(q_yw) - _entropy_terms(q) - _entropy_terms(q_w) if estimator == "miller_madow": k = lambda a: np.count_nonzero(a > 1e-12) # noqa: E731 value += ((k(q_xw) - 1) + (k(q_yw) - 1) - (k(q) - 1) - (k(q_w) - 1)) / (2 * n) - elif estimator != "plugin": - raise ValueError(f"Unknown estimator {estimator!r}; use 'plugin' or 'miller_madow'.") return value / _LOG2 +def evaluation_table(table, Q, estimator): + """ + The table on which `estimator` evaluates the intrinsic flow at channel `Q`. + + Plug-in and Miller--Madow evaluate the degraded table itself; dit's other + estimators need integer counts, so the channel is hardened to its most likely + output per context (the minimizing channel is usually close to deterministic). + """ + if estimator in ("plugin", "miller_madow"): + return _degrade(table, Q) + return _degrade(table, np.eye(Q.shape[1])[Q.argmax(axis=1)]) + + def _degrade(p, Q): """ ``q[y, x, z] = sum_w p[y, x, w] Q[w, z]``. @@ -268,8 +305,10 @@ def edge_flows(y, x, w, estimator="plugin", bound=None, restarts=3, prng=None): The source's past (1D codes or a 2D array whose rows are joint values). w : array_like or None The context (1D codes or 2D rows); None for an empty context. - estimator : {'plugin', 'miller_madow'} - Applied to TE, TDMI, and the intrinsic flow at the minimizing channel. + estimator : str + ``'plugin'``, ``'miller_madow'``, or another of dit's discrete entropy + estimators (:func:`cmi_from_joint`), applied to TE, TDMI, and the intrinsic + flow at the minimizing channel (:func:`evaluation_table`). bound, restarts, prng Passed to :func:`intrinsic_flow`. @@ -285,7 +324,7 @@ def edge_flows(y, x, w, estimator="plugin", bound=None, restarts=3, prng=None): iif_plugin, Q = intrinsic_flow(table, bound=bound, restarts=restarts, prng=prng) te = cmi_from_joint(table, estimator) tdmi = cmi_from_joint(table.sum(axis=2, keepdims=True), estimator) - iif = cmi_from_joint(_degrade(table, Q), estimator, n=n) + iif = cmi_from_joint(evaluation_table(table, Q, estimator), estimator, n=n) te, tdmi, iif, syn, shared, clipped = layers_from_values(te, tdmi, iif) raw = { "te": cmi_from_joint(table), diff --git a/infoflow/network.py b/infoflow/network.py index 2718a1a..87530f4 100644 --- a/infoflow/network.py +++ b/infoflow/network.py @@ -310,8 +310,10 @@ def infer_multiplex( preprocess : 'auto', dict, or None ``'auto'`` preprocesses float data with :func:`~infoflow.preprocess.preprocess`; a dict passes options to it; None treats values as symbols. - estimator : {'miller_madow', 'plugin'} - Evaluation estimator for the flow layers. + estimator : str + Evaluation estimator for the flow layers: ``'miller_madow'``, ``'plugin'``, or + another of dit's discrete entropy estimators (e.g. ``'chao_wang_jost'``; see + :func:`~infoflow.measures.cmi_from_joint`). alpha : float FDR level for the layer tests. fdr_dependent : bool diff --git a/infoflow/selection.py b/infoflow/selection.py index 08d3f19..469e3f1 100644 --- a/infoflow/selection.py +++ b/infoflow/selection.py @@ -127,6 +127,13 @@ class SkeletonSettings: null standard deviation). Raw estimates are not comparable across targets, whose conditioning sets and plug-in biases differ; the z-scores are. With an ensemble, each source's score is the largest over the estimators. + + ``entropy_estimator`` replaces the plug-in entropies inside the ``'plugin'`` + estimator's conditional mutual information with any of dit's discrete entropy + estimators (:data:`dit.inference.estimators.ENTROPY_ESTIMATORS`, e.g. + ``'miller_madow'``, ``'zhang'``, ``'chao_wang_jost'``, ``'shrink'``); joint terms + get the product of their variables' alphabet sizes. The permutation tests stay valid + for any statistic; the estimator only changes their power. """ n_perm_max_stat: int = 200 @@ -173,6 +180,7 @@ class SkeletonSettings: n_perm_prescreen: int = 100 edge_scores: bool = False n_perm_scores: int = 20 + entropy_estimator: str = "plugin" device: str | None = None def check(self): @@ -249,21 +257,33 @@ def _combine(a, Ka, b, Kb): return a * Kb + b, Ka * Kb -def _entropy(code, K): +def _entropy(code, K, estimator="plugin"): + """ + Entropy in nats of integer codes with `K` possible values. + """ counts = np.bincount(code, minlength=0) if K <= 10_000_000 else np.unique(code, return_counts=True)[1] counts = counts[counts > 0].astype(float) n = counts.sum() - return float(np.log(n) - np.sum(counts * np.log(counts)) / n) + if estimator == "plugin": + return float(np.log(n) - np.sum(counts * np.log(counts)) / n) + from dit.inference.estimators import ENTROPY_ESTIMATORS + return float(ENTROPY_ESTIMATORS[estimator](counts, n, K)) -def _cmi(y, Ky, c, Kc, z, Kz): + +def _cmi(y, Ky, c, Kc, z, Kz, estimator="plugin"): """ - Plug-in I[y : c | z] in bits from integer codes. + I[y : c | z] in bits from integer codes, with the plug-in or a dit entropy estimator. """ cz, Kcz = _combine(c, Kc, z, Kz) yz, Kyz = _combine(y, Ky, z, Kz) ycz, Kycz = _combine(y, Ky, cz, Kcz) - value = _entropy(cz, Kcz) + _entropy(yz, Kyz) - _entropy(ycz, Kycz) - _entropy(z, Kz) + value = ( + _entropy(cz, Kcz, estimator) + + _entropy(yz, Kyz, estimator) + - _entropy(ycz, Kycz, estimator) + - _entropy(z, Kz, estimator) + ) return max(value, 0.0) / np.log(2) @@ -272,6 +292,8 @@ class _Columns: Integer columns of one target's realizations, with joint encodings. """ + entropy_estimator = "plugin" + def __init__(self, data, target, variables, max_lag): self.r = realizations(data, target, variables, max_lag=max_lag) self.variables = list(self.r.variables) @@ -301,15 +323,15 @@ def context(self, variables): def cmi(self, c, Kc, z, Kz): """ - Plug-in :math:`I[Y : c \\mid z]` in bits. + :math:`I[Y : c \\mid z]` in bits (plug-in unless :attr:`entropy_estimator` is set). """ - return _cmi(self.y, self.Ky, c, Kc, z, Kz) + return _cmi(self.y, self.Ky, c, Kc, z, Kz, self.entropy_estimator) def cmi_batch(self, c, Kc, z, Kz, idx): """ :meth:`cmi` of ``c[i]`` for every permutation ``i`` in `idx`, batched on :attr:`device` if set. """ - if self.device is not None: + if self.device is not None and self.entropy_estimator == "plugin": from .backend import plugin_cmi_batch return plugin_cmi_batch(self.y, self.Ky, np.asarray(c)[np.asarray(idx)], Kc, z, Kz, self.device) @@ -1255,6 +1277,7 @@ def _make_columns(name, data, target, everything, max_lag, settings, rng): cols = _CoarseColumns(data, target, everything, max_lag, min_cell=settings.coarse_min_cell) else: cols = _Columns(data, target, everything, max_lag) + cols.entropy_estimator = settings.entropy_estimator if settings.device is not None: from .backend import resolve_device diff --git a/reports/dit_estimators.md b/reports/dit_estimators.md new file mode 100644 index 0000000..d53af60 --- /dev/null +++ b/reports/dit_estimators.md @@ -0,0 +1,111 @@ +# dit's new entropy estimators in infoflow + +*Branch `dit-estimators`, October 2026. dit `e3611818` (bias-corrected estimators, dit +PR #295). Scripts: `validation/dit_estimators.py` (layers, exact flows), +`validation/dit_estimators_selection.py` (selection, known graphs), +`validation/dit_estimators_catfr.py` (catFR1).* + +## Summary + +- **Layers: yes.** Chao-Wang-Jost (CWJ) cut the cross-fitted layer RMSE by 25-30% against + the current Miller-Madow default at 300-1000 samples, mostly by removing synergistic + bias in large contexts; on catFR it cut spurious synergy 2.5x. NSB and jackknife are + as good or better but too slow for routine use. +- **Selection: modestly.** Miller-Madow in the selection statistic (already in dit, now + selectable) raised recall by 0.05 at no cost on both synthetic network types. CWJ + raised recall most (0.06 VAR, 0.13 logistic maps) at 2-9x the cost, with a few more + false positives on logistic maps and a slightly worse edge-score ranking. On catFR the + selection estimator made almost no difference. +- **Nothing changes by default.** Both are opt-in: `infer_multiplex(estimator="chao_wang_jost")` + and `SkeletonSettings(entropy_estimator="miller_madow")`. + +## What was added + +- `measures.cmi_from_joint` and everything above it (`edge_flows`, `crossfit_flows`, + layer nulls, `infer_multiplex(estimator=...)`) accept any of + `dit.inference.estimators.ENTROPY_ESTIMATORS`, applied to each entropy term of the + count table with the product of its axes' sizes as the alphabet. The intrinsic flow + is evaluated at the channel hardened to its most likely output per context + (`measures.evaluation_table`), since these estimators need integer counts; + plug-in and Miller-Madow keep the soft channel. +- `SkeletonSettings(entropy_estimator=...)` swaps the entropies inside the plug-in + selection statistic. The permutation tests stay valid for any statistic. +- `uv.lock` moves dit to `e3611818`. Tests: each estimator's layer CMI equals dit's + `conditional_mutual_information`, layer identities hold, selection recovers a chain. + +## 1. Layer accuracy (exact flows) + +Sofic copy, XOR, shared and mixed edges (history 1) with 0-3 independent ternary +nuisance variables added to the context (2-54 contexts; true flows unchanged), 20 +replicates. RMSE in bits, averaged over processes, context sizes and layers: + +| Estimator | Cross-fit n=300 | 1000 | 3000 | Single joint n=300 | 1000 | 3000 | Cost per entropy | +|---|---|---|---|---|---|---|---| +| plugin | 0.044 | 0.024 | 0.010 | 0.043 | 0.018 | 0.008 | 1x | +| miller_madow (default) | 0.047 | 0.019 | 0.007 | 0.052 | 0.019 | 0.008 | 1x | +| grassberger | 0.044 | 0.015 | 0.007 | 0.040 | 0.015 | 0.007 | 2x | +| zhang | 0.046 | 0.018 | 0.007 | 0.040 | 0.014 | 0.007 | 18x | +| **chao_wang_jost** | **0.035** | **0.014** | **0.007** | 0.035 | 0.014 | 0.007 | 13x | +| nsb | 0.030 | 0.013 | 0.007 | 0.033 | 0.014 | 0.007 | ~2500x | +| jackknife | 0.044 | 0.014 | 0.007 | 0.038 | 0.014 | 0.007 | 180x | +| bayes_minimax | 0.031 | 0.018 | 0.009 | 0.030 | 0.014 | 0.008 | 1x | +| shrink, bayes_laplace, bonachela, ansb | 0.05-0.09 | | | | | | | + +The gain is in the synergistic layer: at 54 contexts, CWJ's synergy bias was 0.033 bits +at 300 samples against Miller-Madow's 0.081 (cross-fit), and its synergy RMSE over all +settings 0.024 against 0.038. Intrinsic and shared flow are about equally accurate +under the better estimators. + +## 2. Selection accuracy (known graphs) + +`network_validation`, 10-node random networks simulated as a VAR or coupled logistic +maps, 500 and 2000 samples, five networks each, identical networks for every estimator; +the layer estimator stays Miller-Madow. Paired differences against the plug-in over the +10 networks per kind (better / worse count in parentheses): + +| Estimator | VAR recall | VAR precision | logistic recall | logistic precision | logistic edge-score AUROC | Runtime vs plug-in | +|---|---|---|---|---|---|---| +| miller_madow | +0.053 (7/0) | +0.168 (6/0) | +0.053 (7/0) | +0.037 (5/2) | -0.001 | 1.0-1.1x | +| zhang | +0.053 (7/0) | +0.168 (6/0) | +0.060 (7/0) | +0.038 (5/2) | -0.002 | 2.2-7.5x | +| chao_wang_jost | +0.064 (9/0) | +0.150 (7/1) | +0.125 (9/1) | +0.002 (4/3) | -0.037 (2/8) | 1.4-9.4x | +| grassberger | +0.007 (2/1) | +0.055 (6/3) | +0.082 (7/1) | +0.031 (3/5) | -0.041 (1/8) | 1.0-1.8x | +| bayes_minimax | +0.025 (5/0) | +0.089 (4/1) | +0.013 (3/0) | -0.017 (1/1) | -0.002 | 1.0-1.5x | + +At 2000 samples on logistic maps, recall went from 0.49 (plug-in) to 0.56 (Miller-Madow) +and 0.69 (CWJ) at precision 0.94-0.95; CWJ also found 0.4 more false edges per network. +NSB and jackknife were not run: at 100 ms and 7 ms per CMI they are too slow for +thousands of permutations per candidate. + +## 3. catFR1 (no ground truth) + +**Bias on real data.** For the 25 cohort subjects and 6 region pairs (region-averaged +HFA, 4 bins, the cohort's fixed variables, all trials), the layers with the source +shuffled across samples (every true layer zero, so the estimate is bias) against the real +data, mean bits: + +| Estimator | Shuffled: TE / intrinsic / synergistic / shared | Real: TE / intrinsic / synergistic / shared | +|---|---|---| +| plugin | 0.0182 / 0.0003 / 0.0179 / 0.0000 | 0.0191 / 0.0013 / 0.0178 / 0.0015 | +| miller_madow | 0.0011 / 0.0000 / 0.0010 / 0.0000 | 0.0019 / 0.0001 / 0.0018 / 0.0023 | +| chao_wang_jost | 0.0005 / 0.0000 / 0.0004 / 0.0000 | 0.0011 / 0.0004 / 0.0008 / 0.0021 | +| nsb | 0.0002 / 0.0000 / 0.0002 / 0.0001 | 0.0007 / 0.0003 / 0.0004 / 0.0021 | +| jackknife | 0.0002 / 0.0000 / 0.0002 / 0.0000 | 0.0008 / 0.0003 / 0.0005 / 0.0022 | +| bayes_minimax | 0.0176 / 0.0003 / 0.0173 / 0.0000 | 0.0184 / 0.0019 / 0.0165 / 0.0008 | + +Half of the synergy Miller-Madow reports here is bias; with CWJ, NSB or jackknife it +shrinks to a few ten-thousandths of a bit, while shared flow (about 0.002 bits) is the +same under every corrected estimator. Bayes-minimax, good on the synthetic processes, +is nearly as biased as the plug-in here (large sparse contexts). + +**Selection.** Edges selected on the pilot's HFA epochs (18 channels, all 300 trials): +R1065J 6 / 6 / 6, R1332M 2 / 2 / 2, R1364C 6 / 8 / 6 (plug-in / Miller-Madow / CWJ); +none with any estimator on surrogates whose channels' trials are permuted independently. +CWJ took 1.3-2.6x as long. + +## Recommendation + +Use `estimator="chao_wang_jost"` for layer estimates when contexts are large relative to +the data (it is the best accuracy for its cost here), and +`SkeletonSettings(entropy_estimator="miller_madow")` for selection (a free gain in +recall and precision on both synthetic network types). Making them defaults would +change every existing result, so that is left as a separate decision. diff --git a/tests/test_measures.py b/tests/test_measures.py index db543be..10a4513 100644 --- a/tests/test_measures.py +++ b/tests/test_measures.py @@ -118,6 +118,29 @@ def test_edge_flows_single_joint(): assert edge_flows(y, x, None, prng=0).synergistic == pytest.approx(0, abs=1e-9) +DIT_ESTIMATORS = [ + e for e in ("grassberger", "chao_shen", "zhang", "chao_wang_jost", "shrink", "jackknife") + if e in __import__("dit.inference.estimators", fromlist=["ENTROPY_ESTIMATORS"]).ENTROPY_ESTIMATORS +] # fmt: skip + + +@pytest.mark.parametrize("estimator", DIT_ESTIMATORS) +def test_dit_estimators_in_layers(estimator): + from dit.inference import conditional_mutual_information + + rng = np.random.default_rng(4) + y, x, w = rng.integers(0, 2, 500), rng.integers(0, 3, 500), rng.integers(0, 4, 500) + table = np.zeros((2, 3, 4)) + np.add.at(table, (y, x, w), 1.0) + expected = conditional_mutual_information(y, x, w, estimator=estimator, alphabet_sizes=(2, 3, 4)) + assert cmi_from_joint(table, estimator) == pytest.approx(expected, abs=1e-9) + flow = edge_flows(*_samples("copy", 2000, np.random.default_rng(5)), estimator=estimator, prng=0) + assert flow.te == pytest.approx(flow.intrinsic + flow.synergistic) + assert flow.tdmi == pytest.approx(flow.intrinsic + flow.shared) + crossfit = crossfit_flows(*_samples("copy", 2000, np.random.default_rng(5)), estimator=estimator, prng=0) + assert crossfit.intrinsic > 0.3 and crossfit.synergistic < 0.05 + + def test_intrinsic_flow_guards_oversized_channels(): p = np.random.default_rng(0).random((2, 3, 40)) te, tdmi = cmi_from_joint(p), cmi_from_joint(p.sum(axis=2, keepdims=True)) diff --git a/tests/test_selection.py b/tests/test_selection.py index a028822..b0b9ce0 100644 --- a/tests/test_selection.py +++ b/tests/test_selection.py @@ -38,6 +38,22 @@ def test_chain_recovers_parents_and_lags(): assert result[2].parents() == [1] and result[2].sources == [(1, 2)] +@pytest.mark.parametrize("estimator", ["miller_madow", "grassberger", "chao_shen"]) +def test_entropy_estimator_in_selection(estimator): + from dit.inference import conditional_mutual_information + + from infoflow.selection import _cmi + + rng = np.random.default_rng(1) + y, c, z = rng.integers(0, 2, 800), rng.integers(0, 3, 800), rng.integers(0, 4, 800) + expected = conditional_mutual_information(y, c, z, estimator=estimator, alphabet_sizes=(2, 3, 4)) + assert _cmi(y, 2, c, 3, z, 4, estimator) == pytest.approx(max(expected, 0.0), abs=1e-9) + data = DiscreteData.from_discrete(_chain(3000, np.random.default_rng(0))) + settings = SkeletonSettings(**FAST, entropy_estimator=estimator) + result = infer_skeleton(data, Embedding(max_lag=3), settings=settings, prng=0) + assert [result[t].parents() for t in range(3)] == [[], [0], [1]] + + def test_xor_needs_pair_search(): rng = np.random.default_rng(1) n = 3000 diff --git a/uv.lock b/uv.lock index 1c71878..7cce921 100644 --- a/uv.lock +++ b/uv.lock @@ -724,8 +724,8 @@ wheels = [ [[package]] name = "dit" -version = "2.4.dev3+g42f1a9d6c" -source = { git = "https://github.com/dit/dit?branch=main#42f1a9d6c0fe0337454ffd04696e122cd046718b" } +version = "2.4.dev41+ge3611818a" +source = { git = "https://github.com/dit/dit?branch=main#e3611818a5eed2187ee91183e1bd03306b7074fa" } dependencies = [ { name = "boltons" }, { name = "debtcollector" }, diff --git a/validation/dit_estimators.py b/validation/dit_estimators.py new file mode 100644 index 0000000..deeb648 --- /dev/null +++ b/validation/dit_estimators.py @@ -0,0 +1,115 @@ +""" +Layer accuracy of dit's discrete entropy estimators on processes with exact flows. + +The sofic generators of ``tests/_generators.py`` give the exact transfer entropy and +intrinsic, synergistic and shared flow of an edge (history 1). Each sample adds ``d`` +independent ternary nuisance variables to the context, which leaves every true flow +unchanged while multiplying the number of contexts by ``3**d``, so the estimators' +bias shows up as the context grows. Both the single-joint estimate (``edge_flows``) +and the cross-fitted one ``infer_multiplex`` uses (``crossfit_flows``) are scored. + + python validation/dit_estimators.py --reps 20 --out layer_accuracy.nc +""" + +import argparse +import sys +import warnings +import zlib +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +import numpy as np +import xarray as xr + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) + +ESTIMATORS = ( + "plugin", "miller_madow", "grassberger", "chao_shen", "nsb", "zhang", "chao_wang_jost", "shrink", + "bayes_jeffreys", "bayes_laplace", "bayes_perks", "bayes_minimax", "ansb", "bonachela", "good_turing", + "jackknife", +) # fmt: skip +LAYERS = ("te", "intrinsic", "synergistic", "shared") +PROCESSES = { + "copy": dict(a_copy=0.6), + "xor": dict(a_xor=0.6), + "shared": dict(a_hold=0.5, x_given_y=0.9), + "mixed": dict(a_copy=0.3, a_xor=0.3, a_hold=0.2, x_given_y=0.8), +} + + +def truths(): + from _generators import exact_flows, mixture + + return {name: exact_flows(mixture(**kw)) for name, kw in PROCESSES.items()} + + +def one(args): + from _generators import mixture, sample_pairs + + from infoflow.crossfit import crossfit_flows + from infoflow.measures import edge_flows + + name, n, d, rep = args + seed = zlib.crc32(f"{name}-{n}-{d}-{rep}".encode()) + pairs = sample_pairs(mixture(**PROCESSES[name]), n + 1, seed) + x, y = pairs[:-1, 0], pairs[1:, 1] + rng = np.random.default_rng(seed + 1) + w = np.column_stack([pairs[:-1, 1], rng.integers(0, 3, size=(n, d))]) + out = np.full((2, len(ESTIMATORS), len(LAYERS)), np.nan) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + for e, est in enumerate(ESTIMATORS): + for m, fn in enumerate((edge_flows, crossfit_flows)): + try: + flow = fn(y, x, w, estimator=est, prng=0) + except (ValueError, ZeroDivisionError, FloatingPointError): + continue + values = flow.as_dict() + out[m, e] = [values[k] for k in LAYERS] + return args, out + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--reps", type=int, default=20) + parser.add_argument("--n", type=int, nargs="+", default=[300, 1000, 3000]) + parser.add_argument("--d", type=int, nargs="+", default=[0, 1, 2, 3]) + parser.add_argument("--workers", type=int, default=8) + parser.add_argument("--out", default="layer_accuracy.nc") + args = parser.parse_args() + + exact = truths() + jobs = [(p, n, d, r) for p in PROCESSES for n in args.n for d in args.d for r in range(args.reps)] + shape = (len(PROCESSES), len(args.n), len(args.d), args.reps, 2, len(ESTIMATORS), len(LAYERS)) + values = np.full(shape, np.nan) + with ProcessPoolExecutor(args.workers) as pool: + for (p, n, d, r), out in pool.map(one, jobs, chunksize=4): + values[list(PROCESSES).index(p), args.n.index(n), args.d.index(d), r] = out + truth = np.array([[exact[p][k] for k in LAYERS] for p in PROCESSES]) + ds = xr.Dataset( + { + "estimate": (("process", "n", "d", "rep", "path", "estimator", "layer"), values), + "truth": (("process", "layer"), truth), + }, + coords={ + "process": list(PROCESSES), "n": args.n, "d": args.d, "rep": range(args.reps), + "path": ["single", "crossfit"], "estimator": list(ESTIMATORS), "layer": list(LAYERS), + }, + ) # fmt: skip + ds.to_netcdf(args.out) + summarize(ds) + + +def summarize(ds): + error = ds["estimate"] - ds["truth"] + rmse = np.sqrt((error**2).mean("rep")) + for path in ("single", "crossfit"): + table = rmse.sel(path=path).mean(("process", "d", "layer")) + print(f"\n{path}: RMSE (bits), mean over processes, context sizes and layers") + print(f"{'estimator':16s}" + "".join(f" n={n:<6d}" for n in ds.n.values)) + for est in ds.estimator.values: + print(f"{est:16s}" + "".join(f" {float(v):.4f} " for v in table.sel(estimator=est).values)) + + +if __name__ == "__main__": + main() diff --git a/validation/dit_estimators_catfr.py b/validation/dit_estimators_catfr.py new file mode 100644 index 0000000..4217c9d --- /dev/null +++ b/validation/dit_estimators_catfr.py @@ -0,0 +1,150 @@ +""" +dit's entropy estimators on the catFR1 data (no ground truth). + +``bias``: for each cohort subject and directed region pair, the layers with the +cohort's fixed variables (``examples/catfr/cohort.py``) on the real realizations and +on realizations whose source is shuffled across all samples. The shuffled source is +independent of the target and context with its marginal intact, so every true layer +is zero there and the estimate is the estimator's bias. + +``selection``: edges selected on the pilot's correctly aligned HFA epochs (all trials, +pooled) with each selection estimator, on the real data and on surrogates whose +channels' trials are permuted independently (inter-channel coupling within a trial +destroyed; each channel's dynamics and the stimulus-locked mean kept). + + python validation/dit_estimators_catfr.py bias --cohort ~/data/catfr_cohort + python validation/dit_estimators_catfr.py selection --pilot ~/data/catfr_pilot_v3 +""" + +import argparse +import json +import sys +import time +import warnings +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "examples" / "catfr")) + +LAYER_ESTIMATORS = ( + "plugin", + "miller_madow", + "grassberger", + "nsb", + "zhang", + "chao_wang_jost", + "bayes_minimax", + "jackknife", +) +SELECTION_ESTIMATORS = ("plugin", "miller_madow", "grassberger", "zhang", "chao_wang_jost", "bayes_minimax") +LAYERS = ("te", "intrinsic", "synergistic", "shared") + + +def bias(cohort_dir, out): + import os + + os.environ["CATFR_OUT"] = str(cohort_dir) + from cohort import COHORT, fixed_network, region_trials + from common import OUT, ROIS + + from infoflow.compare import _union_edges + from infoflow.data import realizations + from infoflow.measures import edge_flows + from infoflow.preprocess import EqualFrequency, preprocess + + summary = json.loads((COHORT / "summary.json").read_text()) + edges = _union_edges(fixed_network(summary["source_lag"]), fixed_network(summary["source_lag"])) + rng = np.random.default_rng(0) + rows = [] + for subject in summary["subjects"]: + trials, _ = region_trials(OUT / f"sub-{subject['subject']}_ses-{subject['session']}_hfa.nc") + data = preprocess(trials, names=list(ROIS), max_lag=15, discretizer=EqualFrequency(4), prng=0).data + for (s, t), (x_vars, w_vars) in edges.items(): + r = realizations(data, t, list(dict.fromkeys(x_vars + w_vars)), max_lag=max(v[1] for v in x_vars + w_vars)) + x, w = r.columns(list(x_vars)), r.columns(list(w_vars)) + shuffled = x[rng.permutation(len(x))] + for est in LAYER_ESTIMATORS: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + real = edge_flows(r.present, x, w, estimator=est, prng=0).as_dict() + null = edge_flows(r.present, shuffled, w, estimator=est, prng=0).as_dict() + rows.append( + { + "subject": subject["subject"], + "edge": f"{ROIS[s]}->{ROIS[t]}", + "estimator": est, + "n": len(x), + "real": real, + "shuffled": null, + } + ) + print(subject["subject"], flush=True) + Path(out).write_text(json.dumps(rows)) + print(f"\nmean over {len(summary['subjects'])} subjects x 6 edges (bits): shuffled source (= bias) | real") + for est in LAYER_ESTIMATORS: + sel = [r for r in rows if r["estimator"] == est] + b = {k: np.mean([r["shuffled"][k] for r in sel]) for k in LAYERS} + v = {k: np.mean([r["real"][k] for r in sel]) for k in LAYERS} + print( + f"{est:15s} " + + " ".join(f"{k[:5]} {b[k]:.4f}" for k in LAYERS) + + " | " + + " ".join(f"{k[:5]} {v[k]:.4f}" for k in LAYERS) + ) + + +def selection(pilot_dir, out, estimators, pattern): + import xarray as xr + + from infoflow.auto import _skeleton_settings + from infoflow.parallel import thread_map + from infoflow.preprocess import preprocess + from infoflow.selection import infer_skeleton + + rows = [] + for path in sorted(Path(pilot_dir).glob(pattern)): + ds = xr.open_dataset(path).load() + names = [f"{r}:{c}" for r, c in zip(ds.region.values, ds.channel.values, strict=True)] + x = ds.signal.values.astype(float) + rng = np.random.default_rng(0) + surrogate = np.stack([x[rng.permutation(len(x)), :, p] for p in range(x.shape[2])], axis=2) + for label, values in (("real", x), ("surrogate", surrogate)): + pre = preprocess( + list(values), names=names, max_lag=15, network_resolution="auto", map_fn=thread_map(4), prng=0 + ) + for est in estimators: + settings = _skeleton_settings(("plugin",), False, 1, None, {"entropy_estimator": est}) + start = time.perf_counter() + skeleton = infer_skeleton(pre.data, pre.embeddings, settings=settings, prng=0, map_fn=thread_map(4)) + found = [(p, t) for t, sk in skeleton.items() if sk.significant for p in sk.parents()] + rows.append( + { + "file": path.name, + "data": label, + "estimator": est, + "edges": len(found), + "seconds": time.perf_counter() - start, + } + ) + print(rows[-1], flush=True) + Path(out).write_text(json.dumps(rows)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("part", choices=("bias", "selection")) + parser.add_argument("--cohort", default=str(Path.home() / "data" / "catfr_cohort")) + parser.add_argument("--pilot", default=str(Path.home() / "data" / "catfr_pilot_v3")) + parser.add_argument("--out", default=None) + parser.add_argument("--estimators", nargs="+", default=list(SELECTION_ESTIMATORS)) + parser.add_argument("--files", default="sub-*_hfa.nc") + args = parser.parse_args() + if args.part == "bias": + bias(args.cohort, args.out or "catfr_bias.json") + else: + selection(args.pilot, args.out or "catfr_selection.json", args.estimators, args.files) + + +if __name__ == "__main__": + main() diff --git a/validation/dit_estimators_selection.py b/validation/dit_estimators_selection.py new file mode 100644 index 0000000..4ec0f16 --- /dev/null +++ b/validation/dit_estimators_selection.py @@ -0,0 +1,52 @@ +""" +Parent-selection accuracy with dit's discrete entropy estimators. + +Runs :func:`~infoflow.benchmarks.network_validation` (random networks of known +structure, simulated as a VAR or coupled logistic maps) with the plug-in selection +statistic replaced by one of dit's estimators (``SkeletonSettings.entropy_estimator``). +Every estimator sees the same graphs and simulations (same ``--seed``); the layer +estimator stays Miller--Madow, so differences come from selection. + + python validation/dit_estimators_selection.py --kind var --estimator zhang --out-dir results/ +""" + +import argparse +import time +from pathlib import Path + +from infoflow.benchmarks import network_validation +from infoflow.selection import SkeletonSettings + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--kind", choices=("var", "logistic"), required=True) + parser.add_argument("--estimator", required=True) + parser.add_argument("--nodes", type=int, default=10) + parser.add_argument("--samples", type=int, nargs="+", default=[500, 2000]) + parser.add_argument("--reps", type=int, default=5) + parser.add_argument("--max-lag", type=int, default=3) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--out-dir", default=".") + args = parser.parse_args() + + start = time.perf_counter() + ds = network_validation( + kind=args.kind, + n_nodes=(args.nodes,), + n_samples=tuple(args.samples), + n_reps=args.reps, + max_lag=args.max_lag, + levels=("skeleton", "intrinsic"), + infer_kwargs={"skeleton_settings": SkeletonSettings(entropy_estimator=args.estimator)}, + prng=args.seed, + ) + ds = ds.expand_dims(estimator=[args.estimator]) + ds.attrs["seconds"] = time.perf_counter() - start + out = Path(args.out_dir) / f"selection_{args.kind}_{args.estimator}.nc" + ds.to_netcdf(out) + print(out, round(ds.attrs["seconds"]), "s", flush=True) + + +if __name__ == "__main__": + main() From 5d7cf8187faa1c429ce8602e9d70a2db0fffbcdd Mon Sep 17 00:00:00 2001 From: Ryan James Date: Mon, 26 Oct 2026 13:16:15 -0600 Subject: [PATCH 2/5] Benchmark the proposed estimator defaults together --- reports/dit_estimators.md | 35 +++++++++-- validation/new_defaults.py | 121 +++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 validation/new_defaults.py diff --git a/reports/dit_estimators.md b/reports/dit_estimators.md index d53af60..f738e80 100644 --- a/reports/dit_estimators.md +++ b/reports/dit_estimators.md @@ -102,10 +102,35 @@ R1065J 6 / 6 / 6, R1332M 2 / 2 / 2, R1364C 6 / 8 / 6 (plug-in / Miller-Madow / C none with any estimator on surrogates whose channels' trials are permuted independently. CWJ took 1.3-2.6x as long. +## 4. Both changes together on other benchmarks + +`validation/new_defaults.py`: *current* (plug-in selection, Miller-Madow layers) against +*proposed* (Miller-Madow selection, CWJ layers), same seeds and data. Paired over +networks or instances (better / worse counts in parentheses). + +| Benchmark | Metric | Current | Proposed | +|---|---|---|---| +| VAR, 10 nodes, 500 / 2000 samples | edge recall | 0.12 / 0.33 | 0.17 / 0.39 (4/0, 3/0) | +| | edge precision | 0.59 / 0.84 | 0.89 / 0.87 | +| | intrinsic-layer recall (2000) | 0.29 | 0.39 (4/0), precision 0.98 -> 0.97 | +| Logistic maps, 500 / 2000 | edge recall | 0.20 / 0.49 | 0.24 / 0.56 (3/0, 4/0) | +| | intrinsic-layer recall (2000) | 0.35 | 0.54 (5/0), precision 1.00 -> 1.00 | +| Lorenz-96, 10 nodes, 1000 | edge recall / precision | 0.28 / 0.91 | 0.33 / 0.98 (4/0) | +| | intrinsic-layer recall | 0.04 | 0.33 (5/0), precision 0.98 | +| Motifs, 20 replications | six layer and role checks | 1.00, 1.00, 1.00, 1.00, 1.00, 0.95 | 1.00, 1.00, 1.00, 1.00, 0.95, 0.90 (chain checks) | +| NetSim, sims 1-4 x 5 subjects, `auto` | AUROC | 0.625 | 0.627 (9/11) | +| CausalRivers, 20 three-station graphs, `auto` | AUROC / recall | 0.581 / 0.50 | 0.525 / 0.45 (2/6) | + +Synthetic networks run 50% longer. On CausalRivers the whole loss comes from the +selection change: Miller-Madow selection alone gives the same 0.525, CWJ layers alone +give exactly the current results. NetSim is unchanged because `auto` finds almost no +edges there either way (recall 0.001) and its scores barely move. + ## Recommendation -Use `estimator="chao_wang_jost"` for layer estimates when contexts are large relative to -the data (it is the best accuracy for its cost here), and -`SkeletonSettings(entropy_estimator="miller_madow")` for selection (a free gain in -recall and precision on both synthetic network types). Making them defaults would -change every existing result, so that is left as a separate decision. +CWJ layers are a safe default: they improved every synthetic benchmark (most of all the +intrinsic layer's recall) and changed nothing on NetSim or CausalRivers. Miller-Madow +selection helps the synthetic networks but costs CausalRivers 0.06 AUROC, the one +real-data benchmark with known structure, so it should stay an option rather than the +default. Making either a default changes existing results, so that is left as a +separate decision. diff --git a/validation/new_defaults.py b/validation/new_defaults.py new file mode 100644 index 0000000..a6fc6c0 --- /dev/null +++ b/validation/new_defaults.py @@ -0,0 +1,121 @@ +""" +Current against proposed estimator defaults on infoflow's benchmarks. + +``current``: plug-in selection statistic, Miller--Madow layers (the defaults). +``proposed``: Miller--Madow selection statistic (``SkeletonSettings(entropy_estimator= +"miller_madow")``), Chao--Wang--Jost layers (``estimator="chao_wang_jost"``). + +Each benchmark runs both with the same seeds and data: + +* ``synthetic``: :func:`~infoflow.benchmarks.network_validation` on VAR, + coupled-logistic and Lorenz-96 networks (``infer_multiplex``); +* ``motifs``: :func:`~infoflow.benchmarks.motif_validation` (layers and roles); +* ``netsim``, ``rivers``: ``auto.infer`` (fast preset) on NetSim and CausalRivers, as + ``validation/external.py`` does. + + python validation/new_defaults.py synthetic --kind lorenz96 --config proposed --out-dir results/ +""" + +import argparse +import time +from pathlib import Path + +import numpy as np +import xarray as xr + +CONFIGS = { + "current": {"selection": "plugin", "layers": "miller_madow"}, + "proposed": {"selection": "miller_madow", "layers": "chao_wang_jost"}, + "selection_only": {"selection": "miller_madow", "layers": "miller_madow"}, + "layers_only": {"selection": "plugin", "layers": "chao_wang_jost"}, +} + + +def infer_kwargs(config): + from infoflow.selection import SkeletonSettings + + c = CONFIGS[config] + return {"skeleton_settings": SkeletonSettings(entropy_estimator=c["selection"]), "estimator": c["layers"]} + + +def synthetic(kind, config, reps, seed): + from infoflow.benchmarks import network_validation + + samples = (1000,) if kind == "lorenz96" else (500, 2000) + return network_validation( + kind=kind, n_nodes=(10,), n_samples=samples, n_reps=reps, max_lag=3, + levels=("skeleton", "intrinsic"), infer_kwargs=infer_kwargs(config), prng=seed, + ) # fmt: skip + + +def motifs(config, reps, seed): + from infoflow.benchmarks import motif_validation + + return motif_validation(n_samples=3000, n_reps=reps, infer_kwargs=infer_kwargs(config), prng=seed) + + +def external(bench, config, data_dir, seed, sims, subjects, sizes, graphs): + import sys + + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from external import METRICS, netsim_instances, rivers_instances, undirected_auroc + + from infoflow import auto + from infoflow.benchmarks import score_edges + + c = CONFIGS[config] + if bench == "netsim": + instances = netsim_instances(data_dir, sims, subjects) + else: + instances = rivers_instances(data_dir, sizes, graphs, [24], seed) + rng = np.random.default_rng(seed) + labels, rows = [], [] + for label, data, truth in instances: + s = int(rng.integers(2**32)) + t0 = time.perf_counter() + net = auto.infer( + data, preset="fast", edge_scores=True, entropy_estimator=c["selection"], layer_estimator=c["layers"], + prng=s, + ).network # fmt: skip + elapsed = time.perf_counter() - t0 + found = {(p, t) for t, sk in net.skeleton.items() if sk.significant for p in sk.parents()} + score = net.dataset["score"].values + r = score_edges(found, None, score, truth) + rows.append([r["auroc"], r["auprc"], undirected_auroc(score, truth), r["precision"], r["recall"], elapsed]) + labels.append(label) + print(label, round(r["auroc"], 3), flush=True) + values = np.array(rows) + return xr.Dataset({m: (("instance",), values[:, i]) for i, m in enumerate(METRICS)}, coords={"instance": labels}) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("bench", choices=("synthetic", "motifs", "netsim", "rivers")) + parser.add_argument("--config", choices=tuple(CONFIGS), required=True) + parser.add_argument("--kind", choices=("var", "logistic", "lorenz96"), default="var") + parser.add_argument("--reps", type=int, default=5) + parser.add_argument("--data", default=None) + parser.add_argument("--sims", type=int, nargs="+", default=[1, 2, 3, 4]) + parser.add_argument("--subjects", type=int, default=5) + parser.add_argument("--sizes", type=int, nargs="+", default=[3]) + parser.add_argument("--graphs", type=int, default=20) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--out-dir", default=".") + args = parser.parse_args() + + start = time.perf_counter() + if args.bench == "synthetic": + ds, name = synthetic(args.kind, args.config, args.reps, args.seed), f"synthetic_{args.kind}" + elif args.bench == "motifs": + ds, name = motifs(args.config, args.reps, args.seed), "motifs" + else: + ds = external(args.bench, args.config, args.data, args.seed, args.sims, args.subjects, args.sizes, args.graphs) + name = args.bench + ds.attrs["seconds"] = time.perf_counter() - start + out = Path(args.out_dir) / f"{name}_{args.config}.nc" + ds.to_netcdf(out) + print(out, round(ds.attrs["seconds"]), "s", flush=True) + + +if __name__ == "__main__": + main() From 4127f168a29a6dbf87a7562a6b9015401b48b02f Mon Sep 17 00:00:00 2001 From: Ryan James Date: Mon, 26 Oct 2026 14:59:28 -0600 Subject: [PATCH 3/5] Make Chao-Wang-Jost the default layer estimator DEFAULT_LAYER_ESTIMATOR = 'chao_wang_jost' for infer_multiplex, estimate_edge, layer_statistics, crossfit_flows, the channel layer, and auto's cost pilot. The selection statistic stays plug-in and compare_networks keeps Miller-Madow. On VAR, logistic-map and Lorenz-96 networks it raised intrinsic-layer recall at unchanged precision; NetSim and CausalRivers were unchanged. The uncoupled false-positive test now counts runs with any false intrinsic or synergistic flow. --- docs/channels.rst | 2 +- docs/layers.rst | 20 ++++++++++++++------ infoflow/auto.py | 9 ++++++--- infoflow/channels.py | 6 +++--- infoflow/crossfit.py | 3 ++- infoflow/layers.py | 3 ++- infoflow/measures.py | 4 ++++ infoflow/network.py | 5 +++-- reports/dit_estimators.md | 14 +++++++++++--- tests/test_network.py | 2 +- tests/test_validation_pipeline.py | 9 ++++----- 11 files changed, 51 insertions(+), 26 deletions(-) diff --git a/docs/channels.rst b/docs/channels.rst index 3315a87..447416c 100644 --- a/docs/channels.rst +++ b/docs/channels.rst @@ -23,7 +23,7 @@ Measures * ``mi`` is :math:`I[X : Y]` at the observed input distribution: the information the edge carries as the source actually drives it. It uses the network's estimator - (Miller--Madow by default :cite:`Miller1955`), or + (Chao--Wang--Jost by default :cite:`Chao2013`), or ``ChannelSettings(estimator=...)``. * ``capacity`` is :math:`C = \max_{p(x)} I[X : Y]` of the estimated channel, computed by the Blahut--Arimoto algorithm :cite:`Blahut1972,Arimoto1972` diff --git a/docs/layers.rst b/docs/layers.rst index ae9543c..4030c50 100644 --- a/docs/layers.rst +++ b/docs/layers.rst @@ -23,13 +23,15 @@ sampling noise. :func:`infoflow.crossfit.crossfit_flows` therefore separates the two in the style of double machine learning :cite:`Chernozhukov2018`: contexts are merged (exactly when their predictive distributions coincide, statistically otherwise, as CSSR merges histories :cite:`Shalizi2004`), the channel is fit on -one fold and evaluated with the Miller--Madow estimator :cite:`Miller1955` on the -other, and the folds are swapped and averaged. +one fold and evaluated on the other with the Chao--Wang--Jost entropy estimator +:cite:`Chao2013`, and the folds are swapped and averaged. ``infer_multiplex(estimator=...)`` (and :func:`~infoflow.crossfit.crossfit_flows`) -also accepts dit's other discrete entropy estimators, applied to every entropy term -of the evaluation table; the intrinsic flow is then evaluated at the channel hardened -to its most likely output per context, since these estimators need integer counts. +takes any of dit's discrete entropy estimators, applied to every entropy term of the +evaluation table (``'miller_madow'`` :cite:`Miller1955` was the default before +Chao--Wang--Jost). Except for the plug-in and Miller--Madow, the intrinsic flow is +evaluated at the channel hardened to its most likely output per context, since these +estimators need integer counts. On the sofic processes with exact flows (``validation/dit_estimators.py``; copy, XOR, shared, and mixed edges with up to 54 contexts, 20 replicates), the Chao--Wang--Jost estimator :cite:`Chao2013` lowered the cross-fitted layer RMSE against Miller--Madow @@ -39,7 +41,13 @@ bits; NSB :cite:`Nemenman2002` was slightly more accurate still (0.030, 0.013) b costs thousands of times more per entropy. On real iEEG with the source shuffled (every true layer zero), Miller--Madow left 0.0010 bits of spurious synergy per edge, Chao--Wang--Jost 0.0004 and NSB 0.0002. Shrinkage, Laplace, Bonachela and the -asymptotic NSB estimators were worse than Miller--Madow. +asymptotic NSB estimators were worse than Miller--Madow. Against Miller--Madow +layers, Chao--Wang--Jost raised the recall of significant intrinsic flow on random +VAR, logistic-map and Lorenz-96 networks at unchanged precision and left NetSim and +CausalRivers unchanged (``validation/new_defaults.py``; ``reports/dit_estimators.md``). +It costs about 13 times as much per entropy as Miller--Madow; whole networks took +about 1.5 times as long. :func:`~infoflow.compare.compare_networks` still evaluates +with Miller--Madow. Statistics ---------- diff --git a/infoflow/auto.py b/infoflow/auto.py index 21c151c..3635275 100644 --- a/infoflow/auto.py +++ b/infoflow/auto.py @@ -37,6 +37,7 @@ from .checkpoint import Checkpoint from .data import DiscreteData, as_trials from .embedding import Embedding +from .measures import DEFAULT_LAYER_ESTIMATOR from .network import estimate_edge, infer_multiplex from .parallel import thread_map from .selection import SkeletonSettings, select_parents @@ -512,7 +513,7 @@ def _ksg_seconds(data, embeddings, target, settings, cheap): return per_estimate * evaluations -def _edge_seconds(data, skeletons, n_resamples): +def _edge_seconds(data, skeletons, n_resamples, estimator): """ Measured seconds per edge of layer estimation at `n_resamples`, from one pilot parent. """ @@ -522,7 +523,7 @@ def _edge_seconds(data, skeletons, n_resamples): w = base + [v for v in sk.sources if v[0] != p] trial = 50 start = time.perf_counter() - estimate_edge(data, sk.target, sk.source_variables(p), w, "miller_madow", trial, trial, prng=0) + estimate_edge(data, sk.target, sk.source_variables(p), w, estimator, trial, trial, prng=0) return (time.perf_counter() - start) * n_resamples / trial return 0.0 @@ -719,7 +720,9 @@ def pilot_run(t): "prescreen": cheap["prescreen"], "parents_per_target": parents, "shared_per_target": shared, - "edge_seconds": _edge_seconds(discrete, pilot_skeletons, n_resamples), + "edge_seconds": _edge_seconds( + discrete, pilot_skeletons, n_resamples, pinned_infer.get("estimator", DEFAULT_LAYER_ESTIMATOR) + ), } # Preprocessing, the linearity test, and the pilot are already spent. elapsed = time.perf_counter() - started diff --git a/infoflow/channels.py b/infoflow/channels.py index f2f1edc..c915820 100644 --- a/infoflow/channels.py +++ b/infoflow/channels.py @@ -26,7 +26,7 @@ import xarray as xr from .machines import _aligned, _as_data -from .measures import cmi_from_joint +from .measures import DEFAULT_LAYER_ESTIMATOR, cmi_from_joint __all__ = ("ChannelLayer", "ChannelSettings", "edge_channel", "infer_channels") @@ -38,7 +38,7 @@ class ChannelSettings: Attributes ---------- - estimator : {'miller_madow', 'plugin'}, None + estimator : str, None Estimator of the mutual information (None: the network's). The capacity is always the plug-in capacity of the estimated channel. include_shared_candidates : bool @@ -151,7 +151,7 @@ def infer_channels(data, network, settings=None): settings = ChannelSettings() elif isinstance(settings, dict): settings = ChannelSettings(**settings) - estimator = settings.estimator or network.settings.get("estimator", "miller_madow") + estimator = settings.estimator or network.settings.get("estimator", DEFAULT_LAYER_ESTIMATOR) names = network.names data = _as_data(data, names) P = data.n_processes diff --git a/infoflow/crossfit.py b/infoflow/crossfit.py index 4e4e645..9610bc1 100644 --- a/infoflow/crossfit.py +++ b/infoflow/crossfit.py @@ -15,6 +15,7 @@ from .context import apply_merge, merge_contexts_statistical from .measures import ( + DEFAULT_LAYER_ESTIMATOR, EdgeFlow, cmi_from_joint, dense_codes, @@ -50,7 +51,7 @@ def crossfit_flows( y, x, w, - estimator="miller_madow", + estimator=DEFAULT_LAYER_ESTIMATOR, merge=True, merge_alpha=0.01, gap=0, diff --git a/infoflow/layers.py b/infoflow/layers.py index 6a45f73..e496984 100644 --- a/infoflow/layers.py +++ b/infoflow/layers.py @@ -34,6 +34,7 @@ from .context import apply_merge, merge_contexts_statistical from .crossfit import crossfit_flows from .measures import ( + DEFAULT_LAYER_ESTIMATOR, EdgeFlow, _degrade, cmi_from_joint, @@ -141,7 +142,7 @@ def layer_statistics( n_null=100, confidence=0.95, mean_block_length=None, - estimator="miller_madow", + estimator=DEFAULT_LAYER_ESTIMATOR, merge=True, merge_alpha=0.01, gap=0, diff --git a/infoflow/measures.py b/infoflow/measures.py index 05fce6f..6f3d4a2 100644 --- a/infoflow/measures.py +++ b/infoflow/measures.py @@ -33,6 +33,10 @@ _LOG2 = np.log(2) +#: Estimator of the flow layers unless one is given: least layer error for its cost in +#: ``validation/dit_estimators.py`` and ``validation/new_defaults.py`` :cite:`Chao2013`. +DEFAULT_LAYER_ESTIMATOR = "chao_wang_jost" + def dense_codes(values): """ diff --git a/infoflow/network.py b/infoflow/network.py index 87530f4..f9096f7 100644 --- a/infoflow/network.py +++ b/infoflow/network.py @@ -13,6 +13,7 @@ from .data import DiscreteData, realizations from .embedding import Embedding from .layers import BAYES_THRESHOLD, LAYERS, adjust_layer_pvalues, layer_statistics +from .measures import DEFAULT_LAYER_ESTIMATOR from .selection import SkeletonSettings, _cmi, _Columns, infer_skeleton if TYPE_CHECKING: @@ -181,7 +182,7 @@ def estimate_edge( target, x_vars, w_vars, - estimator="miller_madow", + estimator=DEFAULT_LAYER_ESTIMATOR, n_boot=100, n_null=100, prng=None, @@ -261,7 +262,7 @@ def infer_multiplex( embeddings=None, max_lag=None, preprocess="auto", - estimator="miller_madow", + estimator=DEFAULT_LAYER_ESTIMATOR, alpha=0.05, fdr_dependent=False, n_boot=100, diff --git a/reports/dit_estimators.md b/reports/dit_estimators.md index f738e80..fab12ea 100644 --- a/reports/dit_estimators.md +++ b/reports/dit_estimators.md @@ -131,6 +131,14 @@ edges there either way (recall 0.001) and its scores barely move. CWJ layers are a safe default: they improved every synthetic benchmark (most of all the intrinsic layer's recall) and changed nothing on NetSim or CausalRivers. Miller-Madow selection helps the synthetic networks but costs CausalRivers 0.06 AUROC, the one -real-data benchmark with known structure, so it should stay an option rather than the -default. Making either a default changes existing results, so that is left as a -separate decision. +real-data benchmark with known structure, so it stays an option. + +**Adopted:** the layer estimator now defaults to CWJ +(`measures.DEFAULT_LAYER_ESTIMATOR`, used by `infer_multiplex`, `estimate_edge`, +`layer_statistics`, `crossfit_flows`, the channel layer and `auto`'s cost pilot); the +selection statistic stays plug-in, and `compare_networks` still evaluates with +Miller-Madow. On 40 pairs of independent AR(1) processes (23 edges tested), the false +layers were 3 intrinsic, 5 synergistic and 5 shared with CWJ against 1, 5 and 13 with +Miller-Madow, and 4 runs in each had a false intrinsic or synergistic layer; the +uncoupled test now counts runs with any false flow (at most one of eight), since on one +seed CWJ marks both layers of the edge that Miller-Madow marks once. diff --git a/tests/test_network.py b/tests/test_network.py index e8b1be2..53ecd67 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -76,7 +76,7 @@ def test_result_objects(): assert ("a", "b", "intrinsic") in g.edges(keys=True) assert net.layer("intrinsic").has_edge("a", "b") assert "a" in net.summary() - assert net.settings["estimator"] == "miller_madow" + assert net.settings["estimator"] == "chao_wang_jost" te = ds["weight"].sel(layer="te").values parts = ds["weight"].sel(layer="intrinsic").values + ds["weight"].sel(layer="synergistic").values mask = np.isfinite(te) diff --git a/tests/test_validation_pipeline.py b/tests/test_validation_pipeline.py index af26b60..2a8a452 100644 --- a/tests/test_validation_pipeline.py +++ b/tests/test_validation_pipeline.py @@ -56,16 +56,15 @@ def test_uncoupled_false_positive_rate(): """ Independent autocorrelated processes: almost no significant flow in any layer. """ - false = 0 - runs = 8 - for seed in range(runs): + runs_with_false_flow = 0 + for seed in range(8): rng = np.random.default_rng(seed) x = np.zeros((1500, 2)) for t in range(1, 1500): x[t] = 0.8 * x[t - 1] + rng.normal(size=2) net = infer_multiplex(x, skeleton_settings=FAST, n_boot=40, n_null=40, prng=seed) - false += int(_sig(net, "intrinsic").sum() + _sig(net, "synergistic").sum()) - assert false <= 1 + runs_with_false_flow += int(_sig(net, "intrinsic").sum() + _sig(net, "synergistic").sum() > 0) + assert runs_with_false_flow <= 1 def test_discrete_sofic_generator_matches_exact_and_delay(): From fa31789a6fd1359103fddb7889ff300e28fc3067 Mon Sep 17 00:00:00 2001 From: Ryan James Date: Sun, 1 Nov 2026 08:35:26 -0700 Subject: [PATCH 4/5] Evaluate dit's second estimator batch; guard single-outcome terms Panzeri-Treves, quadratic extrapolation, Wu-Yang, BUB, unseen and PYM for layers and selection, and the Gaussian copula via copula_normalize before auto.infer: none beats the current defaults (report section 5). Single-outcome entropy terms now return 0 for dit estimators (PYM rejects them). dit moves to 6314d14e, which renamed prng to rng; total_correlation_ksg is now called with rng. --- infoflow/benchmarks.py | 2 +- infoflow/measures.py | 6 +- infoflow/selection.py | 2 + infoflow/stats.py | 4 +- reports/dit_estimators.md | 59 +++++++++++++++++ tests/test_measures.py | 11 ++++ uv.lock | 4 +- validation/dit_estimators.py | 16 +++-- validation/dit_estimators_catfr.py | 9 +-- validation/new_defaults.py | 101 +++++++++++++++++++++++------ 10 files changed, 176 insertions(+), 38 deletions(-) diff --git a/infoflow/benchmarks.py b/infoflow/benchmarks.py index 1a9dabf..b0926e9 100644 --- a/infoflow/benchmarks.py +++ b/infoflow/benchmarks.py @@ -137,7 +137,7 @@ def _estimate(x, y, z, estimator, kind, bins, k, rng): data = np.column_stack([x, y, z]).astype(float) crvs = list(range(2, data.shape[1])) or None - return float(total_correlation_ksg(data, [[0], [1]], crvs, k=k, prng=rng)) + return float(total_correlation_ksg(data, [[0], [1]], crvs, k=k, rng=rng)) if kind == "gaussian": x, y, z = _bin(x, bins), _bin(y, bins), _bin(z, bins) if z.shape[1] else z yc, _ = dense_codes(y) diff --git a/infoflow/measures.py b/infoflow/measures.py index 6f3d4a2..03e963f 100644 --- a/infoflow/measures.py +++ b/infoflow/measures.py @@ -81,8 +81,10 @@ def _dit_entropy(counts, estimator, K): if estimator not in ENTROPY_ESTIMATORS: raise ValueError(f"Unknown estimator {estimator!r}; choose from {sorted(ENTROPY_ESTIMATORS)}.") counts = counts[counts > 0] - n = counts.sum() - return float(ENTROPY_ESTIMATORS[estimator](counts, n, K)) / _LOG2 if n > 0 else 0.0 + if len(counts) <= 1: + # One observed outcome (e.g. a context with a single value); some estimators reject it. + return 0.0 + return float(ENTROPY_ESTIMATORS[estimator](counts, counts.sum(), K)) / _LOG2 def cmi_from_joint(p, estimator="plugin", n=None): diff --git a/infoflow/selection.py b/infoflow/selection.py index 469e3f1..0707441 100644 --- a/infoflow/selection.py +++ b/infoflow/selection.py @@ -266,6 +266,8 @@ def _entropy(code, K, estimator="plugin"): n = counts.sum() if estimator == "plugin": return float(np.log(n) - np.sum(counts * np.log(counts)) / n) + if len(counts) <= 1: + return 0.0 from dit.inference.estimators import ENTROPY_ESTIMATORS return float(ENTROPY_ESTIMATORS[estimator](counts, n, K)) diff --git a/infoflow/stats.py b/infoflow/stats.py index 7510521..a3468b5 100644 --- a/infoflow/stats.py +++ b/infoflow/stats.py @@ -258,11 +258,11 @@ def conditional_mutual_information_test_knn( data = np.asarray(data, dtype=np.float64) x_cols, y_cols = (list(r) for r in rvs) crvs = [] if crvs is None else list(crvs) - value = total_correlation_ksg(data, [x_cols, y_cols], crvs, k=k, noise=noise, prng=rng) + value = total_correlation_ksg(data, [x_cols, y_cols], crvs, k=k, noise=noise, rng=rng) z = data[:, crvs] null = np.empty(n_surrogates) for i in range(n_surrogates): shuffled = data.copy() shuffled[:, x_cols] = _local_permutation(data[:, x_cols], z, k_perm, rng) - null[i] = total_correlation_ksg(shuffled, [x_cols, y_cols], crvs, k=k, noise=noise, prng=rng) + null[i] = total_correlation_ksg(shuffled, [x_cols, y_cols], crvs, k=k, noise=noise, rng=rng) return _result(value, null, len(data)) diff --git a/reports/dit_estimators.md b/reports/dit_estimators.md index fab12ea..3d650c1 100644 --- a/reports/dit_estimators.md +++ b/reports/dit_estimators.md @@ -126,6 +126,65 @@ selection change: Miller-Madow selection alone gives the same 0.525, CWJ layers give exactly the current results. NetSim is unchanged because `auto` finds almost no edges there either way (recall 0.001) and its scores barely move. +## 5. dit's second batch (dit PR #305) + +Six more discrete estimators (Panzeri-Treves, quadratic extrapolation, Wu-Yang, BUB, +unseen, PYM) and a Gaussian-copula mutual information, against the current defaults +(CWJ layers, plug-in selection). None is a clear improvement. + +**Cost per CMI** (2000 samples, 1024 cells): Panzeri-Treves 1.1 ms, Wu-Yang 2.3, +quadratic extrapolation 3.5, BUB 7.6, unseen 20, PYM 42 (CWJ 0.5, plug-in 0.03). + +**Layers** (section 1's exact-flow screen, cross-fitted RMSE in bits at 300 / 1000 / 3000 +samples; "better cells": of the 48 process x size x context cells, how many beat CWJ): + +| Estimator | RMSE | Better cells | catFR synergy bias (section 3) | +|---|---|---|---| +| chao_wang_jost (default) | 0.035 / 0.014 / 0.007 | | 0.0004 | +| panzeri_treves | 0.028 / 0.013 / 0.007 | 19 / 48 | 0.0006 | +| nsb | 0.030 / 0.013 / 0.007 | 22 / 48 | 0.0002 | +| quadratic_extrapolation | 0.048 / 0.015 / 0.007 | 18 / 48 | 0.0002 | +| wu_yang | 0.039 / 0.015 / 0.007 | | 0.0012 | +| pym | 0.042 / 0.016 / 0.008 | 12 / 48 | 0.0000, but real TE 0.0002 | +| bub | 0.052 / 0.018 / 0.007 | | 0.0010 | +| unseen | 0.079 / 0.046 / 0.036 | | 0.0000, but real TE 0.0000 | + +Panzeri-Treves has the lowest mean RMSE, from the smallest, most context-heavy settings +(its synergy bias at 54 contexts and 300 samples is 0.019 bits against CWJ's 0.057), but +it is worse than CWJ in most cells and on the catFR bias probe. Quadratic extrapolation +matches NSB on catFR but is worse than CWJ on the synthetic processes. Unseen and PYM +remove the bias by suppressing the flow itself: on the catFR data their transfer entropy +is 0.0000 and 0.0002 bits against CWJ's 0.0011, and they add bias to shared flow. + +**Selection** (section 2's networks, paired against the plug-in over 10 networks): + +| Estimator | VAR recall | VAR precision | VAR AUROC | Logistic recall | Logistic AUROC | Runtime vs plug-in | +|---|---|---|---|---|---|---| +| panzeri_treves | 0.23 -> 0.32 (10/0) | 0.71 -> 0.86 | 0.79 -> 0.77 (3/7) | 0.34 -> 0.47 (9/0) | 0.87 -> 0.83 (2/8) | 12x, 120x | +| wu_yang | 0.23 -> 0.29 (7/0) | 0.71 -> 0.76 | 0.79 -> 0.73 (2/8) | 0.34 -> 0.43 (7/1) | 0.87 -> 0.81 (2/8) | 1.1x, 2.2x | + +Both find more edges with a worse edge-score ranking, the trade-off CWJ selection showed; +Panzeri-Treves is the strongest selection statistic tested but far too slow, and neither +was run on CausalRivers, where a recall-raising selection statistic (Miller-Madow) cost +0.06 AUROC. + +**Gaussian copula.** Copula-normalizing the data before `auto.infer` turns its +linear-Gaussian selection estimator into the Gaussian-copula one :cite:`Ince2017`: + +| Benchmark | AUROC raw -> copula | +|---|---| +| VAR, 500 / 2000 | 0.946 -> 0.948 (3/2), 0.995 -> 0.995 (2/2) | +| Logistic maps, 500 / 2000 | 0.712 -> 0.648 (0/5), 0.970 -> 0.972 (3/2) | +| Lorenz-96, 1000 | 0.936 -> 0.936 (no change) | +| NetSim, 20 subjects | 0.625 -> 0.618 (6/13) | +| CausalRivers, 20 graphs | 0.581 -> 0.606 (8/4) | + +It helps CausalRivers and hurts NetSim and short logistic series, so it is not a default +either; `copula_normalize` before `auto.infer` stays a user option for river-like data. + +`dit` also renamed `prng` to `rng` (PR #306); `total_correlation_ksg` no longer accepts +the old name despite the deprecation alias, so infoflow now passes `rng`. + ## Recommendation CWJ layers are a safe default: they improved every synthetic benchmark (most of all the diff --git a/tests/test_measures.py b/tests/test_measures.py index 10a4513..5e5d68d 100644 --- a/tests/test_measures.py +++ b/tests/test_measures.py @@ -141,6 +141,17 @@ def test_dit_estimators_in_layers(estimator): assert crossfit.intrinsic > 0.3 and crossfit.synergistic < 0.05 +def test_dit_estimators_accept_single_outcome_terms(): + from dit.inference.estimators import ENTROPY_ESTIMATORS + + if "pym" not in ENTROPY_ESTIMATORS: + pytest.skip("needs dit with the PYM estimator") + rng = np.random.default_rng(6) + table = np.zeros((2, 2, 1)) + np.add.at(table, (rng.integers(0, 2, 300), rng.integers(0, 2, 300), 0), 1.0) + assert np.isfinite(cmi_from_joint(table, "pym")) + + def test_intrinsic_flow_guards_oversized_channels(): p = np.random.default_rng(0).random((2, 3, 40)) te, tdmi = cmi_from_joint(p), cmi_from_joint(p.sum(axis=2, keepdims=True)) diff --git a/uv.lock b/uv.lock index 7cce921..75ac5b3 100644 --- a/uv.lock +++ b/uv.lock @@ -724,8 +724,8 @@ wheels = [ [[package]] name = "dit" -version = "2.4.dev41+ge3611818a" -source = { git = "https://github.com/dit/dit?branch=main#e3611818a5eed2187ee91183e1bd03306b7074fa" } +version = "2.4.dev54+g6314d14e3" +source = { git = "https://github.com/dit/dit?branch=main#6314d14e3de37b4e812c87bdec89394e25942374" } dependencies = [ { name = "boltons" }, { name = "debtcollector" }, diff --git a/validation/dit_estimators.py b/validation/dit_estimators.py index deeb648..13cf6f2 100644 --- a/validation/dit_estimators.py +++ b/validation/dit_estimators.py @@ -49,16 +49,16 @@ def one(args): from infoflow.crossfit import crossfit_flows from infoflow.measures import edge_flows - name, n, d, rep = args + name, n, d, rep, estimators = args seed = zlib.crc32(f"{name}-{n}-{d}-{rep}".encode()) pairs = sample_pairs(mixture(**PROCESSES[name]), n + 1, seed) x, y = pairs[:-1, 0], pairs[1:, 1] rng = np.random.default_rng(seed + 1) w = np.column_stack([pairs[:-1, 1], rng.integers(0, 3, size=(n, d))]) - out = np.full((2, len(ESTIMATORS), len(LAYERS)), np.nan) + out = np.full((2, len(estimators), len(LAYERS)), np.nan) with warnings.catch_warnings(): warnings.simplefilter("ignore") - for e, est in enumerate(ESTIMATORS): + for e, est in enumerate(estimators): for m, fn in enumerate((edge_flows, crossfit_flows)): try: flow = fn(y, x, w, estimator=est, prng=0) @@ -66,7 +66,7 @@ def one(args): continue values = flow.as_dict() out[m, e] = [values[k] for k in LAYERS] - return args, out + return args[:4], out def main(): @@ -76,11 +76,13 @@ def main(): parser.add_argument("--d", type=int, nargs="+", default=[0, 1, 2, 3]) parser.add_argument("--workers", type=int, default=8) parser.add_argument("--out", default="layer_accuracy.nc") + parser.add_argument("--estimators", nargs="+", default=list(ESTIMATORS)) args = parser.parse_args() exact = truths() - jobs = [(p, n, d, r) for p in PROCESSES for n in args.n for d in args.d for r in range(args.reps)] - shape = (len(PROCESSES), len(args.n), len(args.d), args.reps, 2, len(ESTIMATORS), len(LAYERS)) + estimators = tuple(args.estimators) + jobs = [(p, n, d, r, estimators) for p in PROCESSES for n in args.n for d in args.d for r in range(args.reps)] + shape = (len(PROCESSES), len(args.n), len(args.d), args.reps, 2, len(estimators), len(LAYERS)) values = np.full(shape, np.nan) with ProcessPoolExecutor(args.workers) as pool: for (p, n, d, r), out in pool.map(one, jobs, chunksize=4): @@ -93,7 +95,7 @@ def main(): }, coords={ "process": list(PROCESSES), "n": args.n, "d": args.d, "rep": range(args.reps), - "path": ["single", "crossfit"], "estimator": list(ESTIMATORS), "layer": list(LAYERS), + "path": ["single", "crossfit"], "estimator": list(estimators), "layer": list(LAYERS), }, ) # fmt: skip ds.to_netcdf(args.out) diff --git a/validation/dit_estimators_catfr.py b/validation/dit_estimators_catfr.py index 4217c9d..519ebb9 100644 --- a/validation/dit_estimators_catfr.py +++ b/validation/dit_estimators_catfr.py @@ -41,7 +41,7 @@ LAYERS = ("te", "intrinsic", "synergistic", "shared") -def bias(cohort_dir, out): +def bias(cohort_dir, out, estimators=LAYER_ESTIMATORS): import os os.environ["CATFR_OUT"] = str(cohort_dir) @@ -64,7 +64,7 @@ def bias(cohort_dir, out): r = realizations(data, t, list(dict.fromkeys(x_vars + w_vars)), max_lag=max(v[1] for v in x_vars + w_vars)) x, w = r.columns(list(x_vars)), r.columns(list(w_vars)) shuffled = x[rng.permutation(len(x))] - for est in LAYER_ESTIMATORS: + for est in estimators: with warnings.catch_warnings(): warnings.simplefilter("ignore") real = edge_flows(r.present, x, w, estimator=est, prng=0).as_dict() @@ -82,7 +82,7 @@ def bias(cohort_dir, out): print(subject["subject"], flush=True) Path(out).write_text(json.dumps(rows)) print(f"\nmean over {len(summary['subjects'])} subjects x 6 edges (bits): shuffled source (= bias) | real") - for est in LAYER_ESTIMATORS: + for est in estimators: sel = [r for r in rows if r["estimator"] == est] b = {k: np.mean([r["shuffled"][k] for r in sel]) for k in LAYERS} v = {k: np.mean([r["real"][k] for r in sel]) for k in LAYERS} @@ -139,9 +139,10 @@ def main(): parser.add_argument("--out", default=None) parser.add_argument("--estimators", nargs="+", default=list(SELECTION_ESTIMATORS)) parser.add_argument("--files", default="sub-*_hfa.nc") + parser.add_argument("--layer-estimators", nargs="+", default=list(LAYER_ESTIMATORS)) args = parser.parse_args() if args.part == "bias": - bias(args.cohort, args.out or "catfr_bias.json") + bias(args.cohort, args.out or "catfr_bias.json", args.layer_estimators) else: selection(args.pilot, args.out or "catfr_selection.json", args.estimators, args.files) diff --git a/validation/new_defaults.py b/validation/new_defaults.py index a6fc6c0..df5bef5 100644 --- a/validation/new_defaults.py +++ b/validation/new_defaults.py @@ -11,7 +11,12 @@ coupled-logistic and Lorenz-96 networks (``infer_multiplex``); * ``motifs``: :func:`~infoflow.benchmarks.motif_validation` (layers and roles); * ``netsim``, ``rivers``: ``auto.infer`` (fast preset) on NetSim and CausalRivers, as - ``validation/external.py`` does. + ``validation/external.py`` does; +* ``auto_synthetic``: ``auto.infer`` on the ``synthetic`` graphs and simulations. + +``--selection``/``--layers`` pick any estimators instead of a named ``--config``, and +``--copula`` copula-normalizes the data before ``auto.infer`` (which turns its +linear-Gaussian selection estimator into the Gaussian-copula one :cite:`Ince2017`). python validation/new_defaults.py synthetic --kind lorenz96 --config proposed --out-dir results/ """ @@ -31,39 +36,88 @@ } -def infer_kwargs(config): +def infer_kwargs(c): from infoflow.selection import SkeletonSettings - c = CONFIGS[config] return {"skeleton_settings": SkeletonSettings(entropy_estimator=c["selection"]), "estimator": c["layers"]} -def synthetic(kind, config, reps, seed): +def synthetic(kind, c, reps, seed): from infoflow.benchmarks import network_validation samples = (1000,) if kind == "lorenz96" else (500, 2000) return network_validation( kind=kind, n_nodes=(10,), n_samples=samples, n_reps=reps, max_lag=3, - levels=("skeleton", "intrinsic"), infer_kwargs=infer_kwargs(config), prng=seed, + levels=("skeleton", "intrinsic"), infer_kwargs=infer_kwargs(c), prng=seed, ) # fmt: skip -def motifs(config, reps, seed): +def motifs(c, reps, seed): from infoflow.benchmarks import motif_validation - return motif_validation(n_samples=3000, n_reps=reps, infer_kwargs=infer_kwargs(config), prng=seed) + return motif_validation(n_samples=3000, n_reps=reps, infer_kwargs=infer_kwargs(c), prng=seed) + + +def run_auto(data, c, seed, **kwargs): + from dit.inference import copula_normalize + + from infoflow import auto + if c.get("copula"): + data = copula_normalize(data) + return auto.infer( + data, preset="fast", edge_scores=True, entropy_estimator=c["selection"], layer_estimator=c["layers"], + prng=seed, **kwargs, + ).network # fmt: skip + + +def auto_synthetic(kind, c, reps, seed): + """ + ``auto.infer`` on the graphs and simulations ``network_validation`` would draw. + """ + from infoflow.benchmarks import ( + METRICS, + lorenz96_graph, + random_network, + score_network, + simulate_network, + ) # fmt: skip + from infoflow.datasets import lorenz96 -def external(bench, config, data_dir, seed, sims, subjects, sizes, graphs): + rng = np.random.default_rng(seed) + samples = (1000,) if kind == "lorenz96" else (500, 2000) + levels = ("skeleton", "intrinsic") + values = {m: np.full((len(levels), len(samples), reps), np.nan) for m in METRICS} + for j, T in enumerate(samples): + for r in range(reps): + if kind == "lorenz96": + graph, data = lorenz96_graph(10), lorenz96(T, n_vars=10, seed=int(rng.integers(2**32))) + else: + graph = random_network(10, 3.0, 3, rng) + data = simulate_network(graph, T, kind, prng=rng) + t0 = time.perf_counter() + net = run_auto(data, c, int(rng.integers(2**32)), max_lag=3) + scores = score_network(net, graph, levels) + for li, level in enumerate(levels): + for m in METRICS[:-1]: + values[m][li, j, r] = scores[level][m] + values["runtime"][li, j, r] = time.perf_counter() - t0 + print(kind, T, r, round(scores["skeleton"]["auroc"], 3), flush=True) + dims = ("level", "n_samples", "rep") + return xr.Dataset( + {m: (dims, v) for m, v in values.items()}, + coords={"level": list(levels), "n_samples": list(samples), "rep": range(reps)}, + ) + + +def external(bench, c, data_dir, seed, sims, subjects, sizes, graphs): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) from external import METRICS, netsim_instances, rivers_instances, undirected_auroc - from infoflow import auto from infoflow.benchmarks import score_edges - c = CONFIGS[config] if bench == "netsim": instances = netsim_instances(data_dir, sims, subjects) else: @@ -73,10 +127,7 @@ def external(bench, config, data_dir, seed, sims, subjects, sizes, graphs): for label, data, truth in instances: s = int(rng.integers(2**32)) t0 = time.perf_counter() - net = auto.infer( - data, preset="fast", edge_scores=True, entropy_estimator=c["selection"], layer_estimator=c["layers"], - prng=s, - ).network # fmt: skip + net = run_auto(data, c, s) elapsed = time.perf_counter() - t0 found = {(p, t) for t, sk in net.skeleton.items() if sk.significant for p in sk.parents()} score = net.dataset["score"].values @@ -90,8 +141,12 @@ def external(bench, config, data_dir, seed, sims, subjects, sizes, graphs): def main(): parser = argparse.ArgumentParser() - parser.add_argument("bench", choices=("synthetic", "motifs", "netsim", "rivers")) - parser.add_argument("--config", choices=tuple(CONFIGS), required=True) + parser.add_argument("bench", choices=("synthetic", "auto_synthetic", "motifs", "netsim", "rivers")) + parser.add_argument("--config", choices=tuple(CONFIGS), default=None) + parser.add_argument("--selection", default="plugin", help="selection entropy estimator (without --config)") + parser.add_argument("--layers", default="chao_wang_jost", help="layer estimator (without --config)") + parser.add_argument("--copula", action="store_true", help="copula-normalize the data before auto.infer") + parser.add_argument("--label", default=None) parser.add_argument("--kind", choices=("var", "logistic", "lorenz96"), default="var") parser.add_argument("--reps", type=int, default=5) parser.add_argument("--data", default=None) @@ -103,16 +158,22 @@ def main(): parser.add_argument("--out-dir", default=".") args = parser.parse_args() + c = dict(CONFIGS[args.config]) if args.config else {"selection": args.selection, "layers": args.layers} + c["copula"] = args.copula + label = args.label or args.config or f"{c['selection']}-{c['layers']}{'-copula' if args.copula else ''}" + start = time.perf_counter() if args.bench == "synthetic": - ds, name = synthetic(args.kind, args.config, args.reps, args.seed), f"synthetic_{args.kind}" + ds, name = synthetic(args.kind, c, args.reps, args.seed), f"synthetic_{args.kind}" + elif args.bench == "auto_synthetic": + ds, name = auto_synthetic(args.kind, c, args.reps, args.seed), f"auto_synthetic_{args.kind}" elif args.bench == "motifs": - ds, name = motifs(args.config, args.reps, args.seed), "motifs" + ds, name = motifs(c, args.reps, args.seed), "motifs" else: - ds = external(args.bench, args.config, args.data, args.seed, args.sims, args.subjects, args.sizes, args.graphs) + ds = external(args.bench, c, args.data, args.seed, args.sims, args.subjects, args.sizes, args.graphs) name = args.bench ds.attrs["seconds"] = time.perf_counter() - start - out = Path(args.out_dir) / f"{name}_{args.config}.nc" + out = Path(args.out_dir) / f"{name}_{label}.nc" ds.to_netcdf(out) print(out, round(ds.attrs["seconds"]), "s", flush=True) From 08d04381bd2641817eb2990e202ba371a6b98c87 Mon Sep 17 00:00:00 2001 From: Ryan James Date: Sun, 1 Nov 2026 12:30:35 -0700 Subject: [PATCH 5/5] Index np.add.at with an array so ty accepts the single-outcome test --- tests/test_measures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_measures.py b/tests/test_measures.py index 5e5d68d..52c47e8 100644 --- a/tests/test_measures.py +++ b/tests/test_measures.py @@ -148,7 +148,7 @@ def test_dit_estimators_accept_single_outcome_terms(): pytest.skip("needs dit with the PYM estimator") rng = np.random.default_rng(6) table = np.zeros((2, 2, 1)) - np.add.at(table, (rng.integers(0, 2, 300), rng.integers(0, 2, 300), 0), 1.0) + np.add.at(table, (rng.integers(0, 2, 300), rng.integers(0, 2, 300), np.zeros(300, dtype=np.int64)), 1.0) assert np.isfinite(cmi_from_joint(table, "pym"))