From 88d33190dd36d81a15b330367a11e80b0378c3f1 Mon Sep 17 00:00:00 2001 From: Ryan James Date: Sat, 14 Nov 2026 10:45:17 -0700 Subject: [PATCH] Lag-0 explanation families under weaker adjacency faithfulness The PC-style lag-0 layer reads adjacencies from dependence, so it cannot see a link whose dependence shows only jointly: for a lag-0 XOR every pair is independent and the layer reports nothing. contemporaneous_families (and infer_multiplex(contemporaneous="family")) instead reports, per component of at most five processes, every DAG that is Markov, weakly adjacency faithful (Marx, Gretton & Mooij 2021) and SGS-minimal with respect to a table of conditional-independence decisions, following Hourican & Quax 2026. - infoflow/families.py: CIDecisions, explanation_family, ExplanationFamily and decisions_from_distribution (exact decisions for a dit distribution). Every DAG on up to five nodes is enumerated once, with vectorised d-separation tables. - Decisions are tested at alpha / m for a component's m statements. One false rejection removes the true DAG, so uncorrected decisions leave many families empty; corrected ones recovered the population family in 20 of 20 samples of each motif (XOR, XOR with a descendant, OR, chain). - Components that are too large, or whose tables are too sparse to test, keep the PC-style entries. The contemporaneous matrix marks only what every explanation shares; contemporaneous_possible marks what any has. - Docs: a lag-0 families page, literature review, interpretation and API. --- docs/api.rst | 1 + docs/families.rst | 110 ++++++++++ docs/index.rst | 1 + docs/interpretation.rst | 3 +- docs/literature.rst | 16 ++ docs/references.bib | 26 +++ infoflow/contemporaneous.py | 274 ++++++++++++++++++++++++- infoflow/families.py | 399 ++++++++++++++++++++++++++++++++++++ infoflow/network.py | 36 +++- tests/test_families.py | 352 +++++++++++++++++++++++++++++++ 10 files changed, 1211 insertions(+), 7 deletions(-) create mode 100644 docs/families.rst create mode 100644 infoflow/families.py create mode 100644 tests/test_families.py diff --git a/docs/api.rst b/docs/api.rst index 19b6484..f0b68f1 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -52,6 +52,7 @@ Nodes and interpretation .. automodule:: infoflow.nodes .. automodule:: infoflow.interpret .. automodule:: infoflow.contemporaneous +.. automodule:: infoflow.families .. automodule:: infoflow.hyperedges .. automodule:: infoflow.oinfo diff --git a/docs/families.rst b/docs/families.rst new file mode 100644 index 0000000..1b5e23d --- /dev/null +++ b/docs/families.rst @@ -0,0 +1,110 @@ +Lag-0 explanation families +========================== + +With coarse sampling, processes interact within one sample, and time order no longer orients +those links. The PC-style lag-0 layer (:func:`infoflow.contemporaneous.contemporaneous_layer`) +reads adjacencies from dependence :cite:`Runge2020`. That assumes faithfulness +:cite:`Spirtes2000`: every independence in the data comes from a separation in the causal +graph. Synergy breaks it. If :math:`Z_t = X_t \oplus Y_t \oplus E_t`, every pair of the three +is independent, so the layer reports no link at all, although :math:`X_t` and :math:`Y_t` +together determine :math:`Z_t`. + +``infer_multiplex(contemporaneous="family")`` +(:func:`infoflow.contemporaneous.contemporaneous_families`) instead reports every explanation +that *weaker adjacency faithfulness* allows :cite:`Marx2021weaker,Hourican2026`. A link may +then be supported by a triple association, as in XOR, and the answer is a family of DAGs. For +the XOR above it is a collider at each of the three variables. Neither a single skeleton nor +the union of the three (a triangle) represents that answer. + +.. code-block:: python + + net = infoflow.infer_multiplex(data, names=["X", "Y", "Z"], contemporaneous="family", prng=0) + for family in net.contemporaneous_families: + print(family.describe(net.names)) + # {X, Y, Z}: 3 explanations: {X -> Y, Z -> Y} | {X -> Z, Y -> Z} | {Y -> X, Z -> X} + +Lagged links need no families: time order already leaves one explanation for them, and the +lagged search finds jointly informative parents with its pair search (:doc:`skeleton`). + +Procedure +--------- + +1. **Screen.** Two processes are linked when their present values are dependent given the + lagged conditioning set, or given that set and the present value of one more process. + The lagged conditioning set holds both processes' selected pasts and lagged parents, as in + the PC-style layer. Linked processes form components. +2. **Decisions.** Each component of at most five processes is decided statement by statement. + Every pairwise statement :math:`X^i_t \perp X^j_t \mid X^S_t`, with :math:`S` a set of the + other processes in the component, is tested given the pair's lagged conditioning set. The + level is :math:`\alpha / m` for the component's :math:`m` statements (6 for three + processes, 24 for four, 80 for five). +3. **Family.** Every DAG on the component (25, 543 or 29,281 of them) is kept when it is + + * *Markov*: every statement it implies by d-separation was decided independent; + * *weakly adjacency faithful*: at each endpoint :math:`x` of each edge :math:`x - w`, either + :math:`x` and :math:`w` are dependent given every conditioning set, or they form a strict + two-association with a third member :math:`z` of :math:`x`'s Markov blanket. Each of the + three pairs is then dependent given the remaining variable and any further set, while + neither :math:`x, w` nor :math:`x, z` is dependent given every set + (``convention="neither"``; ``"not_both"`` only requires that not both are); + * *SGS-minimal*: no DAG with one edge fewer is Markov; + * consistent with background knowledge: ``tiers`` (a known order within a sample) and + ``forbidden`` arcs, passed through ``contemporaneous_settings``. + +Reading the result +------------------ + +``network.contemporaneous_families`` lists one :class:`~infoflow.families.ExplanationFamily` +per component, with its explanations and a status: + +``complete`` + Every DAG was checked, and these are all that qualify. +``empty`` + None qualifies: the assumptions fail for these processes, or the decisions contradict each + other. +``too_large`` + The component has more than ``max_family_size`` processes. +``unavailable`` + A statement's table has fewer than ``min_cell_count`` samples per cell on average, so not + rejecting it would be no evidence of independence. + +The last two keep the PC-style layer's entries and make no further claim. + +``dataset["contemporaneous"]`` marks only what every explanation shares: 2 for an arrow in +all of them, 1 for a link in all of them whose direction varies, 0 otherwise. A lag-0 XOR +therefore gives an all-zero matrix. ``dataset["contemporaneous_possible"]`` marks the links +that appear in at least one explanation. + +Why the decisions are corrected +------------------------------- + +A true DAG implies many independences, and one false rejection among them removes it from +the family. At an uncorrected level of 0.05 per statement, families are therefore often +empty, which is the finite-sample problem :cite:`Hourican2026` report for their family +estimator. We simulated four lag-0 motifs (XOR, XOR with a descendant, OR, and a +four-process chain) at 2000 and 10,000 samples: + +* Uncorrected decisions returned exactly the population family in 55--90% of samples, and + an empty family in most of the rest. +* With the Bonferroni level, asymptotic tests returned it in all 20 samples of every motif. + Permutation tests (the default) returned it in all 20 samples of every motif at 2000 + samples. + +The price is power for weak lag-0 dependence. + +Assumptions and limits +---------------------- + +The explanations assume: + +* no hidden common causes of the measured processes within a sample; +* acyclic effects within a sample; +* no interaction between components; +* lag-0 permutation tests that are adequate despite serial dependence (they condition on the + lagged pasts). + +Components of more than five processes are not enumerated. + +:func:`infoflow.families.decisions_from_distribution` gives exact decisions for a known +distribution. With it, :func:`infoflow.families.explanation_family` computes the population +families that the tests are checked against. diff --git a/docs/index.rst b/docs/index.rst index 598faa1..941e3c6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,7 @@ and false-discovery-rate control. spectral backbone interpretation + families hyperedges machines channels diff --git a/docs/interpretation.rst b/docs/interpretation.rst index a124b62..eede5b8 100644 --- a/docs/interpretation.rst +++ b/docs/interpretation.rst @@ -36,7 +36,8 @@ Optional layers: * a lag-0 layer with collider orientation in the style of PCMCI+ :cite:`Runge2020` (:mod:`infoflow.contemporaneous`), with an adapter to Tigramite - :cite:`Runge2018`; + :cite:`Runge2018`, or every lag-0 explanation that weaker adjacency faithfulness + allows, which also sees a lag-0 XOR (:doc:`families`); * detected and classified partial-information-decomposition hyperedges over sets of parents, with any of dit's PID measures (:doc:`hyperedges`). diff --git a/docs/literature.rst b/docs/literature.rst index a3ed9c3..04badbb 100644 --- a/docs/literature.rst +++ b/docs/literature.rst @@ -79,6 +79,19 @@ dependence by running selection on subsamples and aggregating with e-values control through the e-BH procedure :cite:`Wang2022,Ren2024`. Stability selection :cite:`Meinshausen2010` reports how often each variable is selected over subsamples. +**Faithfulness violations.** Constraint-based learners read adjacencies from dependences, +so they delete links whose dependence shows only jointly: an XOR target is independent of +each of its parents. Weaker adjacency faithfulness :cite:`Marx2021weaker` lets a link be +supported by a triple association instead. :cite:`Hourican2026` operationalise it in two +ways. One is a grow-shrink learner that vetoes a removal when one more conditioning variable +restores dependence, and protects links whose whole-minus-sum information exceeds 0.3 bits. +The other searches for the family of DAGs that are Markov, weakly adjacency faithful and +SGS-minimal. Without time order an XOR has three such explanations; with it, one. Their +finite-sample families are often empty, and their searches did not finish beyond eight +variables. ``infoflow`` already has the first idea in the lagged search (the pair search, +and pruning of each parent given the others). It applies the second at lag 0, where no time +order helps, with decisions corrected for the number of statements in a component. + **Higher-order structure.** The O-information rate :cite:`Faes2022` extends the O-information :cite:`Rosas2019` to processes; its local version (for a link) and gradients (for a node) :cite:`Scagliarini2023` give a network representation of @@ -116,6 +129,9 @@ Regimes :func:`~infoflow.regimes.infer_regimes`, :func:`~infoflow.regimes.compare_regimes` Multiple datasets :func:`~infoflow.regimes.infer_with_context` Stability selection / e-values :func:`~infoflow.stability.stability_selection` +Weaker adjacency faithfulness pair search for lagged parents (``synergy_search``); + lag-0 explanation families, + :func:`~infoflow.contemporaneous.contemporaneous_families` Mixed-type CMI ``SkeletonSettings(estimator="mixed")`` Predictive (AGM-TE style) TE :func:`~infoflow.predictive.predictive_transfer_entropy` O-information rate :mod:`infoflow.oinfo` diff --git a/docs/references.bib b/docs/references.bib index 21f80e6..1aa49d6 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -1145,3 +1145,29 @@ @article{Zhang2012 year = {2012}, doi = {10.1162/NECO_a_00266} } + +@inproceedings{Marx2021weaker, + author = {Marx, Alexander and Gretton, Arthur and Mooij, Joris M.}, + title = {A weaker faithfulness assumption based on triple interactions}, + booktitle = {Proceedings of the 37th Conference on Uncertainty in Artificial Intelligence (UAI)}, + series = {Proceedings of Machine Learning Research}, + volume = {161}, + pages = {451--460}, + year = {2021} +} + +@misc{Hourican2026, + author = {Hourican, Cillian and Quax, Rick}, + title = {Operationalising weaker faithfulness: recovery, ambiguity and computational limits}, + year = {2026}, + eprint = {2610.31724}, + archivePrefix = {arXiv} +} + +@book{Spirtes2000, + author = {Spirtes, Peter and Glymour, Clark and Scheines, Richard}, + title = {Causation, Prediction, and Search}, + edition = {2}, + publisher = {MIT Press}, + year = {2000} +} diff --git a/infoflow/contemporaneous.py b/infoflow/contemporaneous.py index 1223b5c..97a7ad2 100644 --- a/infoflow/contemporaneous.py +++ b/infoflow/contemporaneous.py @@ -7,18 +7,30 @@ pasts and lagged parents (a momentary conditional independence test in the spirit of PCMCI+ :cite:`Runge2020`), removes links explained by a third contemporaneous neighbour (PC-style), and orients unshielded triples ``i - k - j`` with ``k`` -outside the separating set as colliders ``i -> k <- j``. For full latent-variable -output (partial ancestral graphs) use :func:`run_tigramite` with LPCMCI. +outside the separating set as colliders ``i -> k <- j``. That reads adjacencies from +dependence, so it cannot see links whose dependence shows only jointly (a lag-0 XOR); +:func:`contemporaneous_families` reports every explanation that weaker adjacency +faithfulness allows instead :cite:`Marx2021weaker,Hourican2026`. For full +latent-variable output (partial ancestral graphs) use :func:`run_tigramite` with LPCMCI. """ +from dataclasses import dataclass from itertools import combinations import numpy as np from dit.inference._symbols import as_generator -from .selection import _cmi, _Columns +from .families import MAX_NODES, CIDecisions, ExplanationFamily, explanation_family, statements +from .selection import _cmi, _Columns, _null_statistics, _pvalue, check_n_perm -__all__ = ("contemporaneous_layer", "instantaneous_exchange", "run_tigramite", "to_tigramite") +__all__ = ( + "ContemporaneousFamilies", + "contemporaneous_families", + "contemporaneous_layer", + "instantaneous_exchange", + "run_tigramite", + "to_tigramite", +) def _conditioning(skeleton, i, j): @@ -119,6 +131,260 @@ def contemporaneous_layer(data, skeleton, alpha=0.05, n_perm=200, prng=None, exc return graph, values, pvalues +# -- explanation families ----------------------------------------------------------------- + + +def _levels(z, x): + """ + The number of distinct values of `x` in each stratum of `z`, in the order of ``np.unique(z)``. + """ + base = int(x.max()) + 1 + pairs = np.unique(z.astype(np.int64) * base + x) + return np.unique(pairs // base, return_counts=True)[1] + + +def _lag0_test(data, i, j, given, cond, n_perm, rng, alpha=None, batch=None, test="permutation", min_cell_count=None): + """ + :math:`I[X^i_t : X^j_t \\mid X^{given}_t, cond]` in bits, its p-value, and whether the + table has at least `min_cell_count` samples per cell on average. + + ``test='permutation'`` permutes :math:`X^i_t` freely (curtailed at `alpha` with `batch`); + ``'asymptotic'`` compares :math:`G^2 = 2 N \\ln 2 \\, \\hat I` with a chi-squared law + whose degrees of freedom count the observed levels in each stratum :cite:`Agresti2013`. + """ + variables = list(dict.fromkeys([(i, 0), *[(k, 0) for k in given], *cond])) + max_lag = max((v[1] for v in variables), default=0) + cols = _Columns(data, j, variables, max_lag) + x, Kx = cols.column((i, 0)) + z, Kz = cols.joint([v for v in variables if v != (i, 0)]) + value = _cmi(cols.y, cols.Ky, x, Kx, z, Kz) + available = True + if min_cell_count is not None: + cells = len(np.unique(x)) * len(np.unique(cols.y)) * len(np.unique(z)) + available = cols.n >= min_cell_count * cells + if test == "asymptotic": + from scipy.stats import chi2 + + dof = int(np.sum((_levels(z, x) - 1) * (_levels(z, cols.y) - 1))) + p = float(chi2.sf(2 * cols.n * np.log(2) * value, dof)) if dof > 0 else 1.0 + return value, p, available + + def evaluate(idx): + return [_cmi(cols.y, cols.Ky, x[k], Kx, z, Kz) for k in idx] + + null = _null_statistics(n_perm, lambda: rng.permutation(cols.n), evaluate, value, alpha, batch) + return value, _pvalue(null, value), available + + +def _components(linked): + """ + Connected components (sorted tuples) of a boolean adjacency matrix. + """ + P = len(linked) + seen, out = set(), [] + for start in range(P): + if start in seen: + continue + stack, members = [start], {start} + while stack: + for nb in np.flatnonzero(linked[stack.pop()]).tolist(): + if nb not in members: + members.add(nb) + stack.append(nb) + seen |= members + out.append(tuple(sorted(members))) + return out + + +@dataclass +class ContemporaneousFamilies: + """ + Lag-0 explanation families (:func:`contemporaneous_families`). + + Attributes + ---------- + graph : np.ndarray + (processes, processes): 2 where every explanation has ``i -> j``; 1 (both ways) where + every explanation links ``i`` and ``j`` but they disagree on the direction; 0 + otherwise. Components that were not enumerated (``too_large``, ``unavailable``) keep + the entries of :func:`contemporaneous_layer`. + possible : np.ndarray + (processes, processes), boolean and symmetric: ``i`` and ``j`` are linked in at least + one explanation (adjacent in :func:`contemporaneous_layer` for components that were + not enumerated). The union of the explanations is not itself an explanation. + families : list of ExplanationFamily + One per connected component of the screen with at least two processes; nodes are + process indices. + values, pvalues : np.ndarray + Each pair's contemporaneous CMI (bits) given the lagged conditioning set, and its + permutation p-value (:func:`instantaneous_exchange`). + """ + + graph: np.ndarray + possible: np.ndarray + families: list + values: np.ndarray + pvalues: np.ndarray + + +def _family_permutations(n_perm, n_statements, alpha): + """ + Permutations per statement, enough that the smallest p-value clears ``alpha / n_statements``. + """ + if n_perm is None: + return max(200, 2 * int(np.ceil(n_statements / alpha))) + check_n_perm(n_perm, alpha / n_statements) + return n_perm + + +def contemporaneous_families( + data, + skeleton, + alpha=0.05, + n_perm=None, + prng=None, + exchange=None, + max_family_size=MAX_NODES, + convention="neither", + tiers=None, + forbidden=(), + test="permutation", + min_cell_count=5, +): + """ + Every lag-0 explanation that weaker adjacency faithfulness allows, per component. + + A PC-style lag-0 layer (:func:`contemporaneous_layer`) cannot see a link whose + dependence shows only jointly: if :math:`Z_t = X_t \\oplus Y_t \\oplus E_t`, every pair is + independent. Following :cite:`Hourican2026`, this reports instead the family of DAGs + that are Markov, weakly adjacency faithful :cite:`Marx2021weaker`, SGS-minimal and + consistent with background knowledge (:func:`infoflow.families.explanation_family`): + + 1. **Screen.** Two processes are linked when their present values are dependent given + the lagged conditioning set (both processes' selected pasts and lagged parents), or + given that set and one more process's present value, as in XOR. Linked processes form + components. + 2. **Decisions.** In each component of at most `max_family_size` processes, every + pairwise statement :math:`X^i_t \\perp X^j_t \\mid X^S_t` (also given the pair's lagged + conditioning set) is tested at level :math:`\\alpha / m` for :math:`m` statements + (Bonferroni). One false rejection among the many independences a true DAG implies + removes it from the family, so uncorrected levels leave many families empty. + 3. **Family.** Every DAG on the component is checked against the decisions. + + Parameters + ---------- + data : DiscreteData + skeleton : dict + ``target -> TargetSkeleton``. + alpha : float + The screen's level, and the family-wise level of each component's decisions. + n_perm : int, None + Permutations per test; by default 200 for the screen and twice what each + component's corrected level needs for its decisions. + prng : None, int, Generator + exchange : tuple, None + The ``(exchange, pvalues)`` pair from :func:`instantaneous_exchange`, to reuse. + max_family_size : int + Larger components are not enumerated (status ``too_large``) and keep the entries of + :func:`contemporaneous_layer`. At most :data:`~infoflow.families.MAX_NODES`. + convention : {'neither', 'not_both'} + The strictness of the two-association (:func:`~infoflow.families.explanation_family`). + tiers : dict or sequence, None + A tier per process (a sequence follows process order); no lag-0 arc may point to an + earlier tier, e.g. when a within-sample order is known. + forbidden : iterable of (int, int) + Lag-0 arcs (process indices) that no explanation may contain. + test : {'permutation', 'asymptotic'} + How the decisions are tested (see the screen and :func:`_lag0_test`). + min_cell_count : int, None + A statement whose table has fewer samples per cell on average cannot be decided + reliably; its component gets status ``unavailable`` and keeps the entries of + :func:`contemporaneous_layer`, since a non-rejection there is no evidence of + independence. None disables the check. + + Returns + ------- + ContemporaneousFamilies + """ + if not 1 <= max_family_size <= MAX_NODES: + raise ValueError(f"max_family_size must be between 1 and {MAX_NODES}") + if test not in ("permutation", "asymptotic"): + raise ValueError("test must be 'permutation' or 'asymptotic'") + if convention not in ("neither", "not_both"): + raise ValueError("convention must be 'neither' or 'not_both'") + rng = as_generator(prng) + P = data.n_processes + screen_perm = n_perm or 200 + check_n_perm(screen_perm, alpha) + values, pvalues = exchange if exchange is not None else instantaneous_exchange(data, skeleton, screen_perm, rng) + tier = None if tiers is None else (dict(tiers) if isinstance(tiers, dict) else dict(enumerate(tiers))) + forbidden = [tuple(arc) for arc in forbidden] + + linked = (pvalues <= alpha) & ~np.eye(P, dtype=bool) + for i, j in combinations(range(P), 2): + if linked[i, j]: + continue + cond = _conditioning(skeleton, i, j) + for k in range(P): + if k in (i, j): + continue + _, p, _ = _lag0_test(data, i, j, [k], cond, screen_perm, rng, alpha, 20, test) + if p <= alpha: + linked[i, j] = linked[j, i] = True + break + + families = [] + for nodes in _components(linked): + if len(nodes) < 2: + continue + if len(nodes) > max_family_size: + families.append(ExplanationFamily(nodes, status="too_large", convention=convention)) + continue + queries = statements(len(nodes)) + level = alpha / len(queries) + perms = _family_permutations(n_perm, len(queries), alpha) + independent, found = {}, {} + for i, j, S in queries: + a, b = nodes[i], nodes[j] + given = [nodes[k] for k in sorted(S)] + cond = _conditioning(skeleton, a, b) + _, p, ok = _lag0_test(data, a, b, given, cond, perms, rng, level, 20, test, min_cell_count) + if not ok: + break + independent[i, j, S], found[i, j, S] = p > level, p + else: + decisions = CIDecisions(nodes, independent, found, level) + families.append(explanation_family(decisions, convention=convention, tiers=tier, forbidden=forbidden)) + continue + families.append(ExplanationFamily(nodes, status="unavailable", convention=convention)) + + graph = np.zeros((P, P), dtype=np.int64) + possible = np.zeros((P, P), dtype=bool) + fallback = [f for f in families if f.status in ("too_large", "unavailable")] + if fallback: + pc, _, _ = contemporaneous_layer(data, skeleton, alpha, screen_perm, rng, (values, pvalues)) + for family in fallback: + idx = np.ix_(family.nodes, family.nodes) + graph[idx] = pc[idx] + possible[idx] = (pc[idx] != 0) | (pc[idx].T != 0) + for family in families: + if family.status != "complete": + continue + arrows = family.compelled_arrows + for pair in family.compelled_adjacencies: + u, v = sorted(pair) + if (u, v) in arrows: + graph[u, v] = 2 + elif (v, u) in arrows: + graph[v, u] = 2 + else: + graph[u, v] = graph[v, u] = 1 + for pair in family.possible_adjacencies: + u, v = sorted(pair) + possible[u, v] = possible[v, u] = True + return ContemporaneousFamilies(graph, possible, families, values, pvalues) + + def to_tigramite(data, names=None): """ A ``tigramite.data_processing.DataFrame`` of the discrete symbols (one per trial diff --git a/infoflow/families.py b/infoflow/families.py new file mode 100644 index 0000000..4343790 --- /dev/null +++ b/infoflow/families.py @@ -0,0 +1,399 @@ +""" +Explanation families: every DAG that a table of conditional-independence decisions allows. + +Faithfulness lets a structure learner read adjacencies from dependences, so it deletes a link +whose dependence shows only jointly. In noisy XOR, :math:`Z = X \\oplus Y \\oplus E`, every +pair of variables is independent although :math:`X` and :math:`Y` together determine +:math:`Z`. Weaker adjacency faithfulness :cite:`Marx2021weaker` lets an adjacency be supported +by a triple association instead, and :cite:`Hourican2026` take as the target of structure +learning the *family* of DAGs that are Markov to the distribution, satisfy that condition, are +SGS-minimal :cite:`Spirtes2000`, and agree with background knowledge. Without time order XOR +has three explanations, a collider at each variable. No single skeleton represents them, and +their union, a triangle, is none of them. + +:func:`explanation_family` evaluates those predicates over every DAG on up to +:data:`MAX_NODES` nodes, for decisions from tests on data +(:func:`infoflow.contemporaneous.contemporaneous_families`) or from an exact distribution +(:func:`decisions_from_distribution`). +""" + +from dataclasses import dataclass +from functools import cache +from itertools import combinations, permutations, product + +import numpy as np + +__all__ = ( + "MAX_NODES", + "CIDecisions", + "ExplanationFamily", + "decisions_from_distribution", + "explanation_family", + "statements", +) + +#: The most nodes :func:`explanation_family` enumerates DAGs on (29,281 DAGs). +MAX_NODES = 5 + + +def _subsets(items): + items = sorted(items) + for k in range(len(items) + 1): + yield from (frozenset(c) for c in combinations(items, k)) + + +@cache +def statements(n): + """ + Every pairwise conditional-independence statement on `n` nodes. + + Returns + ------- + tuple + ``(i, j, S)`` for positions ``i < j`` and every frozenset ``S`` of the other positions. + """ + return tuple((i, j, S) for i, j in combinations(range(n), 2) for S in _subsets(set(range(n)) - {i, j})) + + +@dataclass(frozen=True) +class CIDecisions: + """ + A decision for every pairwise conditional-independence statement on a set of nodes. + + Attributes + ---------- + nodes : tuple + Node labels. Statements refer to nodes by position. + independent : dict + ``(i, j, S) -> bool`` for every statement of :func:`statements`: True when + :math:`X_i` and :math:`X_j` are judged independent given :math:`X_S`. + pvalues : dict, None + The p-value behind each decision, when the decisions come from tests. + level : float, None + The level each p-value was compared with. + """ + + nodes: tuple + independent: dict + pvalues: dict | None = None + level: float | None = None + + def __post_init__(self): + missing = [s for s in statements(len(self.nodes)) if s not in self.independent] + if missing: + raise ValueError(f"no decision for {len(missing)} statements, e.g. {missing[0]}") + + def is_independent(self, i, j, given=()): + """ + The decision for :math:`X_i \\perp X_j \\mid X_{given}` (positions). + """ + return bool(self.independent[min(i, j), max(i, j), frozenset(given)]) + + +def decisions_from_distribution(dist, tol=1e-10): + """ + Exact decisions for a dit distribution. + + Parameters + ---------- + dist : dit.Distribution + tol : float + A statement is independent when its conditional mutual information is at most + `tol` bits. + + Returns + ------- + CIDecisions + Nodes are the random variables' names, or their indices. + """ + from dit.multivariate import coinformation + + n = dist.outcome_length() + names = dist.get_rv_names() + independent = {(i, j, S): coinformation(dist, [[i], [j]], sorted(S)) <= tol for i, j, S in statements(n)} + return CIDecisions(tuple(names) if names else tuple(range(n)), independent) + + +# -- every DAG on a few nodes ---------------------------------------------------------- + + +@dataclass(frozen=True) +class _DagTable: + """ + Every DAG on `n` nodes, with what each implies. + + ``arcs[d, u, v]`` is the arc ``u -> v`` of DAG ``d``; ``without[d, u * n + v]`` is the index + of DAG ``d`` with that arc removed (-1 if absent); ``separated[d, k]`` is whether DAG ``d`` + implies statement ``k`` of :func:`statements` by d-separation. + """ + + n: int + arcs: np.ndarray + without: np.ndarray + separated: np.ndarray + + +def _closure(arcs): + """ + ``reach[d, u, v]``: a directed path of length at least one from ``u`` to ``v``. + """ + reach = arcs.copy() + for _ in range(arcs.shape[1] - 1): + longer = reach | ((reach.astype(np.uint8) @ arcs.astype(np.uint8)) > 0) + if (longer == reach).all(): + break + reach = longer + return reach + + +@cache +def _dag_table(n): + if not 1 <= n <= MAX_NODES: + raise ValueError(f"DAGs are enumerated on 1 to {MAX_NODES} nodes, not {n}") + pairs = list(combinations(range(n), 2)) + states = np.array(list(product(range(3), repeat=len(pairs))), dtype=np.int8) + arcs = np.zeros((len(states), n, n), dtype=bool) + for k, (i, j) in enumerate(pairs): + arcs[:, i, j] = states[:, k] == 1 + arcs[:, j, i] = states[:, k] == 2 + reach = _closure(arcs) + acyclic = ~reach[:, np.arange(n), np.arange(n)].any(axis=1) + arcs, reach = arcs[acyclic], reach[acyclic] + D = len(arcs) + + # Removing an arc leaves a DAG: find it by its arc bitmask. + flat = arcs.reshape(D, n * n) + bits = np.left_shift(np.int64(1), np.arange(n * n, dtype=np.int64)) + masks = flat.astype(np.int64) @ bits + order = np.argsort(masks) + position = np.clip(np.searchsorted(masks[order], masks[:, None] ^ bits[None, :]), 0, D - 1) + without = np.where(flat, order[position], -1) + + # d-separation: X_i and X_j are separated by X_S iff no path joins them in the moral + # graph of the ancestral set of {i, j} | S once S is removed. + ancestors = reach.transpose(0, 2, 1) | np.eye(n, dtype=bool) # [d, v, u]: u is v or an ancestor of v + linked = arcs | arcs.transpose(0, 2, 1) + coparents = arcs[:, :, None, :] & arcs[:, None, :, :] # [d, u, v, c]: u -> c <- v + queries = statements(n) + separated = np.zeros((D, len(queries)), dtype=bool) + for k, (i, j, S) in enumerate(queries): + keep = ancestors[:, [i, j, *S], :].any(axis=1) + moral = (linked | (coparents & keep[:, None, None, :]).any(axis=3)) & keep[:, :, None] & keep[:, None, :] + allowed = keep.copy() + allowed[:, sorted(S)] = False + reached = np.zeros((D, n), dtype=bool) + reached[:, i] = True + for _ in range(n - 1): + reached |= (reached[:, :, None] & moral).any(axis=1) & allowed + separated[:, k] = ~reached[:, j] + return _DagTable(n, arcs, without, separated) + + +# -- the predicates ---------------------------------------------------------------------- + + +def _associations(decisions, convention): + """ + ``a1[x, u]``: dependent given every conditioning set. ``s2[x, u, v]``: a strict + two-association of ``x`` with ``u`` and ``v`` :cite:`Marx2021weaker,Hourican2026`. + """ + n = len(decisions.nodes) + every = set(range(n)) + + def dependent(i, j, given): + return not decisions.is_independent(i, j, given) + + a1 = np.zeros((n, n), dtype=bool) + for x, u in permutations(range(n), 2): + a1[x, u] = all(dependent(x, u, T) for T in _subsets(every - {x, u})) + s2 = np.zeros((n, n, n), dtype=bool) + for x, u, v in permutations(range(n), 3): + a2 = all( + dependent(x, u, T | {v}) and dependent(x, v, T | {u}) and dependent(u, v, T | {x}) + for T in _subsets(every - {x, u, v}) + ) + if convention == "neither": + s2[x, u, v] = a2 and not a1[x, u] and not a1[x, v] + else: + s2[x, u, v] = a2 and not (a1[x, u] and a1[x, v]) + return a1, s2 + + +def _weakly_faithful(arcs, a1, s2): + """ + Every edge is supported at both endpoints: by a one-association, or by a strict + two-association with a third member of that endpoint's Markov blanket. + """ + n = len(arcs) + parents = [set(np.flatnonzero(arcs[:, v]).tolist()) for v in range(n)] + children = [set(np.flatnonzero(arcs[v]).tolist()) for v in range(n)] + blanket = [(parents[x] | children[x] | set().union(*(parents[c] for c in children[x]))) - {x} for x in range(n)] + for u, v in zip(*np.nonzero(arcs), strict=True): + for x, w in ((int(u), int(v)), (int(v), int(u))): + if not (a1[x, w] or any(s2[x, w, z] for z in blanket[x] - {w})): + return False + return True + + +def _permitted(table, nodes, tiers, forbidden): + """ + DAGs with no arc from a later tier to an earlier one and no forbidden arc. + """ + n = table.n + ok = np.ones((n, n), dtype=bool) + if tiers is not None: + tier = dict(tiers) if isinstance(tiers, dict) else dict(zip(nodes, tiers, strict=True)) + for u, v in permutations(range(n), 2): + tu, tv = tier.get(nodes[u]), tier.get(nodes[v]) + if tu is not None and tv is not None and tu > tv: + ok[u, v] = False + position = {node: k for k, node in enumerate(nodes)} + for u, v in forbidden: + if u in position and v in position: + ok[position[u], position[v]] = False + return ~(table.arcs & ~ok).any(axis=(1, 2)) + + +def _skeleton(arcs): + return frozenset(frozenset(arc) for arc in arcs) + + +@dataclass(frozen=True) +class ExplanationFamily: + """ + The DAGs that a set of decisions allows on one set of nodes. + + Attributes + ---------- + nodes : tuple + Node labels. + candidates : tuple of frozenset + Each explanation as a frozenset of arcs ``(source, target)``. + status : str + ``'complete'``: every DAG on the nodes was checked, and these are all that qualify. + ``'empty'``: every DAG was checked and none qualifies, because the assumptions fail for + these variables or the decisions contradict each other. + ``'too_large'`` or ``'unavailable'``: nothing was enumerated (too many nodes, or a + statement that could not be tested reliably), so no structural claim is made. + convention : str + The strictness of the two-association: ``'neither'`` or ``'not_both'``. + decisions : CIDecisions, None + """ + + nodes: tuple + candidates: tuple = () + status: str = "complete" + convention: str = "neither" + decisions: CIDecisions | None = None + + def __len__(self): + return len(self.candidates) + + @property + def compelled_arrows(self): + """ + Arcs in every explanation. + """ + return frozenset.intersection(*self.candidates) if self.candidates else frozenset() + + @property + def compelled_adjacencies(self): + """ + Pairs (frozensets) linked in every explanation. + """ + return frozenset.intersection(*map(_skeleton, self.candidates)) if self.candidates else frozenset() + + @property + def possible_adjacencies(self): + """ + Pairs linked in at least one explanation. Their union is not itself an explanation. + """ + return frozenset().union(*map(_skeleton, self.candidates)) + + def graphs(self): + """ + Each explanation as a networkx ``DiGraph``. + """ + import networkx as nx + + out = [] + for arcs in self.candidates: + g = nx.DiGraph() + g.add_nodes_from(self.nodes) + g.add_edges_from(sorted(arcs)) + out.append(g) + return out + + def describe(self, names=None): + """ + One line: the nodes and their explanations, with `names[node]` as labels if given. + """ + + def label(v): + return str(names[v]) if names is not None else str(v) + + head = "{" + ", ".join(label(v) for v in self.nodes) + "}" + if self.status != "complete": + return f"{head}: {self.status}" + body = " | ".join( + "{" + ", ".join(f"{label(u)} -> {label(v)}" for u, v in sorted(arcs)) + "}" for arcs in self.candidates + ) + return f"{head}: {len(self)} explanation{'' if len(self) == 1 else 's'}: {body}" + + +def explanation_family(decisions, *, convention="neither", tiers=None, forbidden=()): + """ + Every DAG on the decisions' nodes that weaker adjacency faithfulness allows. + + A DAG qualifies when it is :cite:`Marx2021weaker,Hourican2026` + + * **Markov:** every statement it implies by d-separation was decided independent; + * **weakly adjacency faithful:** at each endpoint ``x`` of each edge ``x - w``, either + ``x`` and ``w`` are dependent given every conditioning set, or they form a strict + two-association with a third member ``z`` of ``x``'s Markov blanket: ``x, w``, ``x, z`` + and ``w, z`` are each dependent given the third and any further conditioning set, while + (``convention='neither'``) neither ``x, w`` nor ``x, z`` is dependent given every set, + or (``'not_both'``) not both are; + * **SGS-minimal** :cite:`Spirtes2000`: no DAG with one arc fewer is Markov; + * **consistent with background knowledge:** no arc from a later tier to an earlier one, + and no forbidden arc. + + Parameters + ---------- + decisions : CIDecisions + At most :data:`MAX_NODES` nodes. + convention : {'neither', 'not_both'} + tiers : dict or sequence, None + A tier per node label (a sequence follows ``decisions.nodes``); arcs may not point to + an earlier tier. Nodes without a tier are unconstrained. + forbidden : iterable of (source, target) + Arcs (node labels) that no explanation may contain. + + Returns + ------- + ExplanationFamily + """ + if convention not in ("neither", "not_both"): + raise ValueError("convention must be 'neither' or 'not_both'") + nodes = tuple(decisions.nodes) + n = len(nodes) + if n > MAX_NODES: + raise ValueError(f"explanation families are enumerated on at most {MAX_NODES} nodes, not {n}") + table = _dag_table(n) + dependent = np.array([not decisions.independent[s] for s in statements(n)], dtype=bool) + markov = ~(table.separated & dependent).any(axis=1) + reducible = np.where(table.without >= 0, markov[table.without], False).any(axis=1) + keep = markov & ~reducible & _permitted(table, nodes, tiers, forbidden) + a1, s2 = _associations(decisions, convention) + candidates = [ + frozenset((nodes[u], nodes[v]) for u, v in zip(*np.nonzero(table.arcs[d]), strict=True)) + for d in np.flatnonzero(keep) + if _weakly_faithful(table.arcs[d], a1, s2) + ] + return ExplanationFamily( + nodes=nodes, + candidates=tuple(sorted(candidates, key=sorted)), + status="complete" if candidates else "empty", + convention=convention, + decisions=decisions, + ) diff --git a/infoflow/network.py b/infoflow/network.py index 7145b08..73bed50 100644 --- a/infoflow/network.py +++ b/infoflow/network.py @@ -66,6 +66,10 @@ class MultiplexNetwork: backbone : BackboneResult, None Separate precomputed spectral structural evidence. Its neighbors guide the lagged parent search; its p-values/rates never replace flow-layer statistics. + contemporaneous_families : list of ExplanationFamily, None + Every lag-0 explanation allowed per component + (:func:`~infoflow.contemporaneous.contemporaneous_families`), with + ``contemporaneous="family"``. """ dataset: xr.Dataset @@ -77,6 +81,7 @@ class MultiplexNetwork: machines: "MachineLayer | None" = None channels: "ChannelLayer | None" = None backbone: "BackboneResult | None" = None + contemporaneous_families: list | None = None @property def names(self): @@ -158,6 +163,9 @@ def summary(self, show_nonsignificant=False): kind = str(self.dataset["kind"].sel(source=src, target=tgt).item()) delay = int(self.dataset["delay"].sel(source=src, target=tgt)) lines.append(f"{src:>6} -> {tgt:<6} {kind:<16} {delay:5d} " + " ".join(cells)) + if self.contemporaneous_families: + lines += ["", "lag-0 explanation families"] + lines += [" " + family.describe(self.names) for family in self.contemporaneous_families] if self.machines is not None: lines += ["", self.machines.summary()] if self.channels is not None: @@ -284,6 +292,7 @@ def infer_multiplex( node_layer=True, interpret=True, contemporaneous=False, + contemporaneous_settings=None, hyperedges=False, hyperedge_measure="ccs", max_sources=3, @@ -348,9 +357,17 @@ def infer_multiplex( Add active information storage and predictability flags per node. interpret : bool Add edge roles (direct / confounded / mediated) and latent-confounding flags. - contemporaneous : bool + contemporaneous : bool or {'pc', 'family'} Add the optional lag-0 layer (:mod:`infoflow.contemporaneous`) as the ``contemporaneous`` variable (0 none, 1 undirected, 2 oriented source -> target). + ``True`` or ``'pc'`` reads adjacencies from dependence and orients colliders, PC-style. + ``'family'`` keeps every explanation that weaker adjacency faithfulness allows + (:func:`~infoflow.contemporaneous.contemporaneous_families`): ``contemporaneous`` + then marks only what every explanation shares, ``contemporaneous_possible`` what any + explanation has, and ``network.contemporaneous_families`` lists the explanations. + contemporaneous_settings : dict, None + Keyword arguments for :func:`~infoflow.contemporaneous.contemporaneous_families` + (e.g. ``max_family_size``, ``tiers``, ``forbidden``, ``test``). hyperedges : bool Detect and classify PID hyperedges of each target's sets of parents (:func:`~infoflow.hyperedges.multivariate_hyperedges`) as ``network.hyperedges``. @@ -455,6 +472,8 @@ def infer_multiplex( raise ValueError("layer_symbols must be 'preprocessed' or 'adaptive'") if layer_method not in ("crossfit", "bayes", "auto"): raise ValueError("layer_method must be 'crossfit', 'bayes', or 'auto'") + if contemporaneous not in (False, True, "pc", "family"): + raise ValueError("contemporaneous must be True, False, 'pc', or 'family'") if layer_method != "crossfit": from .bayes import _pymc @@ -610,7 +629,18 @@ def compute(): dataset["iie"] = (("source", "target"), iie) dataset["iie_pvalue"] = (("source", "target"), iie_p) dataset["di"] = (("source", "target"), te + iie) - if contemporaneous: + lag0_families = None + if contemporaneous == "family": + from .contemporaneous import contemporaneous_families + + lag0 = contemporaneous_families( + estimation_data, skeleton, alpha=alpha, prng=rng, exchange=exchange, **(contemporaneous_settings or {}) + ) + lag0_families = lag0.families + dataset["contemporaneous"] = (("source", "target"), lag0.graph) + dataset["contemporaneous_cmi"] = (("source", "target"), lag0.values) + dataset["contemporaneous_possible"] = (("source", "target"), lag0.possible) + elif contemporaneous: from .contemporaneous import contemporaneous_layer graph, values, _ = contemporaneous_layer(estimation_data, skeleton, alpha=alpha, prng=rng, exchange=exchange) @@ -640,6 +670,7 @@ def compute(): "node_layer": node_layer, "interpret": interpret, "contemporaneous": contemporaneous, + "contemporaneous_settings": contemporaneous_settings, "directed_information": directed_information, "hyperedges": hyperedges, "layer_symbols": layer_symbols, @@ -687,6 +718,7 @@ def compute(): report=report, hyperedges=pid, backbone=backbone, + contemporaneous_families=lag0_families, ) if machines: from .machines import infer_machines diff --git a/tests/test_families.py b/tests/test_families.py new file mode 100644 index 0000000..f338060 --- /dev/null +++ b/tests/test_families.py @@ -0,0 +1,352 @@ +""" +Tests for explanation families and the lag-0 family layer. +""" + +import itertools + +import dit +import networkx as nx +import numpy as np +import pytest + +from infoflow.contemporaneous import contemporaneous_families, contemporaneous_layer +from infoflow.data import DiscreteData +from infoflow.embedding import Embedding +from infoflow.families import ( + MAX_NODES, + CIDecisions, + ExplanationFamily, + _dag_table, + decisions_from_distribution, + explanation_family, + statements, +) +from infoflow.selection import SkeletonSettings, infer_skeleton + +FAST = SkeletonSettings( + n_perm_max_stat=50, n_perm_min_stat=50, n_perm_omnibus=100, n_perm_max_seq=100, n_perm_pairs=50, n_perm_tdmi=50 +) + +X, Y, Z, W = range(4) + + +def _b(p, bit): + return p if bit else 1 - p + + +def _law(n, weight): + """ + A binary dit distribution over `n` variables with probabilities proportional to `weight`. + """ + outcomes = [o for o in itertools.product((0, 1), repeat=n) if weight(o) > 0] + pmf = np.array([weight(o) for o in outcomes]) + return dit.Distribution(outcomes, list(pmf / pmf.sum())) + + +# The mechanisms of Hourican & Quax (2026), App. D; "flip" is an independent error added mod 2. +LAWS = { + "xor": _law(3, lambda o: _b(0.1, o[2] ^ o[0] ^ o[1])), + "xor + independent": _law(4, lambda o: _b(0.1, o[2] ^ o[0] ^ o[1]) * _b(1 / 3, o[3])), + "biased xor": _law(3, lambda o: _b(1 / 3, o[0]) * _b(0.1, o[2] ^ o[0] ^ o[1])), + "xor + descendant": _law(4, lambda o: _b(0.1, o[2] ^ o[0] ^ o[1]) * _b(0.1, o[3] ^ o[2])), + "chain": _law(4, lambda o: _b(1 / 3, o[0]) * _b(0.1, o[1] ^ o[0]) * _b(0.1, o[2] ^ o[1]) * _b(0.1, o[3] ^ o[2])), + "or": _law(4, lambda o: _b(0.1, o[2] ^ (o[0] | o[1])) * _b(1 / 3, o[3])), + "asymmetric": _law( + 4, lambda o: _b(0.35, o[0]) * _b(0.6, o[1]) * _b((0.1, 0.9, 0.8, 0.2)[2 * o[0] + o[1]], o[2]) * _b(1 / 3, o[3]) + ), + "independent baseline": _law(4, lambda o: _b(0.1, o[2] ^ o[0] ^ o[1]) * _b(0.1, o[3] ^ o[1])), + "dependent baseline": _law(4, lambda o: _b(0.2, o[1] ^ o[0]) * _b(0.1, o[2] ^ o[0] ^ o[1]) * _b(0.1, o[3] ^ o[0])), +} + +COLLIDERS = {frozenset({(X, Z), (Y, Z)}), frozenset({(X, Y), (Z, Y)}), frozenset({(Y, X), (Z, X)})} +XOR_DESCENDANT = { + frozenset({(X, Y), (Z, Y), (Z, W)}), + frozenset({(X, Y), (Z, Y), (W, Z)}), + frozenset({(X, Z), (Y, Z), (Z, W)}), + frozenset({(Y, X), (Z, X), (Z, W)}), + frozenset({(Y, X), (Z, X), (W, Z)}), +} +CHAIN = { + frozenset({(X, Y), (Y, Z), (Z, W)}), + frozenset({(Y, X), (Y, Z), (Z, W)}), + frozenset({(Y, X), (Z, Y), (Z, W)}), + frozenset({(Y, X), (Z, Y), (W, Z)}), +} +COLLIDER_AT_Z = {frozenset({(X, Z), (Y, Z)})} + + +# -- the DAG table ------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("n", "count"), [(1, 1), (2, 3), (3, 25), (4, 543), (5, 29281)]) +def test_dag_counts(n, count): + table = _dag_table(n) + assert len(table.arcs) == count + assert table.separated.shape == (count, len(statements(n))) + + +def test_dag_table_size_limit(): + with pytest.raises(ValueError, match="1 to 5"): + _dag_table(MAX_NODES + 1) + + +@pytest.mark.parametrize("n", [3, 4, 5]) +def test_separation_matches_networkx(n): + d_separated = getattr(nx, "is_d_separator", None) or nx.d_separated + table = _dag_table(n) + rng = np.random.default_rng(n) + rows = range(len(table.arcs)) if n < 5 else rng.choice(len(table.arcs), 150, replace=False) + for d in rows: + g = nx.DiGraph() + g.add_nodes_from(range(n)) + g.add_edges_from((int(u), int(v)) for u, v in zip(*np.nonzero(table.arcs[d]), strict=True)) + for k, (i, j, S) in enumerate(statements(n)): + assert table.separated[d, k] == d_separated(g, {i}, {j}, set(S)) + + +def test_without_removes_one_arc(): + n = 4 + table = _dag_table(n) + for d in range(len(table.arcs)): + for u, v in zip(*np.nonzero(table.arcs[d]), strict=True): + fewer = table.arcs[d].copy() + fewer[u, v] = False + assert (table.arcs[table.without[d, u * n + v]] == fewer).all() + assert (table.without[~table.arcs.reshape(len(table.arcs), -1)] == -1).all() + + +# -- exact families ------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + ("name", "tiers", "convention", "expected"), + [ + ("xor", None, "neither", COLLIDERS), + ("xor + independent", None, "neither", COLLIDERS), + ("biased xor", None, "neither", set()), + ("biased xor", None, "not_both", 4), + ("xor + descendant", None, "neither", XOR_DESCENDANT), + ("xor + descendant", (0, 0, 1, 2), "neither", {frozenset({(X, Z), (Y, Z), (Z, W)})}), + ("chain", None, "neither", CHAIN), + ("or", None, "neither", COLLIDER_AT_Z), + ("or", None, "not_both", 5), + ("asymmetric", None, "neither", COLLIDER_AT_Z), + ("asymmetric", None, "not_both", 5), + ("independent baseline", None, "neither", 5), + ("independent baseline", (0, 0, 1, 1), "neither", {frozenset({(X, Z), (Y, Z), (Y, W)})}), + ("dependent baseline", None, "neither", set()), + ("dependent baseline", None, "not_both", 6), + ("dependent baseline", (0, 0, 1, 1), "not_both", 2), + ], +) +def test_population_families(name, tiers, convention, expected): + """ + Every reference family of Hourican & Quax (2026), App. A and Sec. 3, under both conventions. + """ + family = explanation_family(decisions_from_distribution(LAWS[name]), convention=convention, tiers=tiers) + if isinstance(expected, int): + assert len(family) == expected + else: + assert set(family.candidates) == expected + assert family.status == ("complete" if len(family) else "empty") + + +def _minimal_imap(decisions, order): + """ + The minimal I-map for an order: each earlier node dependent on x given the rest of x's predecessors. + """ + arcs = set() + for k, x in enumerate(order): + before = set(order[:k]) + arcs |= {(u, x) for u in before if not decisions.is_independent(x, u, before - {u})} + return frozenset(arcs) + + +@pytest.mark.parametrize("name", ["xor + independent", "xor + descendant", "chain", "or", "asymmetric"]) +def test_every_explanation_is_a_minimal_imap_of_an_order(name): + """ + For positive laws every SGS-minimal DAG is the minimal I-map of one of its orders. + """ + decisions = decisions_from_distribution(LAWS[name]) + imaps = {_minimal_imap(decisions, order) for order in itertools.permutations(range(4))} + family = explanation_family(decisions) + assert family.candidates and set(family.candidates) <= imaps + + +def test_family_summaries(): + xor = explanation_family(decisions_from_distribution(LAWS["xor"])) + assert xor.compelled_arrows == frozenset() and xor.compelled_adjacencies == frozenset() + assert xor.possible_adjacencies == {frozenset(p) for p in itertools.combinations(range(3), 2)} + assert ( + xor.describe(names="XYZ") == "{X, Y, Z}: 3 explanations: {X -> Y, Z -> Y} | {X -> Z, Y -> Z} | {Y -> X, Z -> X}" + ) + assert [g.number_of_edges() for g in xor.graphs()] == [2, 2, 2] + chain = explanation_family(decisions_from_distribution(LAWS["chain"])) + assert chain.compelled_adjacencies == {frozenset({X, Y}), frozenset({Y, Z}), frozenset({Z, W})} + assert chain.compelled_arrows == frozenset() + gate = explanation_family(decisions_from_distribution(LAWS["or"])) + assert gate.compelled_arrows == {(X, Z), (Y, Z)} + assert gate.describe() == "{0, 1, 2, 3}: 1 explanation: {0 -> 2, 1 -> 2}" + assert ExplanationFamily((0, 1, 2), status="too_large").describe() == "{0, 1, 2}: too_large" + + +def test_background_knowledge_and_errors(): + decisions = decisions_from_distribution(LAWS["xor"]) + assert set(explanation_family(decisions, forbidden=[(X, Z)]).candidates) == COLLIDERS - COLLIDER_AT_Z + assert set(explanation_family(decisions, tiers={X: 0, Y: 0, Z: 1}).candidates) == COLLIDER_AT_Z + assert len(explanation_family(decisions, tiers={Z: 1})) == 3 # nodes without a tier are unconstrained + with pytest.raises(ValueError): + explanation_family(decisions, tiers=(0, 1)) + with pytest.raises(ValueError, match="convention"): + explanation_family(decisions, convention="both") + with pytest.raises(ValueError, match="no decision"): + CIDecisions((0, 1, 2), {}) + six = CIDecisions(tuple(range(6)), dict.fromkeys(statements(6), True)) + with pytest.raises(ValueError, match="at most 5"): + explanation_family(six) + assert decisions.is_independent(Z, X) == decisions.is_independent(X, Z) is True + assert decisions.is_independent(X, Z, [Y]) is False + + +# -- lag-0 families from data -------------------------------------------------------------- + + +def _flip(rng, n, p): + return (rng.random(n) < p).astype(int) + + +def _lag0(kind, n, seed): + """ + Static samples of the motifs, so every interaction is within one sample. + """ + rng = np.random.default_rng(seed) + x, y = rng.integers(0, 2, n), rng.integers(0, 2, n) + if kind == "xor": + columns = [x, y, x ^ y ^ _flip(rng, n, 0.1)] + elif kind == "xor + descendant": + z = x ^ y ^ _flip(rng, n, 0.1) + columns = [x, y, z, z ^ _flip(rng, n, 0.1)] + elif kind == "or": + columns = [x, y, (x | y) ^ _flip(rng, n, 0.1)] + else: + a = (rng.random(n) < 1 / 3).astype(int) + b = a ^ _flip(rng, n, 0.1) + c = b ^ _flip(rng, n, 0.1) + columns = [a, b, c, c ^ _flip(rng, n, 0.1)] + return DiscreteData.from_discrete(np.stack(columns, axis=1)) + + +def _fit(kind, n=4000, seed=0, **kwargs): + data = _lag0(kind, n, seed) + skeleton = infer_skeleton(data, Embedding(max_lag=1), settings=FAST, prng=seed) + return data, skeleton, contemporaneous_families(data, skeleton, prng=seed, **kwargs) + + +def _graph(n, undirected=(), directed=()): + g = np.zeros((n, n), dtype=int) + for u, v in undirected: + g[u, v] = g[v, u] = 1 + for u, v in directed: + g[u, v] = 2 + return g + + +@pytest.mark.parametrize( + ("kind", "expected", "graph"), + [ + ("xor", COLLIDERS, _graph(3)), + ("xor + descendant", XOR_DESCENDANT, _graph(4, undirected=[(Z, W)])), + ("or", COLLIDER_AT_Z, _graph(3, directed=[(X, Z), (Y, Z)])), + ("chain", CHAIN, _graph(4, undirected=[(X, Y), (Y, Z), (Z, W)])), + ], +) +def test_lag0_families_recover_the_population_family(kind, expected, graph): + """ + The graph marks only what every explanation shares; ``possible`` marks what any has. + """ + _, _, result = _fit(kind) + (family,) = result.families + assert family.status == "complete" and set(family.candidates) == expected + assert family.decisions is not None + assert family.decisions.level == pytest.approx(0.05 / len(statements(len(family.nodes)))) + assert (result.graph == graph).all() + assert result.possible.sum() == 2 * len(family.possible_adjacencies) + + +def test_pc_layer_misses_the_lag0_xor(): + data, skeleton, result = _fit("xor") + assert not contemporaneous_layer(data, skeleton, n_perm=99, prng=0)[0].any() + assert result.possible.sum() == 6 + + +def test_lag0_families_options(): + _, _, timed = _fit("xor + descendant", tiers=(0, 0, 1, 2)) + assert timed.families[0].candidates == (frozenset({(X, Z), (Y, Z), (Z, W)}),) + assert timed.graph[X, Z] == timed.graph[Y, Z] == timed.graph[Z, W] == 2 + _, _, asymptotic = _fit("xor", test="asymptotic") + assert set(asymptotic.families[0].candidates) == COLLIDERS + _, _, forbidden = _fit("xor", forbidden=[(X, Z)]) + assert set(forbidden.families[0].candidates) == COLLIDERS - COLLIDER_AT_Z + + +def test_lag0_families_fall_back_without_enumeration(): + data, skeleton, large = _fit("xor", max_family_size=2) + assert [f.status for f in large.families] == ["too_large"] + pc = contemporaneous_layer(data, skeleton, n_perm=200, prng=0)[0] + assert (large.graph == pc).all() + _, _, sparse = _fit("xor", min_cell_count=10**6) + assert [f.status for f in sparse.families] == ["unavailable"] + assert not sparse.possible.any() + + +def test_lag0_families_arguments(): + data = _lag0("xor", 500, 0) + skeleton = infer_skeleton(data, Embedding(max_lag=1), settings=FAST, prng=0) + for kwargs in ({"max_family_size": 6}, {"test": "exact"}, {"convention": "both"}, {"n_perm": 10}): + with pytest.raises(ValueError): + contemporaneous_families(data, skeleton, prng=0, **kwargs) + with pytest.raises(ValueError, match="cannot reach"): + contemporaneous_families(data, skeleton, prng=0, n_perm=50) # the 6 statements need 119 + + +def test_infer_multiplex_lag0_families(tmp_path): + from infoflow.network import MultiplexNetwork, infer_multiplex + + net = infer_multiplex( + _xor_columns(3000), + names=["X", "Y", "Z"], + max_lag=1, + skeleton_settings=FAST, + contemporaneous="family", + prng=0, + ) + (family,) = net.contemporaneous_families + assert set(family.candidates) == COLLIDERS + assert "contemporaneous_possible" in net.dataset and not net.dataset["contemporaneous"].values.any() + assert "{X, Y, Z}: 3 explanations" in net.summary() + assert net.settings["contemporaneous"] == "family" + net.save(tmp_path / "net.pkl") + assert MultiplexNetwork.load(tmp_path / "net.pkl").contemporaneous_families == net.contemporaneous_families + with pytest.raises(ValueError, match="contemporaneous"): + infer_multiplex(_xor_columns(500), max_lag=1, skeleton_settings=FAST, contemporaneous="lag0", prng=0) + + +def _xor_columns(n, seed=0): + rng = np.random.default_rng(seed) + x, y = rng.integers(0, 2, n), rng.integers(0, 2, n) + return np.stack([x, y, x ^ y ^ _flip(rng, n, 0.1)], axis=1) + + +@pytest.mark.slow +@pytest.mark.parametrize("kind", ["xor", "xor + descendant", "or", "chain"]) +def test_lag0_families_are_reliable(kind): + """ + With Bonferroni-corrected decisions the family equals the population family nearly always. + """ + expected = {"xor": COLLIDERS, "xor + descendant": XOR_DESCENDANT, "or": COLLIDER_AT_Z, "chain": CHAIN}[kind] + hits = 0 + for seed in range(20): + _, _, result = _fit(kind, n=2000, seed=100 + seed) + if result.families and set(result.families[0].candidates) == expected: + hits += 1 + assert hits >= 18