fix(context): quoted allowed values, dotted names, explicit confidence phrases, strict protection after resolution

docs/context-policies.md

@@ -54,7 +54,20 @@ network call, and no randomness anywhere in this path.
 
 Confidence phrases are normalized: `>95%`, `confidence > 95%`,
 `confidence above 0.95`, and `only if confidence >= 95 percent` all compile to
-`0.95`.
+`0.95`. A number counts as a confidence only when a confidence word
+(`confidence`, `certainty`, `probability`, `sure`) introduces it or it carries
+`%`/`percent`; a condition such as `only if 3 neighbours agree` is reported as
+unparsed rather than read as a gate.
+
+Allowed-values lists are split on `,` and `;` when present, with `and`/`or`
+separating only the last item (`active, inactive or pending`); without commas,
+`and`/`or` separate the values. `/` never splits a value, so `N/A` stays whole.
+A value that contains `and`/`or` where a separator could be (for example
+`Chile or Trinidad and Tobago`) is ambiguous: quote it, as in
+`Allowed country values are Chile or 'Trinidad and Tobago'.`
+
+A period ends a sentence only before whitespace or the end of a line, so dotted
+column names (`file.name is unique.`) and decimals stay in one sentence.
 
 ## Column references resolve against your real schema
 

src/freshdata/context/compiler.py

@@ -389,6 +389,7 @@ def resolve_policy(
     schema = [str(c) for c in columns]
     constraints: list[ColumnConstraint] = []
     unresolved = list(policy.unresolved)
+    newly_resolved = False
     for c in policy.constraints:
         if c.rule == "dedup_key" and c.resolution_confidence == 0.0:
             resolved_cols: list[str] = []
@@ -414,6 +415,7 @@ def resolve_policy(
                     resolved_cols.append(str(resolution.column))
             if failed:
                 continue
+            newly_resolved = True
             constraints.append(
                 ColumnConstraint(
                     id=c.id,
@@ -450,6 +452,7 @@ def resolve_policy(
         constraint_params = dict(c.params)
         if resolution.method == "embedding":
             constraint_params["resolution_evidence"] = _resolution_evidence(resolution)
+        newly_resolved = True
         constraints.append(
             ColumnConstraint(
                 id=c.id,
@@ -463,12 +466,20 @@ def resolve_policy(
                 provenance=c.provenance,
             )
         )
+    issues = list(policy.issues)
+    if newly_resolved:
+        # Columns only just became known, so the passes compile_context runs on
+        # resolved columns (restatement, protection-vs-repair) run now; the
+        # caller's strict check then sees any protection_conflict.
+        new_issues: list[PolicyIssue] = []
+        constraints = _enforce_protection(_drop_superseded(constraints, new_issues), new_issues)
+        issues.extend(i for i in dict.fromkeys(new_issues) if i not in issues)
     return ContextPolicy(
         policy_version=policy.policy_version,
         dataset_domain=policy.dataset_domain,
         constraints=tuple(constraints),
         unresolved=tuple(unresolved),
-        issues=policy.issues,
+        issues=tuple(issues),
         source_text_sha256=policy.source_text_sha256,
         strict=policy.strict,
     )

src/freshdata/context/lexicon.py

@@ -173,30 +173,47 @@ def format_for(word: str) -> str | None:
 # Confidence phrases -> fraction in [0, 1].
 # ---------------------------------------------------------------------------
 
-_CONFIDENCE = re.compile(
-    r"""
-    (?:confidence|certainty|probability|sure(?:ness)?)?   # optional subject
-    \s*
+_COMPARATOR = r"(?:>=|>|≥|above|over|at\ least|exceeds|greater\ than|more\ than)?"
+_VALUE = r"(?P<value>\d+(?:\.\d+)?)"
+_PERCENT = r"(?P<pct>%|percent\b|per\ cent\b)"
+
+#: A value introduced by a confidence subject: "confidence above 0.95".
+_CONFIDENCE_SUBJECT = re.compile(
+    rf"""
+    \b(?:confidence|certainty|probability|sure(?:ness)?)
+    (?:\s+(?:level|score|threshold))?
+    \s*(?:[:=]\s*)?
     (?:is\s+|of\s+)?
-    (?:>=|>|≥|above|over|at\ least|exceeds|greater\ than|more\ than)?
+    {_COMPARATOR}
     \s*
-    (?P<value>\d+(?:\.\d+)?)
+    {_VALUE}
     \s*
-    (?P<pct>%|percent|per\ cent)?
+    {_PERCENT}?
     """,
     re.IGNORECASE | re.VERBOSE,
 )
+#: A value that is explicitly a percentage: ">95%", "at least 90 percent".
+_CONFIDENCE_PERCENT = re.compile(
+    rf"{_COMPARATOR}\s*{_VALUE}\s*{_PERCENT}",
+    re.IGNORECASE | re.VERBOSE,
+)
 
 
 def parse_confidence(text: str) -> float | None:
     """Extract a confidence threshold from prose, normalized to [0, 1].
 
+    A number counts only when a confidence word (confidence, certainty,
+    probability, sure) introduces it or it carries ``%``/``percent``; any
+    other number (``3 neighbours``, ``age > 50``) is not a confidence.
+
     >>> parse_confidence(">95%"), parse_confidence("confidence above 0.95")
     (0.95, 0.95)
     >>> parse_confidence("only if confidence >= 95 percent")
     0.95
+    >>> parse_confidence("3 neighbours agree") is None
+    True
     """
-    match = _CONFIDENCE.search(text)
+    match = _CONFIDENCE_SUBJECT.search(text) or _CONFIDENCE_PERCENT.search(text)
     if match is None:
         return None
     value = float(match.group("value"))

src/freshdata/context/normalize.py

@@ -69,11 +69,68 @@ def tokens(snake: str) -> tuple[str, ...]:
     return tuple(singular(t) for t in snake.split("_") if t)
 
 
+#: A quoted value that starts and ends at a token boundary, so an apostrophe
+#: inside a word (``Don't``) never opens a quote.
+_QUOTED_VALUE = re.compile(
+    r"(?<![^\s,;:(\[{])"
+    r"(?:\"[^\"]*\"|'[^']*'|`[^`]*`|“[^”]*”|‘[^’]*’)"
+    r"(?![^\s,;:.)\]}])"
+)
+_PLACEHOLDER = re.compile("\x00(\\d+)\x00")
+_LIST_SEPARATOR = re.compile(r"[,;]")
+#: ``and``/``or`` between two items (lower-case only, so ``OR`` stays a value).
+_CONJUNCTION = re.compile(r"\s+(?:and|or)\s+")
+_LEADING_CONJUNCTION = re.compile(r"^(?:and|or)\s+")
+#: The last ``and``/``or`` in an item (greedy head), for ``a, b or c``.
+_TRAILING_CONJUNCTION = re.compile(r"^(?P<head>.*\S)\s+(?:and|or)\s+(?P<tail>\S.*)$")
+
+
 def split_value_list(raw: str) -> tuple[str, ...]:
-    """Split ``"active, inactive or pending"`` into cleaned value tokens."""
-    cleaned = re.sub(r"[{}\[\]()]", "", raw)
-    parts = re.split(r",|;|\bor\b|\band\b|/", cleaned)
-    return tuple(p for p in (strip_quotes(x) for x in parts) if p)
+    """Split ``"active, inactive or pending"`` into cleaned value tokens.
+
+    Splitting rules, applied in order:
+
+    - A quoted value (``'N/A'``, ``"Trinidad and Tobago"``) is always kept
+      whole, and ``/`` never separates values.
+    - When the list contains ``,`` or ``;``, only those separate values; the
+      last item may still split once on a trailing ``and``/``or``
+      (``a, b or c``), and a leading ``and``/``or`` (``a, b, and c``) is
+      dropped.
+    - Without ``,``/``;``, ``and``/``or`` separate values
+      (``email and phone``).
+
+    A value that itself contains ``and``/``or`` in the position of a list
+    separator is ambiguous; quote it.
+
+    >>> split_value_list("Trinidad and Tobago, Chile, N/A")
+    ('Trinidad and Tobago', 'Chile', 'N/A')
+    >>> split_value_list("active, inactive or pending")
+    ('active', 'inactive', 'pending')
+    >>> split_value_list("'Bosnia and Herzegovina' or Chile")
+    ('Bosnia and Herzegovina', 'Chile')
+    """
+    quoted: list[str] = []
+
+    def _stash(match: re.Match[str]) -> str:
+        quoted.append(match.group(0)[1:-1].strip())
+        return f"\x00{len(quoted) - 1}\x00"
+
+    text = re.sub(r"[{}\[\]()]", "", _QUOTED_VALUE.sub(_stash, raw))
+    if _LIST_SEPARATOR.search(text):
+        parts = [p.strip() for p in _LIST_SEPARATOR.split(text)]
+        last = _LEADING_CONJUNCTION.sub("", parts.pop())
+        split_last = _TRAILING_CONJUNCTION.match(last)
+        if split_last is None:
+            parts.append(last)
+        else:
+            parts.extend([split_last.group("head"), split_last.group("tail")])
+    else:
+        parts = _CONJUNCTION.split(text)
+
+    def _restore(part: str) -> str:
+        return _PLACEHOLDER.sub(lambda m: quoted[int(m.group(1))], strip_quotes(part))
+
+    return tuple(v for v in (_restore(p) for p in parts) if v)
 
 
 def parse_scalar(raw: str) -> object:

src/freshdata/context/parser.py

@@ -22,8 +22,9 @@
 _NUM = r"-?\d+(?:\.\d+)?"
 
 _BULLET = re.compile(r"^\s*(?:[-*•]|\d+[.)])\s+")
-#: Sentence enders; a period *between* digits (a decimal like 0.95) never splits.
-_SPLIT = re.compile(r"[!?;]|(?<=\D)\.|\.(?=\D|$)")
+#: Sentence enders. A period ends a sentence only before whitespace or the end
+#: of the line, so decimals (``0.95``) and dotted names (``file.name``) survive.
+_SPLIT = re.compile(r"[!?;]|\.(?=\s|$)")
 _WS = re.compile(r"\s+")