From 8b7aad37946635ea7b499f9b59d87650ff31a0cd Mon Sep 17 00:00:00 2001 From: Marvin Ma Date: Mon, 16 Nov 2026 08:25:26 +0900 Subject: [PATCH 1/2] A 4-digit phone on a large file is no placeholder, and a large generated file repeats no persona: a placeholder is held by at least ten times the field's records per value; each generated part after the first pairs every surname with another persona's given name Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012o8JNBKSY1aREPbMQDG8g2 --- docs/architecture.md | 4 ++-- docs/commands.md | 4 ++-- docs/data.md | 2 +- docs/lessons.md | 4 ++-- er/auto.py | 23 ++++++++++++++++++----- er/cli.py | 9 +++++---- er/run.py | 7 +++++-- tests/test_auto.py | 16 ++++++++++++++++ tests/test_keys.py | 9 +++++++++ 9 files changed, 60 insertions(+), 18 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 60f52b5..c1a5464 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -117,12 +117,12 @@ Every module sits in one layer: the tools (`nayose`: `er/cli.py`, its commands; ### 1 · Load and standardize · `O(N·L)` -Opens the input through `er.source`, the one door every input file goes through, people's answers and decisions included. A file given to nayose has been read as it comes before (`er.intake`: in its workspace part when it was loaded, else for the one command, in a folder of its own in the system's temporary folder, which needs room for the reader's Parquet of a large file given that way): a CSV, TSV or Parquet, each value decoded on its own, what loses nothing fixed and counted, what would be a guess proposed for people to accept, as the run's notes say. The door finds the Parquet files (a path, a glob, a folder, or a part's read files), checks that each is Parquet, reads several together by column name, and uses a column whose name differs from the mapping only in case, width or separators. Then it checks the columns against `[fields]`: a missing mandatory field stops the run, with suggestions from the file's values; a missing optional one turns off what reads it, and a file that turns off every rule or every candidate generator stops the run. Records it can't resolve are set aside, with the reason: no ID, or one ID holding different records. Finally, in chunks of 200,000 records, it builds each logical field from its columns, each record's kind ([7](#7--person-or-company)), and the normalized copies that keys and comparisons use: kana folded, legal forms removed, one key per address. Last, a value of an identifying field (`[fields] identifying`: a customer ID, one customer's each) held by 64 records or more is a placeholder, such as 00000000, not one customer's. So is a value of a field a comparison's first level tests for equality (the name, the phone, the birth date) that 64 customers or more hold and whose pairs outnumber those of all the values held by fewer: a placeholder name on thousands of records would otherwise be what the exact level's u describes, and every exact name would weigh what agreeing on it weighs. Copies of one customer's record (one identifying value) are no such crowd; a record without an identifying value is a customer of its own. Each is made missing in the field, every copy of it and every field built from the same columns on those records (a name read into `person_name`, `full_name` and `any_name`), so no rule, key, comparison, probe or lookup reads it as two records agreeing, and the summary names it. A run that told placeholders otherwise takes no increment until it runs again. +Opens the input through `er.source`, the one door every input file goes through, people's answers and decisions included. A file given to nayose has been read as it comes before (`er.intake`: in its workspace part when it was loaded, else for the one command, in a folder of its own in the system's temporary folder, which needs room for the reader's Parquet of a large file given that way): a CSV, TSV or Parquet, each value decoded on its own, what loses nothing fixed and counted, what would be a guess proposed for people to accept, as the run's notes say. The door finds the Parquet files (a path, a glob, a folder, or a part's read files), checks that each is Parquet, reads several together by column name, and uses a column whose name differs from the mapping only in case, width or separators. Then it checks the columns against `[fields]`: a missing mandatory field stops the run, with suggestions from the file's values; a missing optional one turns off what reads it, and a file that turns off every rule or every candidate generator stops the run. Records it can't resolve are set aside, with the reason: no ID, or one ID holding different records. Finally, in chunks of 200,000 records, it builds each logical field from its columns, each record's kind ([7](#7--person-or-company)), and the normalized copies that keys and comparisons use: kana folded, legal forms removed, one key per address. Last, a value of an identifying field (`[fields] identifying`: a customer ID, one customer's each) held by 64 records or more is a placeholder, such as 00000000, not one customer's. So is a value of a field a comparison's first level tests for equality (the name, the phone, the birth date) that 64 customers or more hold, held by at least ten times the field's records per value (its filled records over its distinct values: on a large file every value of a 4-digit phone is on a hundred records, and none is a placeholder), and whose pairs outnumber those of all the values held by fewer: a placeholder name on thousands of records would otherwise be what the exact level's u describes, and every exact name would weigh what agreeing on it weighs. Copies of one customer's record (one identifying value) are no such crowd; a record without an identifying value is a customer of its own. Each is made missing in the field, every copy of it and every field built from the same columns on those records (a name read into `person_name`, `full_name` and `any_name`), so no rule, key, comparison, probe or lookup reads it as two records agreeing, and the summary names it. A run that told placeholders otherwise takes no increment until it runs again. - **Why:** every later step compares values, so the same fact must be written the same way first. Chunks keep memory flat at any size. A mistake in the input either gets fixed and reported, or stops the run with what to fix (exit code 2): never a traceback. - **Why a reader before the door, and why on bytes:** a file from another system can mix encodings by column or by row, and a CSV reader that decodes the whole file drops or garbles the rows in another encoding. `er.intake` splits the file on its own bytes and decodes each value alone: its plain lines by Arrow's CSV parser, and the lines that hold a quote, a NUL or nothing by Python's `csv` over a latin-1 view (in which every byte is one character), from each one until a record ends before a plain line, so the records are the ones Python's `csv` alone gives (a file where more than one line in 50 holds a quote, or with a CR alone, is read by Python's `csv` throughout); delimiters and quotes are ASCII, which no UTF-8 or EUC-JP character holds; a CP932 character's second byte can be `|`, so a file delimited by `|` that is mostly CP932 is decoded whole first, as UTF-16 and JIS files are: more than half of the lines with non-ASCII bytes in its first MiB fail UTF-8, nine in ten of those reading as CP932 and not as EUC-JP. Not one whose head merely reads as CP932: UTF-8 and EUC-JP text nearly always does, and the whole file would be garbled. A reading two encodings allow is judged by a character bigram model of the reference names and addresses and by what its column holds, learned from the values read for sure. Only what loses nothing is applied by itself; a guess waits for people (`accept repairs`), so a value the engine compares is never one the reader invented. Reading a part again is a function of its files, `[input]` and the kinds accepted. `O(bytes)` in one pass, in chunks of 200,000 rows, each read by column: a chunk's values that reading keeps as they are (plain ASCII, Japanese UTF-8 with nothing to fix) settled in Arrow, each tested only as what it can be, and only the rest one by one; a CSV column whose fields are all UTF-8 becomes Arrow text of its own bytes at once, one that isn't is decoded field by field. About 10 s a million records of 25 columns from CSV, 9 s from Parquet. The first chunk teaches each column what it holds (its first 50,000 values), counted column by column; the bigram model and the reference names are built once each time their files or their code change and kept on disk (`~/.cache/nayose`), not once a process; `O(rows · width · C(width + extra − 1, extra))` more for the rows that don't line up, every way of joining their extra fields tried. A file where more than 1,000 records, and more than a tenth of those read, don't line up is refused, judged once a chunk is read or on the whole of a smaller file: its delimiter or its header is wrong. - **Code:** `Run.load` → `source.open_source` (reads only each file's schema), `source.suggest` (a fixed-size sample), `settings.fit_to_file` → `config.check_fields`, `restrict`; `Run._set_aside`; `Run._placeholders`; people's answers through `Run._answers`, `Run._given` → `source.answers`, and `source.known` (every ID a record of the file, the same check as decisions'); `kind.classifier`; `standardize.prepare` -- **Increment:** the batch only, `O(Δ·L)`. It first finishes any interrupted write. A record already stored is set aside, so a batch given twice adds nothing. The batch is read with the run's fields: one it lacks (a file without birth dates) is read as missing (`Run._lacks`), so it is compared as the run's records were. The stored records and the batch then form one table, `records`, which every later step reads. The placeholders the run read as missing (`state/placeholders.json`) are missing in the batch too; a value the batch takes past 64 records is from then on, and the summary says that pairs joined on it before stay until the run is run again. +- **Increment:** the batch only, `O(Δ·L)`. It first finishes any interrupted write. A record already stored is set aside, so a batch given twice adds nothing. The batch is read with the run's fields: one it lacks (a file without birth dates) is read as missing (`Run._lacks`), so it is compared as the run's records were. The stored records and the batch then form one table, `records`, which every later step reads. The placeholders the run read as missing (`state/placeholders.json`) are missing in the batch too; a value the batch takes past the count above is from then on, and the summary says that pairs joined on it before stay until the run is run again. ### 2 · Budget · `O(N)` per key diff --git a/docs/commands.md b/docs/commands.md index 6afb311..94e5ba7 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -230,7 +230,7 @@ Generate synthetic records into a workspace, then load them as a file. nayose generate records [--records N] [--difficulty easy|medium|hard] --workspace WORKSPACE [--format text|json] ``` -People and companies from the reference names, companies and towns (config/generator.toml), each under one or more records with typos, other notations, moves and missing values, as hard to match as --difficulty says ([difficulty]); written in parts of 50,000 entities, each with its own seed, exactly N records, then loaded as load records loads a file. The same size and difficulty give the same records, so a second time copies nothing. The workspace's mapping must read the generator's columns, as the project's does. Past 60,000 entities names repeat more than in a real file, so name keys make more pairs. O(N) to generate (about 100 s a million records), memory of 50,000 entities, then load records'. +People and companies from the reference names, companies and towns (config/generator.toml), each under one or more records with typos, other notations, moves and missing values, as hard to match as --difficulty says ([difficulty]); written in parts of 50,000 entities, each with its own seed, exactly N records, then loaded as load records loads a file. The same size and difficulty give the same records, so a second time copies nothing. The workspace's mapping must read the generator's columns, as the project's does. Each part after the first pairs every surname with another persona's given name, so no person repeats; past 31,596 companies their names repeat more than in a real file. O(N) to generate (about 100 s a million records), memory of 50,000 entities, then load records'. | Parameter | | |---|---| @@ -617,7 +617,7 @@ Run a generated file of the same size and shape, and measure what this machine n nayose rehearse records [INPUT] --workspace WORKSPACE [--pipeline PIPELINE] [--compute KEY=VALUE …] [--format text|json] ``` -The file's records and each field's share filled (no value kept); a file of that size and shape generated in parts of 50,000 entities, each field filled at the file's share, a field it lacks left out; that file resolved in a process of its own, under the same limits and the workspace's method (method.toml; a key on a name feature added to the workspace left out: the generated file holds none), its spill folder's size read every half second. Says the seconds of each step, the peak memory, the peak spill, the pairs, and the generated shape beside the file's; if the run stopped (memory, disk), why, as the real file would. Removes what it made; keeps rehearsal.json in the workspace. Past 60,000 entities names repeat more than in a real file, so name keys make more pairs: check records counts the file's own. O(N) to generate, O(a full run) to measure. +The file's records and each field's share filled (no value kept); a file of that size and shape generated in parts of 50,000 entities, each field filled at the file's share, a field it lacks left out; that file resolved in a process of its own, under the same limits and the workspace's method (method.toml; a key on a name feature added to the workspace left out: the generated file holds none), its spill folder's size read every half second. Says the seconds of each step, the peak memory, the peak spill, the pairs, and the generated shape beside the file's; if the run stopped (memory, disk), why, as the real file would. Removes what it made; keeps rehearsal.json in the workspace. Past 31,596 companies the generator's company names repeat more than in a real file, so name keys make more pairs: check records counts the file's own. O(N) to generate, O(a full run) to measure. | Parameter | | |---|---| diff --git a/docs/data.md b/docs/data.md index 0bdbfe1..fbf172d 100644 --- a/docs/data.md +++ b/docs/data.md @@ -55,7 +55,7 @@ Two traps in the sources: All parameters live in `config/generator.toml`: `[population]` (`er.generate.Spec`) and `[split]` (shares and seed). `er.generate.generate` builds the population, `check` asserts its invariants, and `scripts/generate.py` writes `medallion/bronze/.parquet` and `labels/.parquet` (`--entities` and `--seed` change the population; `--out` writes it there, unsplit, instead of the splits). - **Entities:** 40,000, 20% companies, and their household members (below). Each entity has `1 + Poisson(0.25)` records. - - **Persons** are sampled personas, each at most once, with realistic surname frequencies (サトウ, スズキ, タカハシ …). Their addresses lie in the persona's prefecture. Sampling with replacement would make clones (same name, same prefecture) that no matcher can tell apart, so same-name pairs come only from the persona file itself. + - **Persons** are sampled personas, each at most once, with realistic surname frequencies (サトウ, スズキ, タカハシ …). Their addresses lie in the persona's prefecture. Sampling with replacement would make clones (same name, same prefecture) that no matcher can tell apart, so same-name pairs come only from the persona file itself. A file of more than 50,000 entities is generated in parts (`er.auto.part_persons`): the first draws from the personas as they are, each later one from personas whose given names, with their sex and age, are shuffled among the surnames by the part's seed, so no persona repeats across parts either. - **Companies** are sampled real names with a legal form before or after the name, written as each record's system writes it: カブシキガイシャ, (カ), カ), カブシキガイシャ or left out; one record in ten adds a branch (ホンシャ, トウキョウシテン …). - **Addresses** are real towns with random 番/号, and a real postal code and JIS code. - **Multi-valued entities:** each entity owns `1 + Poisson(λ)` phones and addresses (λ = 2.0 for persons, 3.0 for companies: moves and new phones) and names (λ = 0.3 and 1.0: married names, other company names). Each record shows one of them. diff --git a/docs/lessons.md b/docs/lessons.md index 086824d..da58fbc 100644 --- a/docs/lessons.md +++ b/docs/lessons.md @@ -20,8 +20,8 @@ Each entry names a way an entity-resolution pipeline like this one goes wrong, a ## A generator that makes impossible twins - **Pitfall:** synthetic data holds pairs no matcher could separate, which count as false merges that are no fault of the matcher. They show as false pairs that cluster in one over-merged entity, or as a threshold that fails to transfer from cross-validation to val. -- **Why:** drawing personas with replacement makes entities with the same name and prefecture. When two of them also draw the same address, they differ only in CIF ID and phone. Real data wouldn't contain such a pair. -- **Guard:** the generator draws each persona at most once, and refuses a population larger than the persona file. Same-name pairs come only from the persona file itself. +- **Why:** drawing personas with replacement makes entities with the same name and prefecture. When two of them also draw the same address, they differ only in CIF ID and phone. Drawing every part of a large file from the same personas does the same across parts: at a million records, each full name in its prefecture belongs to about 15 unrelated people. Real data wouldn't contain such pairs. +- **Guard:** the generator draws each persona at most once, and refuses a population larger than the persona file. A larger file is made in parts, and each part after the first pairs every surname with another persona's given name (`er.auto.part_persons`), so no persona repeats across parts. Same-name pairs come only from the persona file itself. - **Check:** `reports/choose.md` prints the transfer check every round: at 6 bits, 0 false of 2,569 accepted pairs out-of-fold, and 0 of 918 on val. False pairs that cluster into one over-merged entity are a sign of one cause, not several. ## Levels without an estimate diff --git a/er/auto.py b/er/auto.py index 3c7d9be..0de1d54 100644 --- a/er/auto.py +++ b/er/auto.py @@ -8,8 +8,8 @@ folder on the same disk; the parent reads its size every half second (DuckDB deletes spill files as it goes, so only polling sees the peak); seconds and peak memory from the run's steps.json -O(N) to generate and O(a full run) to measure. Past 60,000 entities the generator's personas repeat, so names repeat -more than in a real file and name keys make more pairs: the result says so, and a check of the real file +O(N) to generate and O(a full run) to measure. Past 31,596 companies the generator's company names repeat, so they +repeat more than in a real file and name keys make more pairs: the result says so, and a check of the real file (check records) counts its pairs exactly. Auto mode (nayose try everything): the commands a person would run on a file, in order, each in a process of its own, @@ -53,6 +53,7 @@ import duckdb import numpy as np +import pandas as pd from er.config import InputError, Pipeline, lit from er.folder import method, runs, write_method @@ -66,8 +67,8 @@ CHUNK = 50_000 # entities a generated part: generation's memory, whatever the file's size PILOT = 5_000 # entities of a first, small draw: each field's share measured on it, then each rate set right POLL = 0.5 # seconds between two readings of the spill folder's size -LIMIT = ("past 60,000 entities the generator's people repeat, so names repeat more than in a real file and name keys make more " - "pairs: the real file's own pairs are in check records") +LIMIT = ("past 31,596 companies the generator's company names repeat, so they repeat more than in a real file and name keys make " + "more pairs: the real file's own pairs are in check records") SAMPLE = 20_000 # records whose values the report is checked against HIDDEN = "[redacted]" JAPANESE = re.compile(r"[\u3005\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9f\U00020000-\U0003134f]+") # kana, kanji, half-width kana @@ -155,7 +156,7 @@ def _parts(spec, entities: int, out: Path, ref: tuple, uid: str, done: Callable[ for i, start in enumerate(range(0, entities, CHUNK)): if limit is not None and records >= limit: break - g = generate(replace(spec, entities=min(CHUNK, entities - start), seed=spec.seed + i), *ref) + g = generate(replace(spec, entities=min(CHUNK, entities - start), seed=spec.seed + i), part_persons(ref[0], spec.seed, i), *ref[1:]) df = g.df.drop(columns=[c for c in unused if c in g.df.columns]).iloc[: None if limit is None else limit - records] rng = np.random.default_rng(spec.seed + 20_000 + i) for c, keep in (keep_built or {}).items(): @@ -168,6 +169,18 @@ def _parts(spec, entities: int, out: Path, ref: tuple, uid: str, done: Callable[ return records +def part_persons(persons: pd.DataFrame, seed: int, i: int) -> pd.DataFrame: + """The personas part i draws from: the reference's for the first part; for each later one, every surname (with its + prefecture) beside another persona's given name, sex and age, shuffled by the part's seed. Each part draws its + entities from all of them, so the same personas in every part would make each full name, in its prefecture, about + entities ÷ CHUNK unrelated people of a large file: namesakes no matcher can tell apart.""" + if i == 0: + return persons + out, given = persons.copy(), ["given", "given_kana", "sex", "age"] + out[given] = persons[given].to_numpy()[np.random.default_rng(seed + 30_000 + i).permutation(len(persons))] + return out + + def size_of(folder: Path) -> int: # the bytes under a folder now; a file DuckDB just removed is skipped total = 0 for root, _, files in os.walk(folder): diff --git a/er/cli.py b/er/cli.py index 61d7758..4a882bf 100644 --- a/er/cli.py +++ b/er/cli.py @@ -1241,8 +1241,9 @@ def open_ui(a: dict, emit: "Emit") -> dict: "more records with typos, other notations, moves and missing values, as hard to match as --difficulty says " "([difficulty]); written in parts of 50,000 entities, each with its own seed, exactly N records, then loaded as " "load records loads a file. The same size and difficulty give the same records, so a second time copies nothing. The " - "workspace's mapping must read the generator's columns, as the project's does. Past 60,000 entities names repeat " - "more than in a real file, so name keys make more pairs. O(N) to generate (about 100 s a million records), " + "workspace's mapping must read the generator's columns, as the project's does. Each part after " + "the first pairs every surname with another persona's given name, so no person repeats; past 31,596 companies their " + "names repeat more than in a real file. O(N) to generate (about 100 s a million records), " "memory of 50,000 entities, then load records'.", (Param("records", "how many (default 10000)", metavar="N"), Param("difficulty", "easy, medium or hard (default medium)", "choice", ("easy", "medium", "hard")), WORKSPACE), @@ -1368,8 +1369,8 @@ def open_ui(a: dict, emit: "Emit") -> dict: "added to the workspace left out: the generated file holds none), its spill folder's size read every half second. Says the seconds " "of each step, the peak memory, the peak spill, the pairs, and the generated shape beside the file's; if the run " "stopped (memory, disk), why, as the real file would. Removes what it made; keeps rehearsal.json in the workspace. " - "Past 60,000 entities names repeat more than in a real file, so name keys make more pairs: check records counts " - "the file's own. O(N) to generate, O(a full run) to measure.", + "Past 31,596 companies the generator's company names repeat more than in a real file, so name keys make more pairs: " + "check records counts the file's own. O(N) to generate, O(a full run) to measure.", (replace(INPUT, help=INPUT.help + ", its shape read, not copied in; without it, the file loaded into the workspace last", optional=True), WORKSPACE, PIPELINE, COMPUTE), {"ok": "whether the run went through", "records": "records generated", "generate_seconds": "seconds to generate and describe them", diff --git a/er/run.py b/er/run.py index 8bb31b1..68542a0 100644 --- a/er/run.py +++ b/er/run.py @@ -333,7 +333,9 @@ def blank(values: list[str], f: str) -> None: if before := self.placeholders.get(f, []): blank(before, f) who = f"coalesce({q(next((x for x in self.pipeline.fields.identifying if x in cols and x != f), self.uid))}, {q(self.uid)})" - held = self.con.execute(f"SELECT {q(f)}, count(*) FROM {table} WHERE {q(f)} IS NOT NULL GROUP BY 1 HAVING count(*) >= {m}" + filled, distinct = self.con.execute(f"SELECT count({q(f)}), count(DISTINCT {q(f)}) FROM {table}").fetchall()[0] + at = max(m, math.ceil(SPREAD * filled / max(distinct, 1))) # far past the field's own records per value: a 4-digit phone's values are no crowd however many records hold each + held = self.con.execute(f"SELECT {q(f)}, count(*) FROM {table} WHERE {q(f)} IS NOT NULL GROUP BY 1 HAVING count(*) >= {at}" f"{'' if f in ids else f' AND count(DISTINCT {who}) >= {crowd}'} ORDER BY count(*) DESC, 1").fetchall() if f not in ids and held: # a compared field: a crowd whose pairs outnumber all the others' (two placeholders never hide each other) every = self.con.execute(f"SELECT coalesce(sum(c * (c - 1) // 2), 0) FROM (SELECT count(*) AS c FROM {table} WHERE {q(f)} IS NOT NULL GROUP BY {q(f)})").fetchall()[0][0] @@ -345,7 +347,7 @@ def blank(values: list[str], f: str) -> None: continue blank(new, f) n = self.con.execute(f"SELECT count(*) FROM {table} WHERE {q(f)} IS NULL").fetchall()[0][0] - self.notes.append(f"{f}: {len(new):,} value{'s' if len(new) > 1 else ''} held by {m}+ records read as missing, a placeholder and not " + self.notes.append(f"{f}: {len(new):,} value{'s' if len(new) > 1 else ''} held by {at:,}+ records read as missing, a placeholder and not " f"{'one entity' if f in ids else 'evidence'}'s ({n:,} records without one)" + ("; pairs the run joined on it before stay until it runs again (nayose rerun records)" if also else "")) self.examples.append(f"{f} = {new[0]}: a placeholder, read as missing") self.placeholders[f] = [*before, *new] @@ -1171,6 +1173,7 @@ def stored_keys(ws: Path, p: Pipeline, s: Settings) -> Pipeline: # the keys the BLOCKS = "blocks.json" # in state/: each key's oversized blocks a full run listed, by value CROWD = 64 # the fewest customers a compared field's placeholder is held by, however small the budget's oversized blocks +SPREAD = 10 # a placeholder is held by this many times the field's records per value, or more PLACEHOLDERS = 2 # how placeholders are told (Run._placeholders): a run that told them otherwise takes no increment until it runs again diff --git a/tests/test_auto.py b/tests/test_auto.py index 5028317..b3de604 100644 --- a/tests/test_auto.py +++ b/tests/test_auto.py @@ -68,6 +68,22 @@ def test_a_generated_file_takes_the_shape_it_is_given_across_its_parts(tmp_path, assert 0.1 < auto.shape(load_settings(), tmp_path / "b")["fields"]["company_name"] < 0.35 # the generator's share of companies, not 90% +def test_the_parts_of_a_large_generated_file_draw_no_persona_twice(): + from er import auto + from er.generate import generate, load_reference, load_spec + from er.settings import load_settings + s = load_settings() + spec, _ = load_spec(s.root / "config" / "generator.toml") + persons, *rest = load_reference(s.paths["reference"]) + assert auto.part_persons(persons, spec.seed, 0) is persons # a file of one part is as it was + names = [] + for i in range(2): # two parts of 3,000 of 60,000 personas: about 150 drawn by both, were they the same personas + g = generate(replace(spec, entities=3000, seed=spec.seed + i, company_share=0.0), auto.part_persons(persons, spec.seed, i), *rest) + w = g.clean.merge(g.truth, on="CONT_ID").dropna(subset=["LAST_NAME_KANJI", "GIVEN_NAME_KANJI"]).drop_duplicates("ENTITY_ID") + names.append(set(w["LAST_NAME_KANJI"] + w["GIVEN_NAME_KANJI"])) + assert len(names[0] & names[1]) < 20 # chance namesakes only + + def test_generated_records_go_into_a_workspace_as_a_file_does_and_the_same_twice_copies_nothing(say, tmp_path): from er import folder assert say("create", "workspace", "w", "--home", tmp_path)[0] == 0 diff --git a/tests/test_keys.py b/tests/test_keys.py index 0f97eb7..fad525d 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -180,6 +180,15 @@ def test_copies_of_one_customer_s_record_are_no_crowd_and_a_known_placeholder_is assert run.con.execute("SELECT count(*) FROM records WHERE phone IS NOT NULL").fetchall()[0][0] == 0 +def test_a_short_phone_held_by_many_records_each_is_kept_and_a_dummy_far_past_them_is_still_read_as_missing(population): + s, p, g = population + uid = s.data.unique_id + phones = [f"{i % 100:04d}" for i in range(20_000)] + ["9999"] * 3_000 # 100 values, 200 customers each, as 4 digits on a large file; one dummy + rows = {uid: [f"r{i:05d}" for i in range(len(phones))], "customer_id": [None] * len(phones), "phone": phones} + run, out = _placeholders(population, rows) + assert run.placeholders == {"phone": ["9999"]} and out["phone"].notna().sum() == 20_000 + + def test_the_gap_counts_a_pair_as_the_rules_only_where_they_kept_it(): import duckdb from types import SimpleNamespace From 1b37dc480eb1d958425dd20a77002c33d16ba160 Mon Sep 17 00:00:00 2001 From: Marvin Ma Date: Mon, 16 Nov 2026 08:26:36 +0900 Subject: [PATCH 2/2] Placeholders are told another way, so a workspace run before asks for one rerun before an increment Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012o8JNBKSY1aREPbMQDG8g2 --- er/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/er/run.py b/er/run.py index 68542a0..f8f7017 100644 --- a/er/run.py +++ b/er/run.py @@ -1174,7 +1174,7 @@ def stored_keys(ws: Path, p: Pipeline, s: Settings) -> Pipeline: # the keys the BLOCKS = "blocks.json" # in state/: each key's oversized blocks a full run listed, by value CROWD = 64 # the fewest customers a compared field's placeholder is held by, however small the budget's oversized blocks SPREAD = 10 # a placeholder is held by this many times the field's records per value, or more -PLACEHOLDERS = 2 # how placeholders are told (Run._placeholders): a run that told them otherwise takes no increment until it runs again +PLACEHOLDERS = 3 # how placeholders are told (Run._placeholders): a run that told them otherwise takes no increment until it runs again FINGERPRINTED_ROWS = 30_000 # the fit's sample, as every run's fingerprint holds it: whatever a run samples, its increments take no rerun