diff --git a/plugins/grobid/README.md b/plugins/grobid/README.md index 415302e..94b9996 100644 --- a/plugins/grobid/README.md +++ b/plugins/grobid/README.md @@ -18,7 +18,7 @@ Reviewers can download a complete training package for a collection as a ZIP fil Segmentation documents mark the start of every PDF page with `` (`n`: the page number in the PDF, from 1), placed at the start of the page: before the zone that holds the first word of the page if the page starts with that zone, otherwise before that word inside its zone. The extractor inserts them from the page starts of GROBID's feature file, unless GROBID already put a `` within three lines of a page start. GROBID's trainer ignores ``, so the training data is unchanged. The pages of the unit review and the LLM review come from the `` elements, so a document keeps its pages when the extraction cache is gone; the layout hints for the LLM still come from the feature file. -For documents extracted before, the TEI Wizard enhancement **Insert page breaks** adds the missing `` elements from the cached feature file (fetch it with [Reload feature file](#reload-feature-file) if it is missing) and shows the result in the merge view. Running it again adds nothing to pages that have a ``. When the feature file is cached, the sync-check lint warns about a page start without a `` nearby, a `` that is not near a page start, and a `` whose `n` is not its page number. Documents without any `` get no such warnings. +For documents extracted before, the TEI Wizard enhancement **Insert page breaks** adds the missing `` elements from the cached feature file (fetch it with [Reload feature file](#reload-feature-file) if it is missing) and shows the result in the merge view. Running it again adds nothing to pages that have a ``. When the feature file is cached, the sync-check lint warns about a `` that is not near a page start and a `` whose `n` is not its page number. Missing `` elements get no warning. The node navigation of the XML editor offers **Page (``)** for segmentation documents: it moves from page to page, focus mode focuses the whole page (from its `` to the next one), and the **LLM Review** button of focus mode reviews that page. @@ -220,7 +220,7 @@ The resulting ZIP mirrors `grobid-trainer/resources/dataset/` and can be unpacke 3. `#diffTokens` performs a forward-scan diff with a lookahead window of 10 tokens to recover alignment after a mismatch. It emits: - `warning` — when the feature file has tokens absent from the XML (missing tokens) or the XML has extra tokens not in the feature file. - `error` — when neither side recovers alignment within the window. -4. For segmentation documents with `` elements, `pageBreakDiagnostics()` aligns the lines with the feature rows (the forward scan of `layout.py`) and warns about page starts without a `` within `PAGE_BREAK_VICINITY` (3) lines, `` elements without a page start within that distance, and `` values that differ from the page number. `pageBreaksOf()` assigns each `` to the first line whose first word follows it. Both are exported and unit-tested in `tests/grobid-sync.test.js`. +4. For segmentation documents with `` elements, `pageBreakDiagnostics()` aligns the lines with the feature rows (the forward scan of `layout.py`) and warns about `` elements without a page start within `PAGE_BREAK_VICINITY` (3) lines and `` values that differ from the page number. `pageBreaksOf()` assigns each `` to the first line whose first word follows it. Both are exported and unit-tested in `tests/grobid-sync.test.js`. **Tokenisation rules:** a token is either a maximal run of non-delimiter, non-whitespace characters, or a single non-whitespace delimiter character, where the delimiter set matches `TextUtilities.fullPunctuations` from `grobid-core`. XML entity references (`&`, `<`, `<`) are decoded before tokenisation so that `&` produces one token `&` and `<www` produces one token `` elements that disagree with the page starts of the feature file: a page - * start without a `` within PAGE_BREAK_VICINITY lines, a `` without a page start within - * that distance, and a `` whose `n` is not the number of its page. Documents without `` - * get none. + * Diagnostics for `` elements that disagree with the page starts of the feature file: a + * `` without a page start within PAGE_BREAK_VICINITY lines, and a `` whose `n` is not the + * number of its page. A page start without a `` gets none. * @param {LineToken[]} lines - The line tokens of the document * @param {PageBreakTag[]} pageBreaks - Its `` elements * @param {string[]} featureTokens - First tokens of the feature file rows @@ -136,16 +135,6 @@ export function pageBreakDiagnostics(lines, pageBreaks, featureTokens, pageStart }); /** @type {Diagnostic[]} */ const diagnostics = []; - for (const start of starts) { - if (pageBreaks.some(pb => Math.abs(pb.line - start.line) <= PAGE_BREAK_VICINITY)) continue; - const at = /** @type {LineToken} */ (lines[start.line]); - diagnostics.push({ - from: at.from, - to: at.to, - severity: 'warning', - message: `Page ${start.page} of the PDF starts here, but there is no (TEI Wizard: "Insert page breaks")` - }); - } for (const pb of pageBreaks) { const near = starts .filter(s => Math.abs(s.line - pb.line) <= PAGE_BREAK_VICINITY) diff --git a/plugins/grobid/src/pte_plugin_grobid/tests/grobid-sync.test.js b/plugins/grobid/src/pte_plugin_grobid/tests/grobid-sync.test.js index 449e61a..fd7c51b 100644 --- a/plugins/grobid/src/pte_plugin_grobid/tests/grobid-sync.test.js +++ b/plugins/grobid/src/pte_plugin_grobid/tests/grobid-sync.test.js @@ -64,14 +64,11 @@ describe('grobid-sync page breaks', () => { assert.deepStrictEqual(pageBreakDiagnostics(nearLines, pageBreaksOf(near, nearLines), FEATURE_TOKENS, PAGE_STARTS), []); }); - it('reports missing, misplaced and misnumbered page breaks', () => { + it('reports misplaced and misnumbered page breaks, but not missing ones', () => { const lines = lineTokens(DOC); const breaks = pageBreaksOf(DOC, lines); // no for page 2 - const missing = pageBreakDiagnostics(lines, [breaks[0]], FEATURE_TOKENS, PAGE_STARTS); - assert.strictEqual(missing.length, 1); - assert.match(missing[0].message, /Page 2 of the PDF starts here/); - assert.strictEqual(missing[0].from, lines[4].from); + assert.deepStrictEqual(pageBreakDiagnostics(lines, [breaks[0]], FEATURE_TOKENS, PAGE_STARTS), []); // a far from any page start (with more lines in between) const far = pageBreakDiagnostics(lines, [...breaks, { from: 1, to: 2, n: null, line: 20 }], FEATURE_TOKENS, PAGE_STARTS); assert.deepStrictEqual(far.map(d => d.message), ['This is not at a page start of the PDF (feature file)']);