fix(parsers): tighten benchmark-found edge cases

Ambiguous archives and binary layouts stay on the SheetJS and legacy
Word routes instead of being refused; line-end hyphens are removed only
when the document shows the joined word; page numbers printed inside a
wide margin are dropped from a page's edge lines; time-of-day cells no
longer carry the 1899 epoch; table cells with several paragraphs keep a
space between them.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: waleedlatif1

apps/sim/lib/file-parsers/html-parser.test.ts

@@ -9,6 +9,15 @@ import { HtmlComplexityError, HtmlParser } from '@/lib/file-parsers/html-parser'
 
 const parser = new HtmlParser()
 
+describe('table cells with several paragraphs', () => {
+  it('separates block children inside a cell with a space', async () => {
+    const html = '<table><tr><td><p>Заказчик</p><p>Исполняющий</p></td><td>ok</td></tr></table>'
+    const result = await new HtmlParser().parseBuffer(Buffer.from(html))
+
+    expect(result.content).toContain('| Заказчик Исполняющий | ok |')
+  })
+})
+
 describe('HtmlParser', () => {
   it('reports empty input with the typed parser taxonomy', async () => {
     await expect(parser.parseBuffer(Buffer.alloc(0))).rejects.toMatchObject({

apps/sim/lib/file-parsers/html-parser.ts

@@ -105,6 +105,9 @@ export function assertHtmlStringWithinLimits(html: string): void {
 
 const NON_CONTENT_SELECTOR = 'script, style, noscript, meta, link, iframe, object, embed, svg'
 
+/** Block elements inside a table cell; `.text()` would otherwise glue their words together. */
+const CELL_BLOCK_SELECTOR = 'p, div, li, br, h1, h2, h3, h4, h5, h6, tr'
+
 /** mammoth renders a footnote's return link as `<a href="#footnote-ref-N">↑</a>`. */
 const FOOTNOTE_BACKLINK_SELECTOR = 'a[href^="#footnote-ref"]'
 
@@ -347,7 +350,9 @@ function processTable(
     const cells: string[] = []
 
     $row.find('td, th').each((_, cell) => {
-      const cellText = $(cell).text().replace(/\s+/g, ' ').trim()
+      const $cell = $(cell)
+      $cell.find(CELL_BLOCK_SELECTOR).after(' ')
+      const cellText = $cell.text().replace(/\s+/g, ' ').trim()
       cells.push(cellText || '')
     })
 

apps/sim/lib/file-parsers/pdf-furniture.test.ts

@@ -33,6 +33,25 @@ function texts(pages: PdfLine[][]): string[][] {
   return pages.map((lines) => lines.map((entry) => entry.text))
 }
 
+describe('page numbers outside the band', () => {
+  it('drops a folio that is the last line of a page even inside a wide margin', () => {
+    const page = (n: number) => ({
+      pageHeight: 842,
+      lines: [
+        { text: 'Body text of the page.', y: 700, height: 10 },
+        { text: String(n), y: 189, height: 10 },
+      ],
+    })
+    const result = suppressFurniture([page(1), page(2), page(3)])
+
+    expect(result.map((lines) => lines.map((line) => line.text))).toEqual([
+      ['Body text of the page.'],
+      ['Body text of the page.'],
+      ['Body text of the page.'],
+    ])
+  })
+})
+
 describe('suppressFurniture', () => {
   it('drops a header repeated on enough pages but keeps its first occurrence', () => {
     const pages = [1, 2, 3, 4].map((i) => page(i, { header: 'ACME Corp — Internal Use Only' }))

apps/sim/lib/file-parsers/pdf-furniture.ts

@@ -64,6 +64,9 @@ export function suppressFurniture(pages: readonly PdfPageLines[]): PdfLine[][] {
 
   pages.forEach((page, pageIndex) => {
     const seen = new Set<string>()
+    for (const index of edgeLineIndices(page)) {
+      if (isPageNumber(page.lines[index].text, pages.length)) drops[pageIndex].add(index)
+    }
     for (const group of bandGroups(page)) {
       if (isPageNumber(group.text, pages.length)) {
         for (const index of group.indices) drops[pageIndex].add(index)
@@ -137,6 +140,24 @@ export function furnitureThreshold(pageCount: number): number | undefined {
   return Math.max(MIN_FURNITURE_REPEATS, Math.ceil(FURNITURE_PAGE_FRACTION * pageCount))
 }
 
+/**
+ * The first and last non-blank lines of a page. A folio printed inside a wide
+ * margin sits outside the band, but it is still the edge of the page's text.
+ */
+function edgeLineIndices(page: PdfPageLines): number[] {
+  const indices: number[] = []
+  let first = -1
+  let last = -1
+  page.lines.forEach((line, index) => {
+    if (line.text.trim().length === 0) return
+    if (first === -1) first = index
+    last = index
+  })
+  if (first !== -1) indices.push(first)
+  if (last !== -1 && last !== first) indices.push(last)
+  return indices
+}
+
 /** Groups consecutive band lines that share a baseline into one furniture row. */
 function bandGroups(page: PdfPageLines): BandGroup[] {
   const { lines, pageHeight } = page

apps/sim/lib/file-parsers/pdf-lines.test.ts

@@ -4,6 +4,7 @@
 import { describe, expect, it } from 'vitest'
 import {
   collectCompounds,
+  collectWords,
   dominantLineHeight,
   joinLines,
   normalizePdfWhitespace,
@@ -113,14 +114,25 @@ describe('joinLines', () => {
   })
 
   describe('dehyphenation', () => {
-    it('removes a line-end hyphen when the next line continues the word', () => {
+    it('removes a line-end hyphen when the document shows the joined word', () => {
       const lines = paragraph(['archived by the Infra-', 'structure team.'], 627.4)
+      const words = collectWords([{ text: 'The Infrastructure team owns it.', height: BODY }])
 
-      expect(joinLines(lines, { headingMarkers: false })).toBe(
+      expect(joinLines(lines, { words, headingMarkers: false })).toBe(
         'archived by the Infrastructure team.'
       )
     })
 
+    it('keeps an unknown line-end hyphen rather than inventing a word', () => {
+      const lines = paragraph(['we ship high-', 'quality builds'], 627.4)
+      const words = collectWords(lines)
+
+      expect(joinLines(lines, { words, headingMarkers: false })).toBe(
+        'we ship high-quality builds'
+      )
+      expect(joinLines(lines, { headingMarkers: false })).toBe('we ship high-quality builds')
+    })
+
     it('keeps the hyphen when the compound appears intact elsewhere in the document', () => {
       const lines = paragraph(['we compare attention-', 'based models with others'], 700)
       const compounds = collectCompounds([{ text: 'Attention-based models win.', height: BODY }])

apps/sim/lib/file-parsers/pdf-lines.ts

@@ -41,6 +41,12 @@ export type PdfLineSeparator = '' | ' ' | '\n'
 export interface JoinLinesOptions {
   /** Lowercase `a-b` compounds seen intact in the document; a line break on their hyphen keeps it. */
   compounds?: ReadonlySet<string>
+  /**
+   * Lowercase words seen in the document. A line-end hyphen is dropped only when
+   * the joined word occurs elsewhere, so `high-` / `quality` keeps its hyphen
+   * while `Infra-` / `structure` rejoins when `Infrastructure` appears intact.
+   */
+  words?: ReadonlySet<string>
   /** Dominant body-text height for the document; enables heading markers and heading pitch scaling. */
   bodyHeight?: number
   /** Prefixes short, oversized lines with `## ` so Markdown-aware chunkers split on them. */
@@ -81,6 +87,7 @@ const SOFT_HYPHEN = '\u00AD'
 const TRAILING_HYPHEN = /(\p{L}+)-$/u
 const LEADING_LOWERCASE_WORD = /^(\p{Ll}\p{L}*)/u
 const INTACT_COMPOUND = /\p{L}+-\p{L}+/gu
+const WORD_TOKEN = /\p{L}{3,}/gu
 
 /**
  * Reads an item's placement, or undefined when the item is rotated, vertical,
@@ -194,6 +201,15 @@ export function collectCompounds(lines: Iterable<PdfLine>): Set<string> {
   return compounds
 }
 
+/** Lowercase words of three or more letters seen anywhere in the document. */
+export function collectWords(lines: Iterable<PdfLine>): Set<string> {
+  const words = new Set<string>()
+  for (const line of lines) {
+    for (const match of line.text.matchAll(WORD_TOKEN)) words.add(match[0].toLowerCase())
+  }
+  return words
+}
+
 /** Character-weighted modal line height across the document; 0 when unknown. */
 export function dominantLineHeight(lines: Iterable<PdfLine>): number {
   const weights = new Map<number, number>()
@@ -224,6 +240,7 @@ export function joinLines(lines: readonly PdfLine[], options: JoinLinesOptions =
   const bodyHeight = options.bodyHeight ?? 0
   const headingMarkers = options.headingMarkers ?? PDF_HEADING_MARKERS_ENABLED
   const compounds = options.compounds
+  const words = options.words
 
   let out = decorate(lines[0], bodyHeight, headingMarkers)
   for (let i = 1; i < lines.length; i++) {
@@ -234,7 +251,7 @@ export function joinLines(lines: readonly PdfLine[], options: JoinLinesOptions =
       continue
     }
     if (separator === '\n') {
-      const joined = dehyphenate(out, line.text, compounds)
+      const joined = dehyphenate(out, line.text, compounds, words)
       if (joined !== undefined) {
         out = joined
         continue
@@ -260,20 +277,25 @@ function decorate(line: PdfLine, bodyHeight: number, headingMarkers: boolean): s
 
 /**
  * Joins `next` onto `out` across a hyphen that ended the line, or undefined
- * when the break is not a hyphenation. A compound seen intact elsewhere in the
- * document keeps its hyphen.
+ * when the break is not a hyphenation. The hyphen is removed only when the
+ * document itself shows the joined word; a compound seen intact keeps it, and
+ * an unknown pair keeps it too, because `high-quality` split at a line end is
+ * far more common in real documents than a word the document never repeats.
  */
 function dehyphenate(
   out: string,
   next: string,
-  compounds: ReadonlySet<string> | undefined
+  compounds: ReadonlySet<string> | undefined,
+  words: ReadonlySet<string> | undefined
 ): string | undefined {
   const head = TRAILING_HYPHEN.exec(out)
   const tail = LEADING_LOWERCASE_WORD.exec(next)
   if (!head || !tail) return undefined
   const compound = `${head[1]}-${tail[1]}`.toLowerCase()
   if (compounds?.has(compound)) return out + next
-  return out.slice(0, -1) + next
+  const joined = `${head[1]}${tail[1]}`.toLowerCase()
+  if (words?.has(joined)) return out.slice(0, -1) + next
+  return out + next
 }
 
 function separatorBetween(

apps/sim/lib/file-parsers/pdf-parser-structure.test.ts

@@ -103,7 +103,7 @@ describe('PdfParser structure reconstruction', () => {
       'review the attached appendix before the next checkpoint, and the owning',
       'team retains sign-off authority for scope changes above five percent.',
       'Exceptions require written approval from a director or above. This',
-      'supersedes the guidance circulated on 14 March and applies immediately.',
+      'supersedes the guidance the Infrastructure team circulated on 14 March.',
     ]
     const p1Top = 676.6
     const p2Top = p1Top - 5 * PITCH - PARAGRAPH_GAP

apps/sim/lib/file-parsers/pdf-parser.ts

@@ -5,6 +5,7 @@ import { FileParserError } from '@/lib/file-parsers/errors'
 import { type PdfPageLines, suppressFurniture } from '@/lib/file-parsers/pdf-furniture'
 import {
   collectCompounds,
+  collectWords,
   dominantLineHeight,
   joinLines,
   normalizePdfWhitespace,
@@ -278,6 +279,7 @@ function assemblePages(pages: readonly PdfPageLines[], complete: boolean): strin
   const allLines = filteredPages.flat()
   const options = {
     compounds: collectCompounds(allLines),
+    words: collectWords(allLines),
     bodyHeight: dominantLineHeight(allLines),
   }
   const pageTexts: string[] = []

apps/sim/lib/file-parsers/sheet-display-text.test.ts

@@ -115,6 +115,11 @@ describe('isoDateText', () => {
   it('renders an invalid date as empty text', () => {
     expect(isoDateText(new Date(Number.NaN))).toBe('')
   })
+
+  it('renders a duration or time-of-day cell without the 1899 epoch date', () => {
+    expect(isoDateText(new Date(Date.UTC(1899, 11, 30, 0, 30, 0)))).toBe('00:30:00')
+    expect(isoDateText(new Date(Date.UTC(1899, 11, 31, 13, 5, 9)))).toBe('13:05:09')
+  })
 })
 
 describe('normalizeSheetDisplayText', () => {

apps/sim/lib/file-parsers/sheet-display-text.ts

@@ -25,6 +25,8 @@ interface CellLookup {
 export function isoDateText(date: Date): string {
   if (Number.isNaN(date.getTime())) return ''
   const iso = date.toISOString()
+  /** A serial below 1 is a duration or time of day; Excel shows it without the 1899 epoch date. */
+  if (date.getUTCFullYear() < 1900) return iso.slice(11, 19)
   return iso.endsWith('T00:00:00.000Z') ? iso.slice(0, 10) : iso.slice(0, 19)
 }