Merge branch 'fix/parsers-pdf-structure' into eval/kb-parser-quality

Author: waleedlatif1

apps/sim/lib/file-parsers/pdf-furniture.test.ts

@@ -0,0 +1,193 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  furnitureThreshold,
+  isPageNumber,
+  normalizeFurnitureText,
+  type PdfPageLines,
+  suppressFurniture,
+} from '@/lib/file-parsers/pdf-furniture'
+import type { PdfLine } from '@/lib/file-parsers/pdf-lines'
+
+const PAGE_HEIGHT = 792
+
+function line(text: string, y: number): PdfLine {
+  return { text, y, height: 11 }
+}
+
+/** A page with a top header, a bottom footer, and two body lines. */
+function page(index: number, options: { header?: string; footer?: string } = {}): PdfPageLines {
+  const lines: PdfLine[] = []
+  if (options.header) lines.push(line(options.header, 729))
+  lines.push(
+    line(`Body line one of page ${index}`, 600),
+    line(`Body line two of page ${index}`, 585)
+  )
+  if (options.footer) lines.push(line(options.footer, 55))
+  return { lines, pageHeight: PAGE_HEIGHT }
+}
+
+function texts(pages: PdfLine[][]): string[][] {
+  return pages.map((lines) => lines.map((entry) => entry.text))
+}
+
+describe('suppressFurniture', () => {
+  it('drops a header repeated on enough pages but keeps its first occurrence', () => {
+    const pages = [1, 2, 3, 4].map((i) => page(i, { header: 'ACME Corp — Internal Use Only' }))
+
+    const result = texts(suppressFurniture(pages))
+
+    expect(result[0]).toContain('ACME Corp — Internal Use Only')
+    for (const remaining of result.slice(1)) {
+      expect(remaining).not.toContain('ACME Corp — Internal Use Only')
+      expect(remaining).toHaveLength(2)
+    }
+  })
+
+  it('never applies the frequency rule to a single page', () => {
+    const result = texts(suppressFurniture([page(1, { header: 'Draft' })]))
+
+    expect(result[0]).toContain('Draft')
+  })
+
+  it('treats two matching pages as furniture when the document has exactly two pages', () => {
+    const pages = [1, 2].map((i) =>
+      page(i, { footer: `Confidential draft, do not distribute — Page ${i} of 2` })
+    )
+
+    const result = texts(suppressFurniture(pages))
+
+    expect(result[0]).toContain('Confidential draft, do not distribute — Page 1 of 2')
+    expect(result[1]).not.toContain('Confidential draft, do not distribute — Page 2 of 2')
+  })
+
+  it('requires the fraction threshold on longer documents without a streak', () => {
+    const pages = Array.from({ length: 10 }, (_, i) =>
+      page(i, { footer: i % 4 === 0 ? 'Sporadic note' : undefined })
+    )
+
+    const result = texts(suppressFurniture(pages))
+
+    expect(result.flat().filter((text) => text === 'Sporadic note')).toHaveLength(3)
+  })
+
+  it('detects a footer by a three-page streak even when its title changes per chapter', () => {
+    const pages = Array.from({ length: 12 }, (_, i) => {
+      const chapter = i < 6 ? 'Chapter 1 Filing Information' : 'Chapter 2 Filing Status'
+      const pageNumber = i + 6
+      const footer =
+        i % 2 === 0
+          ? `${pageNumber} ${chapter} Publication 17 (2025)`
+          : `Publication 17 (2025) ${chapter} ${pageNumber}`
+      return page(i, { footer })
+    })
+
+    const result = texts(suppressFurniture(pages))
+    const footers = result.flat().filter((text) => text.includes('Publication 17'))
+
+    expect(footers).toEqual([
+      '6 Chapter 1 Filing Information Publication 17 (2025)',
+      '12 Chapter 2 Filing Status Publication 17 (2025)',
+    ])
+  })
+
+  it('drops page numbers in the bands regardless of repetition', () => {
+    const pages: PdfPageLines[] = [
+      { lines: [line('Page 1 of 3', 55), line('Body', 600)], pageHeight: PAGE_HEIGHT },
+      { lines: [line('2', 55), line('Body', 600), line('iv', 729)], pageHeight: PAGE_HEIGHT },
+      { lines: [line('- 3 -', 55), line('Body', 600), line('925', 60)], pageHeight: PAGE_HEIGHT },
+    ]
+
+    expect(texts(suppressFurniture(pages))).toEqual([['Body'], ['Body'], ['Body', '925']])
+  })
+
+  it('ignores band text longer than the furniture cap, such as a repeated table header', () => {
+    const header = `SKU Product Unit price Lead time Notes ${'Column '.repeat(14)}`.trim()
+    expect(header.length).toBeGreaterThan(120)
+    const pages = [1, 2, 3, 4].map((i) => page(i, { header }))
+
+    const result = texts(suppressFurniture(pages))
+
+    for (const kept of result) expect(kept).toContain(header)
+  })
+
+  it('merges same-baseline fragments into one key before matching', () => {
+    const pages = [1, 2, 3, 4].map((i) => ({
+      lines: [
+        line('Body', 600),
+        line(`${i}`, 31.3),
+        line('Chapter 1', 31.3),
+        line('Publication 17 (2025)', 32.5),
+      ],
+      pageHeight: PAGE_HEIGHT,
+    }))
+
+    const result = texts(suppressFurniture(pages))
+
+    expect(result[0]).toEqual(['Body', '1', 'Chapter 1', 'Publication 17 (2025)'])
+    expect(result[3]).toEqual(['Body'])
+  })
+
+  it('leaves body text alone even when it repeats', () => {
+    const pages = [1, 2, 3, 4].map(() => ({
+      lines: [line('Repeated body sentence.', 400)],
+      pageHeight: PAGE_HEIGHT,
+    }))
+
+    for (const kept of texts(suppressFurniture(pages))) {
+      expect(kept).toEqual(['Repeated body sentence.'])
+    }
+  })
+
+  it('skips pages without a page height or line geometry', () => {
+    const pages: PdfPageLines[] = [1, 2, 3].map(() => ({
+      lines: [{ text: 'Header', height: 0 }, line('Header', 729)],
+    }))
+
+    for (const kept of texts(suppressFurniture(pages))) expect(kept).toEqual(['Header', 'Header'])
+  })
+})
+
+describe('normalizeFurnitureText', () => {
+  it('keys mirrored facing-page footers identically', () => {
+    expect(normalizeFurnitureText('6 Chapter 1 Filing Information Publication 17 (2025)')).toBe(
+      normalizeFurnitureText('Publication 17 (2025) Chapter 1 Filing Information 17')
+    )
+  })
+
+  it('collapses case, digits, and edge punctuation', () => {
+    expect(normalizeFurnitureText('  Confidential DRAFT — Page 12 of 40. ')).toBe(
+      '# # confidential draft of page'
+    )
+  })
+})
+
+describe('isPageNumber', () => {
+  it('matches the page-number shapes', () => {
+    expect(isPageNumber('Page 3', 10)).toBe(true)
+    expect(isPageNumber('page 3 of 10', 10)).toBe(true)
+    expect(isPageNumber('3 / 10', 10)).toBe(true)
+    expect(isPageNumber('7', 10)).toBe(true)
+    expect(isPageNumber('xiv', 10)).toBe(true)
+    expect(isPageNumber('— 12 —', 20)).toBe(true)
+  })
+
+  it('rejects bare numbers beyond the page count and ordinary text', () => {
+    expect(isPageNumber('925', 142)).toBe(false)
+    expect(isPageNumber('2120', 142)).toBe(false)
+    expect(isPageNumber('Chapter 1', 10)).toBe(false)
+    expect(isPageNumber('civilian', 10)).toBe(false)
+  })
+})
+
+describe('furnitureThreshold', () => {
+  it('scales with the page count', () => {
+    expect(furnitureThreshold(1)).toBeUndefined()
+    expect(furnitureThreshold(2)).toBe(2)
+    expect(furnitureThreshold(3)).toBe(3)
+    expect(furnitureThreshold(10)).toBe(5)
+    expect(furnitureThreshold(142)).toBe(71)
+  })
+})

apps/sim/lib/file-parsers/pdf-furniture.ts

@@ -0,0 +1,185 @@
+import type { PdfLine } from '@/lib/file-parsers/pdf-lines'
+
+/**
+ * Detects running headers, footers, and page numbers across a document's pages
+ * and removes every copy but the first, so a footer repeated on 140 pages does
+ * not land mid-sentence in most retrieval chunks while one copy stays
+ * searchable.
+ */
+
+export interface PdfPageLines {
+  lines: PdfLine[]
+  /** Page height in PDF user space; absent when the page could not report it. */
+  pageHeight?: number
+}
+
+/** Fraction of the page height at the top and bottom where furniture lives. */
+const FURNITURE_BAND_RATIO = 0.12
+
+/** Furniture is short; longer repeated band text is a table header or real prose. */
+const MAX_FURNITURE_CHARS = 120
+
+/** Lines whose baselines differ by at most this many points share one furniture row. */
+const SAME_BASELINE_TOLERANCE = 2
+
+/** Minimum repeats for a key to count as furniture on longer documents. */
+const MIN_FURNITURE_REPEATS = 3
+
+/** Fraction of pages a key must cover when it never runs on consecutive pages. */
+const FURNITURE_PAGE_FRACTION = 0.5
+
+/** Consecutive-page run that marks furniture even when its total count is modest. */
+const MIN_FURNITURE_STREAK = 3
+
+const PAGE_NUMBER_PATTERNS = [
+  /^page\s*\d{1,4}(\s*(of|\/)\s*\d{1,4})?$/i,
+  /^\d{1,4}\s*(of|\/)\s*\d{1,4}$/i,
+  /^[ivxlcdm]{1,6}$/i,
+  /^[-–—]\s*\d+\s*[-–—]$/,
+] as const
+
+/** A bare number is a page number only when the document could have that many pages. */
+const BARE_NUMBER = /^\d{1,4}$/
+
+const EDGE_PUNCTUATION = /^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu
+
+type Band = 'top' | 'bottom'
+
+interface BandGroup {
+  band: Band
+  /** Indices into the page's `lines`. */
+  indices: number[]
+  text: string
+}
+
+interface KeyOccurrences {
+  pages: number[]
+  groups: Array<{ page: number; group: BandGroup }>
+}
+
+/** Returns each page's lines with repeated furniture and page numbers removed. */
+export function suppressFurniture(pages: readonly PdfPageLines[]): PdfLine[][] {
+  const drops = pages.map(() => new Set<number>())
+  const occurrences = new Map<string, KeyOccurrences>()
+
+  pages.forEach((page, pageIndex) => {
+    const seen = new Set<string>()
+    for (const group of bandGroups(page)) {
+      if (isPageNumber(group.text, pages.length)) {
+        for (const index of group.indices) drops[pageIndex].add(index)
+        continue
+      }
+      if (group.text.length > MAX_FURNITURE_CHARS) continue
+      const key = `${group.band}|${normalizeFurnitureText(group.text)}`
+      if (key.endsWith('|')) continue
+      const entry = occurrences.get(key) ?? { pages: [], groups: [] }
+      if (!seen.has(key)) {
+        seen.add(key)
+        entry.pages.push(pageIndex)
+      }
+      entry.groups.push({ page: pageIndex, group })
+      occurrences.set(key, entry)
+    }
+  })
+
+  const threshold = furnitureThreshold(pages.length)
+  for (const entry of occurrences.values()) {
+    const count = entry.pages.length
+    const byFrequency = threshold !== undefined && count >= threshold
+    const byStreak =
+      count >= MIN_FURNITURE_REPEATS && longestRun(entry.pages) >= MIN_FURNITURE_STREAK
+    if (!byFrequency && !byStreak) continue
+    const firstPage = entry.pages[0]
+    for (const { page, group } of entry.groups) {
+      if (page === firstPage) continue
+      for (const index of group.indices) drops[page].add(index)
+    }
+  }
+
+  return pages.map((page, pageIndex) =>
+    drops[pageIndex].size === 0
+      ? page.lines
+      : page.lines.filter((_, index) => !drops[pageIndex].has(index))
+  )
+}
+
+/**
+ * Lowercases each word, strips its edge punctuation, maps digit runs to `#`, and
+ * sorts the words so facing-page footers that mirror their layout (`6 Chapter
+ * 1 … Publication 17 (2025)` vs `Publication 17 (2025) Chapter 1 … 7`) share
+ * one key.
+ */
+export function normalizeFurnitureText(text: string): string {
+  const words = text
+    .toLowerCase()
+    .split(/\s+/)
+    .map((word) => word.replace(EDGE_PUNCTUATION, '').replace(/\d+/g, '#'))
+    .filter((word) => word.length > 0)
+  words.sort()
+  return words.join(' ')
+}
+
+/**
+ * Whether a band line is nothing but a page number. A bare number qualifies
+ * only when it does not exceed `pageCount`, so form and publication numbers
+ * listed near the page edge survive.
+ */
+export function isPageNumber(text: string, pageCount: number): boolean {
+  const compact = text.replace(/\s+/g, ' ').trim()
+  if (BARE_NUMBER.test(compact)) return Number(compact) <= pageCount
+  return PAGE_NUMBER_PATTERNS.some((pattern) => pattern.test(compact))
+}
+
+/** Repeat count that marks a key as furniture, or undefined when the document is too short. */
+export function furnitureThreshold(pageCount: number): number | undefined {
+  if (pageCount < 2) return undefined
+  if (pageCount === 2) return 2
+  return Math.max(MIN_FURNITURE_REPEATS, Math.ceil(FURNITURE_PAGE_FRACTION * pageCount))
+}
+
+/** Groups consecutive band lines that share a baseline into one furniture row. */
+function bandGroups(page: PdfPageLines): BandGroup[] {
+  const { lines, pageHeight } = page
+  if (pageHeight === undefined || !(pageHeight > 0)) return []
+  const groups: BandGroup[] = []
+  let current: (BandGroup & { y: number }) | undefined
+
+  lines.forEach((line, index) => {
+    const band = bandOf(line, pageHeight)
+    if (!band) {
+      current = undefined
+      return
+    }
+    if (
+      current &&
+      current.band === band &&
+      Math.abs(current.y - (line.y as number)) <= SAME_BASELINE_TOLERANCE
+    ) {
+      current.indices.push(index)
+      current.text = `${current.text} ${line.text.trim()}`
+      return
+    }
+    current = { band, indices: [index], text: line.text.trim(), y: line.y as number }
+    groups.push(current)
+  })
+
+  return groups
+}
+
+function bandOf(line: PdfLine, pageHeight: number): Band | undefined {
+  if (line.y === undefined) return undefined
+  if (line.y >= (1 - FURNITURE_BAND_RATIO) * pageHeight) return 'top'
+  if (line.y <= FURNITURE_BAND_RATIO * pageHeight) return 'bottom'
+  return undefined
+}
+
+/** Longest run of consecutive page indices in an ascending list. */
+function longestRun(pages: readonly number[]): number {
+  let best = 0
+  let run = 0
+  for (let i = 0; i < pages.length; i++) {
+    run = i > 0 && pages[i] === pages[i - 1] + 1 ? run + 1 : 1
+    if (run > best) best = run
+  }
+  return best
+}