Skip to content

Commit 1de3a00

Browse files
waleedlatif1claude
andcommitted
fix(parsers): rebuild PDF line and paragraph structure from item geometry
The PDF parser collapsed every page to a single line and concatenated items without separators, so the chunker fell back to sentence splits, words fused across Form XObject boundaries and backwards x-moves, and running headers/footers landed mid-sentence in most chunks. - Build positioned lines from pdf.js item transforms; derive separators from baseline shifts, backwards x-moves, and word-sized gaps, falling back to hasEOL when an item carries no geometry - Join lines per page with paragraph breaks from the median pitch and height changes, rejoin same-row and wrapped table cells, and dehyphenate line-end breaks unless the compound appears intact in the document - Suppress repeated header/footer furniture and page numbers across pages, keeping the first occurrence of each - Prefix short oversized lines with a heading marker - Replace the whitespace collapse with a structure-preserving normaliser and join pages with a paragraph break Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 parent f6d5676 commit 1de3a00

6 files changed

Lines changed: 1256 additions & 18 deletions

File tree

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
/**
2+
* @vitest-environment node
3+
*/
4+
import { describe, expect, it } from 'vitest'
5+
import {
6+
furnitureThreshold,
7+
isPageNumber,
8+
normalizeFurnitureText,
9+
type PdfPageLines,
10+
suppressFurniture,
11+
} from '@/lib/file-parsers/pdf-furniture'
12+
import type { PdfLine } from '@/lib/file-parsers/pdf-lines'
13+
14+
const PAGE_HEIGHT = 792
15+
16+
function line(text: string, y: number): PdfLine {
17+
return { text, y, height: 11 }
18+
}
19+
20+
/** A page with a top header, a bottom footer, and two body lines. */
21+
function page(index: number, options: { header?: string; footer?: string } = {}): PdfPageLines {
22+
const lines: PdfLine[] = []
23+
if (options.header) lines.push(line(options.header, 729))
24+
lines.push(
25+
line(`Body line one of page ${index}`, 600),
26+
line(`Body line two of page ${index}`, 585)
27+
)
28+
if (options.footer) lines.push(line(options.footer, 55))
29+
return { lines, pageHeight: PAGE_HEIGHT }
30+
}
31+
32+
function texts(pages: PdfLine[][]): string[][] {
33+
return pages.map((lines) => lines.map((entry) => entry.text))
34+
}
35+
36+
describe('suppressFurniture', () => {
37+
it('drops a header repeated on enough pages but keeps its first occurrence', () => {
38+
const pages = [1, 2, 3, 4].map((i) => page(i, { header: 'ACME Corp — Internal Use Only' }))
39+
40+
const result = texts(suppressFurniture(pages))
41+
42+
expect(result[0]).toContain('ACME Corp — Internal Use Only')
43+
for (const remaining of result.slice(1)) {
44+
expect(remaining).not.toContain('ACME Corp — Internal Use Only')
45+
expect(remaining).toHaveLength(2)
46+
}
47+
})
48+
49+
it('never applies the frequency rule to a single page', () => {
50+
const result = texts(suppressFurniture([page(1, { header: 'Draft' })]))
51+
52+
expect(result[0]).toContain('Draft')
53+
})
54+
55+
it('treats two matching pages as furniture when the document has exactly two pages', () => {
56+
const pages = [1, 2].map((i) =>
57+
page(i, { footer: `Confidential draft, do not distribute — Page ${i} of 2` })
58+
)
59+
60+
const result = texts(suppressFurniture(pages))
61+
62+
expect(result[0]).toContain('Confidential draft, do not distribute — Page 1 of 2')
63+
expect(result[1]).not.toContain('Confidential draft, do not distribute — Page 2 of 2')
64+
})
65+
66+
it('requires the fraction threshold on longer documents without a streak', () => {
67+
const pages = Array.from({ length: 10 }, (_, i) =>
68+
page(i, { footer: i % 4 === 0 ? 'Sporadic note' : undefined })
69+
)
70+
71+
const result = texts(suppressFurniture(pages))
72+
73+
expect(result.flat().filter((text) => text === 'Sporadic note')).toHaveLength(3)
74+
})
75+
76+
it('detects a footer by a three-page streak even when its title changes per chapter', () => {
77+
const pages = Array.from({ length: 12 }, (_, i) => {
78+
const chapter = i < 6 ? 'Chapter 1 Filing Information' : 'Chapter 2 Filing Status'
79+
const pageNumber = i + 6
80+
const footer =
81+
i % 2 === 0
82+
? `${pageNumber} ${chapter} Publication 17 (2025)`
83+
: `Publication 17 (2025) ${chapter} ${pageNumber}`
84+
return page(i, { footer })
85+
})
86+
87+
const result = texts(suppressFurniture(pages))
88+
const footers = result.flat().filter((text) => text.includes('Publication 17'))
89+
90+
expect(footers).toEqual([
91+
'6 Chapter 1 Filing Information Publication 17 (2025)',
92+
'12 Chapter 2 Filing Status Publication 17 (2025)',
93+
])
94+
})
95+
96+
it('drops page numbers in the bands regardless of repetition', () => {
97+
const pages: PdfPageLines[] = [
98+
{ lines: [line('Page 1 of 3', 55), line('Body', 600)], pageHeight: PAGE_HEIGHT },
99+
{ lines: [line('2', 55), line('Body', 600), line('iv', 729)], pageHeight: PAGE_HEIGHT },
100+
{ lines: [line('- 3 -', 55), line('Body', 600), line('925', 60)], pageHeight: PAGE_HEIGHT },
101+
]
102+
103+
expect(texts(suppressFurniture(pages))).toEqual([['Body'], ['Body'], ['Body', '925']])
104+
})
105+
106+
it('ignores band text longer than the furniture cap, such as a repeated table header', () => {
107+
const header = `SKU Product Unit price Lead time Notes ${'Column '.repeat(14)}`.trim()
108+
expect(header.length).toBeGreaterThan(120)
109+
const pages = [1, 2, 3, 4].map((i) => page(i, { header }))
110+
111+
const result = texts(suppressFurniture(pages))
112+
113+
for (const kept of result) expect(kept).toContain(header)
114+
})
115+
116+
it('merges same-baseline fragments into one key before matching', () => {
117+
const pages = [1, 2, 3, 4].map((i) => ({
118+
lines: [
119+
line('Body', 600),
120+
line(`${i}`, 31.3),
121+
line('Chapter 1', 31.3),
122+
line('Publication 17 (2025)', 32.5),
123+
],
124+
pageHeight: PAGE_HEIGHT,
125+
}))
126+
127+
const result = texts(suppressFurniture(pages))
128+
129+
expect(result[0]).toEqual(['Body', '1', 'Chapter 1', 'Publication 17 (2025)'])
130+
expect(result[3]).toEqual(['Body'])
131+
})
132+
133+
it('leaves body text alone even when it repeats', () => {
134+
const pages = [1, 2, 3, 4].map(() => ({
135+
lines: [line('Repeated body sentence.', 400)],
136+
pageHeight: PAGE_HEIGHT,
137+
}))
138+
139+
for (const kept of texts(suppressFurniture(pages))) {
140+
expect(kept).toEqual(['Repeated body sentence.'])
141+
}
142+
})
143+
144+
it('skips pages without a page height or line geometry', () => {
145+
const pages: PdfPageLines[] = [1, 2, 3].map(() => ({
146+
lines: [{ text: 'Header', height: 0 }, line('Header', 729)],
147+
}))
148+
149+
for (const kept of texts(suppressFurniture(pages))) expect(kept).toEqual(['Header', 'Header'])
150+
})
151+
})
152+
153+
describe('normalizeFurnitureText', () => {
154+
it('keys mirrored facing-page footers identically', () => {
155+
expect(normalizeFurnitureText('6 Chapter 1 Filing Information Publication 17 (2025)')).toBe(
156+
normalizeFurnitureText('Publication 17 (2025) Chapter 1 Filing Information 17')
157+
)
158+
})
159+
160+
it('collapses case, digits, and edge punctuation', () => {
161+
expect(normalizeFurnitureText(' Confidential DRAFT — Page 12 of 40. ')).toBe(
162+
'# # confidential draft of page'
163+
)
164+
})
165+
})
166+
167+
describe('isPageNumber', () => {
168+
it('matches the page-number shapes', () => {
169+
expect(isPageNumber('Page 3', 10)).toBe(true)
170+
expect(isPageNumber('page 3 of 10', 10)).toBe(true)
171+
expect(isPageNumber('3 / 10', 10)).toBe(true)
172+
expect(isPageNumber('7', 10)).toBe(true)
173+
expect(isPageNumber('xiv', 10)).toBe(true)
174+
expect(isPageNumber('— 12 —', 20)).toBe(true)
175+
})
176+
177+
it('rejects bare numbers beyond the page count and ordinary text', () => {
178+
expect(isPageNumber('925', 142)).toBe(false)
179+
expect(isPageNumber('2120', 142)).toBe(false)
180+
expect(isPageNumber('Chapter 1', 10)).toBe(false)
181+
expect(isPageNumber('civilian', 10)).toBe(false)
182+
})
183+
})
184+
185+
describe('furnitureThreshold', () => {
186+
it('scales with the page count', () => {
187+
expect(furnitureThreshold(1)).toBeUndefined()
188+
expect(furnitureThreshold(2)).toBe(2)
189+
expect(furnitureThreshold(3)).toBe(3)
190+
expect(furnitureThreshold(10)).toBe(5)
191+
expect(furnitureThreshold(142)).toBe(71)
192+
})
193+
})
Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
import type { PdfLine } from '@/lib/file-parsers/pdf-lines'
2+
3+
/**
4+
* Detects running headers, footers, and page numbers across a document's pages
5+
* and removes every copy but the first, so a footer repeated on 140 pages does
6+
* not land mid-sentence in most retrieval chunks while one copy stays
7+
* searchable.
8+
*/
9+
10+
export interface PdfPageLines {
11+
lines: PdfLine[]
12+
/** Page height in PDF user space; absent when the page could not report it. */
13+
pageHeight?: number
14+
}
15+
16+
/** Fraction of the page height at the top and bottom where furniture lives. */
17+
const FURNITURE_BAND_RATIO = 0.12
18+
19+
/** Furniture is short; longer repeated band text is a table header or real prose. */
20+
const MAX_FURNITURE_CHARS = 120
21+
22+
/** Lines whose baselines differ by at most this many points share one furniture row. */
23+
const SAME_BASELINE_TOLERANCE = 2
24+
25+
/** Minimum repeats for a key to count as furniture on longer documents. */
26+
const MIN_FURNITURE_REPEATS = 3
27+
28+
/** Fraction of pages a key must cover when it never runs on consecutive pages. */
29+
const FURNITURE_PAGE_FRACTION = 0.5
30+
31+
/** Consecutive-page run that marks furniture even when its total count is modest. */
32+
const MIN_FURNITURE_STREAK = 3
33+
34+
const PAGE_NUMBER_PATTERNS = [
35+
/^page\s*\d{1,4}(\s*(of|\/)\s*\d{1,4})?$/i,
36+
/^\d{1,4}\s*(of|\/)\s*\d{1,4}$/i,
37+
/^[ivxlcdm]{1,6}$/i,
38+
/^[-–—]\s*\d+\s*[-–—]$/,
39+
] as const
40+
41+
/** A bare number is a page number only when the document could have that many pages. */
42+
const BARE_NUMBER = /^\d{1,4}$/
43+
44+
const EDGE_PUNCTUATION = /^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu
45+
46+
type Band = 'top' | 'bottom'
47+
48+
interface BandGroup {
49+
band: Band
50+
/** Indices into the page's `lines`. */
51+
indices: number[]
52+
text: string
53+
}
54+
55+
interface KeyOccurrences {
56+
pages: number[]
57+
groups: Array<{ page: number; group: BandGroup }>
58+
}
59+
60+
/** Returns each page's lines with repeated furniture and page numbers removed. */
61+
export function suppressFurniture(pages: readonly PdfPageLines[]): PdfLine[][] {
62+
const drops = pages.map(() => new Set<number>())
63+
const occurrences = new Map<string, KeyOccurrences>()
64+
65+
pages.forEach((page, pageIndex) => {
66+
const seen = new Set<string>()
67+
for (const group of bandGroups(page)) {
68+
if (isPageNumber(group.text, pages.length)) {
69+
for (const index of group.indices) drops[pageIndex].add(index)
70+
continue
71+
}
72+
if (group.text.length > MAX_FURNITURE_CHARS) continue
73+
const key = `${group.band}|${normalizeFurnitureText(group.text)}`
74+
if (key.endsWith('|')) continue
75+
const entry = occurrences.get(key) ?? { pages: [], groups: [] }
76+
if (!seen.has(key)) {
77+
seen.add(key)
78+
entry.pages.push(pageIndex)
79+
}
80+
entry.groups.push({ page: pageIndex, group })
81+
occurrences.set(key, entry)
82+
}
83+
})
84+
85+
const threshold = furnitureThreshold(pages.length)
86+
for (const entry of occurrences.values()) {
87+
const count = entry.pages.length
88+
const byFrequency = threshold !== undefined && count >= threshold
89+
const byStreak =
90+
count >= MIN_FURNITURE_REPEATS && longestRun(entry.pages) >= MIN_FURNITURE_STREAK
91+
if (!byFrequency && !byStreak) continue
92+
const firstPage = entry.pages[0]
93+
for (const { page, group } of entry.groups) {
94+
if (page === firstPage) continue
95+
for (const index of group.indices) drops[page].add(index)
96+
}
97+
}
98+
99+
return pages.map((page, pageIndex) =>
100+
drops[pageIndex].size === 0
101+
? page.lines
102+
: page.lines.filter((_, index) => !drops[pageIndex].has(index))
103+
)
104+
}
105+
106+
/**
107+
* Lowercases each word, strips its edge punctuation, maps digit runs to `#`, and
108+
* sorts the words so facing-page footers that mirror their layout (`6 Chapter
109+
* 1 … Publication 17 (2025)` vs `Publication 17 (2025) Chapter 1 … 7`) share
110+
* one key.
111+
*/
112+
export function normalizeFurnitureText(text: string): string {
113+
const words = text
114+
.toLowerCase()
115+
.split(/\s+/)
116+
.map((word) => word.replace(EDGE_PUNCTUATION, '').replace(/\d+/g, '#'))
117+
.filter((word) => word.length > 0)
118+
words.sort()
119+
return words.join(' ')
120+
}
121+
122+
/**
123+
* Whether a band line is nothing but a page number. A bare number qualifies
124+
* only when it does not exceed `pageCount`, so form and publication numbers
125+
* listed near the page edge survive.
126+
*/
127+
export function isPageNumber(text: string, pageCount: number): boolean {
128+
const compact = text.replace(/\s+/g, ' ').trim()
129+
if (BARE_NUMBER.test(compact)) return Number(compact) <= pageCount
130+
return PAGE_NUMBER_PATTERNS.some((pattern) => pattern.test(compact))
131+
}
132+
133+
/** Repeat count that marks a key as furniture, or undefined when the document is too short. */
134+
export function furnitureThreshold(pageCount: number): number | undefined {
135+
if (pageCount < 2) return undefined
136+
if (pageCount === 2) return 2
137+
return Math.max(MIN_FURNITURE_REPEATS, Math.ceil(FURNITURE_PAGE_FRACTION * pageCount))
138+
}
139+
140+
/** Groups consecutive band lines that share a baseline into one furniture row. */
141+
function bandGroups(page: PdfPageLines): BandGroup[] {
142+
const { lines, pageHeight } = page
143+
if (pageHeight === undefined || !(pageHeight > 0)) return []
144+
const groups: BandGroup[] = []
145+
let current: (BandGroup & { y: number }) | undefined
146+
147+
lines.forEach((line, index) => {
148+
const band = bandOf(line, pageHeight)
149+
if (!band) {
150+
current = undefined
151+
return
152+
}
153+
if (
154+
current &&
155+
current.band === band &&
156+
Math.abs(current.y - (line.y as number)) <= SAME_BASELINE_TOLERANCE
157+
) {
158+
current.indices.push(index)
159+
current.text = `${current.text} ${line.text.trim()}`
160+
return
161+
}
162+
current = { band, indices: [index], text: line.text.trim(), y: line.y as number }
163+
groups.push(current)
164+
})
165+
166+
return groups
167+
}
168+
169+
function bandOf(line: PdfLine, pageHeight: number): Band | undefined {
170+
if (line.y === undefined) return undefined
171+
if (line.y >= (1 - FURNITURE_BAND_RATIO) * pageHeight) return 'top'
172+
if (line.y <= FURNITURE_BAND_RATIO * pageHeight) return 'bottom'
173+
return undefined
174+
}
175+
176+
/** Longest run of consecutive page indices in an ascending list. */
177+
function longestRun(pages: readonly number[]): number {
178+
let best = 0
179+
let run = 0
180+
for (let i = 0; i < pages.length; i++) {
181+
run = i > 0 && pages[i] === pages[i - 1] + 1 ? run + 1 : 1
182+
if (run > best) best = run
183+
}
184+
return best
185+
}

0 commit comments

Comments
 (0)