Merge branch 'fix/parsers-office-structure' into eval/kb-parser-quality

Author: waleedlatif1

apps/sim/lib/file-parsers/docx-parser.ts

@@ -6,23 +6,26 @@ import {
   isEncryptedOfficeParserError,
   toFileParserError,
 } from '@/lib/file-parsers/errors'
+import {
+  assertHtmlStringWithinLimits,
+  htmlToStructuredText,
+  isHtmlComplexityError,
+} from '@/lib/file-parsers/html-parser'
 import { parseOfficeText } from '@/lib/file-parsers/officeparser-module'
 import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types'
 import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
 import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard'
 
 const logger = createLogger('DocxParser')
 
-interface MammothMessage {
-  type: 'warning' | 'error'
-  message: string
-}
-
-interface MammothResult {
-  value: string
-  messages: MammothMessage[]
-}
-
+/**
+ * Extracts DOCX text by rendering the document to HTML with mammoth and walking
+ * that HTML with the shared structured-text walker. mammoth's HTML keeps the
+ * heading levels, list nesting, table rows, and footnotes that its raw-text mode
+ * flattens to one paragraph per cell, so the output matches what the HTML parser
+ * produces for the same document. (mammoth's Markdown mode is deprecated and
+ * drops tables, so it is deliberately not used.)
+ */
 export class DocxParser implements FileParser {
   async parseFile(filePath: string, options: FileParseOptions = {}): Promise<FileParseResult> {
     if (!filePath) {
@@ -46,24 +49,29 @@ export class DocxParser implements FileParser {
       let parserReturnedEmpty = false
 
       try {
-        const result = await mammoth.extractRawText({ buffer })
+        const htmlResult = await mammoth.convertToHtml({ buffer })
         options.signal?.throwIfAborted()
 
-        if (result.value && result.value.trim().length > 0) {
-          let htmlResult: MammothResult = { value: '', messages: [] }
-          try {
-            htmlResult = await mammoth.convertToHtml({ buffer })
-          } catch {
-            // HTML conversion is optional
+        const structured = this.structuredTextFromHtml(htmlResult.value)
+        if (structured) {
+          return {
+            content: sanitizeTextForUTF8(structured),
+            metadata: {
+              extractionMethod: 'mammoth-html',
+              messages: htmlResult.messages,
+            },
           }
-          options.signal?.throwIfAborted()
+        }
 
+        const rawResult = await mammoth.extractRawText({ buffer })
+        options.signal?.throwIfAborted()
+
+        if (rawResult.value && rawResult.value.trim().length > 0) {
           return {
-            content: sanitizeTextForUTF8(result.value),
+            content: sanitizeTextForUTF8(rawResult.value),
             metadata: {
               extractionMethod: 'mammoth',
-              messages: [...result.messages, ...htmlResult.messages],
-              html: htmlResult.value,
+              messages: [...htmlResult.messages, ...rawResult.messages],
             },
           }
         }
@@ -140,4 +148,23 @@ export class DocxParser implements FileParser {
       throw toFileParserError(error, 'invalid_format', 'Failed to parse DOCX buffer')
     }
   }
+
+  /**
+   * Walks mammoth's HTML rendering under the HTML parser's size caps. A rendering
+   * too large to walk safely falls back to the raw-text path by returning empty,
+   * since mammoth has already materialised the document once at that point.
+   */
+  private structuredTextFromHtml(html: string): string {
+    if (!html || html.trim().length === 0) return ''
+    try {
+      assertHtmlStringWithinLimits(html)
+    } catch (error) {
+      if (isHtmlComplexityError(error)) {
+        logger.warn('mammoth HTML exceeds walker limits, using raw text:', error.message)
+        return ''
+      }
+      throw error
+    }
+    return htmlToStructuredText(html).trim()
+  }
 }

apps/sim/lib/file-parsers/html-parser.test.ts

@@ -118,5 +118,31 @@ describe('HtmlParser', () => {
       expect(result.metadata?.listCount).toBe(1)
       expect(result.metadata?.tableCount).toBe(1)
     })
+
+    it('numbers ordered lists and keeps markers on nested items', async () => {
+      const buffer = Buffer.from(
+        `<body><ol start="3"><li>third</li><li>fourth<ul><li>nested</li></ul></li></ol></body>`
+      )
+
+      const result = await parser.parseBuffer(buffer)
+
+      expect(result.content).toContain('3. third')
+      expect(result.content).toContain('4. fourth')
+      expect(result.content).toContain('  • nested')
+      expect(result.content).not.toContain('fourth nested')
+    })
+
+    it('drops footnote return links but keeps the footnote text', async () => {
+      const buffer = Buffer.from(
+        `<body><p>Body<sup><a href="#footnote-1" id="footnote-ref-1">[1]</a></sup></p>` +
+          `<ol><li id="footnote-1"><p>Note text <a href="#footnote-ref-1">↑</a></p></li></ol></body>`
+      )
+
+      const result = await parser.parseBuffer(buffer)
+
+      expect(result.content).toContain('Body[1]')
+      expect(result.content).toContain('1. Note text')
+      expect(result.content).not.toContain('↑')
+    })
   })
 })