fix(parsers): decode text by encoding, sniff bytes before routing, read legacy .doc

Text parsers decoded every buffer as UTF-8 and then stripped U+FFFD, so a
Latin-1 or Windows-1252 file silently lost every accented character, a UTF-8
BOM leaked into content and broke JSON.parse, and UTF-16 only worked for
ASCII. `decodeTextBuffer` (BOM > strict UTF-8 with a guarded truncated-tail
retry > Windows-1252) now backs txt/md/csv/json/jsonl/yaml, the .doc plain-text
fallback and the connectors' text decode, and records `encoding`/`warning`
in metadata.

`parseBuffer` routed on the caller-supplied extension alone. `sniff.ts` now
identifies the bytes (PDF, OLE2, ZIP central-directory part names, ODF
mimetype, UTF-16 layout, HTML head) and reconciles them with the extension's
family: a sniffed kind with its own parser overrides the route and records
`detectedType`; binary/unknown bytes under a mismatched family are a typed
`invalid_format` instead of mojibake or placeholder prose.

Legacy OLE .doc goes through word-extractor (body, headers, footers,
footnotes, endnotes; Word 6/95 magic maps to `unsupported_type`); the byte
scrape that returned ZIP part names as degraded prose is deleted. Legacy .ppt
is dropped from the registry, upload and connector allowlists and Chat's
parseable set so it is refused up front. Chat's file reader and the internal
file tool now treat `degraded` output as a parse failure.

pdf.js `InvalidPDFException`/`FormatError`/`PasswordException` are mapped to
typed parser errors at the single `openPdfDocument` choke point, and the zip
guard's `ArchiveIntegrityError` surfaces from `parseBuffer` as a typed
`invalid_format`, so neither classifies as transient and retries forever.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: waleedlatif1

apps/sim/connectors/azure-devops/azure-devops.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { azureDevopsConnectorMeta } from '@/connectors/azure-devops/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -1182,7 +1183,7 @@ async function getFileDocument(
     return null
   }
 
-  const content = buffer.toString('utf8')
+  const content = decodeTextBuffer(buffer).text
   if (!content.trim()) return null
 
   const title = path.split('/').filter(Boolean).pop() || path

apps/sim/connectors/box/box.ts

@@ -1,6 +1,7 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { sleep } from '@sim/utils/helpers'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { boxConnectorMeta } from '@/connectors/box/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -319,7 +320,7 @@ async function fetchPlainTextContent(
   extension: string
 ): Promise<string> {
   const buffer = await downloadWithinLimit(`${BOX_API_BASE}/files/${fileId}/content`, accessToken)
-  const text = buffer.toString('utf8')
+  const { text } = decodeTextBuffer(buffer)
   return HTML_EXTENSIONS.has(extension) ? htmlToPlainText(text) : text
 }
 
@@ -347,7 +348,7 @@ async function fetchExtractedText(
         urlTemplate.replace('{+asset_path}', ''),
         accessToken
       )
-      return buffer.toString('utf8')
+      return decodeTextBuffer(buffer).text
     }
     if (state === 'error' || !infoUrl) return null
     if (attempt === REPRESENTATION_POLL_ATTEMPTS) break

apps/sim/connectors/dropbox/dropbox.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { dropboxConnectorMeta } from '@/connectors/dropbox/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -147,7 +148,7 @@ async function downloadFileContent(
     throw new ConnectorFileTooLargeError(MAX_FILE_SIZE)
   }
 
-  const text = buffer.toString('utf8')
+  const { text } = decodeTextBuffer(buffer)
 
   return isHtml ? htmlToPlainText(text) : text
 }

apps/sim/connectors/github/github.ts

@@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { z } from 'zod'
 import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { type RetryOptions, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { parseGitHubRepository } from '@/lib/oauth/github-repository'
 import { githubConnectorMeta } from '@/connectors/github/meta'
@@ -296,7 +297,7 @@ async function fetchBlobContent(
     throw new ConnectorFileTooLargeError(maxBytes)
   }
   if (isBinaryBuffer(buffer)) return null
-  return buffer.toString('utf8')
+  return decodeTextBuffer(buffer).text
 }
 
 /** Resolves links within one snapshot; Contents can truncate dereferenced targets at 1 MiB. */

apps/sim/connectors/gitlab/gitlab.ts

@@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
 import { isPlainRecord } from '@sim/utils/object'
 import type { SecureFetchResponse } from '@/lib/core/security/input-validation.server'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server'
 import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { gitlabConnectorMeta } from '@/connectors/gitlab/meta'
@@ -435,7 +436,7 @@ function fileToDocument(
     return skipped(sizeLimitSkipReason(MAX_FILE_SIZE), buffer.byteLength)
   }
 
-  const content = buffer.toString('utf8')
+  const content = decodeTextBuffer(buffer).text
   const body = composeBody(title, content)
   if (!body.trim()) return null
 

apps/sim/connectors/utils.test.ts

@@ -1498,19 +1498,15 @@ describe('htmlToPlainText entity decoding', () => {
 
 describe('isIndexableConnectorFile', () => {
   it('accepts the Office and PDF formats the knowledge base can parse', () => {
-    for (const name of [
-      'sop.pdf',
-      'sop.doc',
-      'sop.docx',
-      'sheet.xls',
-      'sheet.xlsx',
-      'deck.ppt',
-      'deck.pptx',
-    ]) {
+    for (const name of ['sop.pdf', 'sop.doc', 'sop.docx', 'sheet.xls', 'sheet.xlsx', 'deck.pptx']) {
       expect(isIndexableConnectorFile(name)).toBe(true)
     }
   })
 
+  it('refuses legacy .ppt up front because no parser reads it', () => {
+    expect(isIndexableConnectorFile('deck.ppt')).toBe(false)
+  })
+
   it('still accepts the plain-text formats connectors already synced', () => {
     for (const name of ['a.txt', 'a.md', 'a.html', 'a.htm', 'a.csv', 'a.log', 'a.tsv', 'a.rst']) {
       expect(isIndexableConnectorFile(name)).toBe(true)
@@ -1578,6 +1574,30 @@ describe('extractConnectorText', () => {
   it('leaves whitespace-only content alone for the caller to reject', () => {
     expect(extractConnectorText(Buffer.from('   '), 'blank.txt')).toBe('   ')
   })
+
+  it('decodes a Latin-1 file as Windows-1252 instead of indexing mojibake', () => {
+    expect(extractConnectorText(Buffer.from('Caf\xe9 \xa3 42', 'latin1'), 'notes.txt')).toBe(
+      'Café £ 42'
+    )
+  })
+
+  it('strips a UTF-8 BOM', () => {
+    expect(
+      extractConnectorText(
+        Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('a,b')]),
+        'data.csv'
+      )
+    ).toBe('a,b')
+  })
+
+  it('decodes UTF-16 with a BOM', () => {
+    expect(
+      extractConnectorText(
+        Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from('<p>Hällo</p>', 'utf16le')]),
+        'page.html'
+      )
+    ).toBe('Hällo')
+  })
 })
 
 describe('pipelineParsedMimeType', () => {

apps/sim/connectors/utils.ts

@@ -4,6 +4,7 @@ import {
   isPayloadSizeLimitError,
   readResponseToBufferWithLimit,
 } from '@/lib/core/utils/stream-limits'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { MAX_FILE_SIZE as KB_DOCUMENT_MAX_BYTES } from '@/lib/uploads/utils/validation'
 import type { ExternalDocument } from '@/connectors/types'
 
@@ -427,7 +428,6 @@ export const PIPELINE_PARSED_MIME_TYPES: ReadonlyMap<string, string> = new Map([
   ['xlsm', 'application/vnd.ms-excel.sheet.macroEnabled.12'],
   ['xlsb', 'application/vnd.ms-excel.sheet.binary.macroEnabled.12'],
   ['xltx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.template'],
-  ['ppt', 'application/vnd.ms-powerpoint'],
   ['pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'],
   ['pptm', 'application/vnd.ms-powerpoint.presentation.macroEnabled.12'],
   ['potx', 'application/vnd.openxmlformats-officedocument.presentationml.template'],
@@ -497,16 +497,18 @@ export function pipelineParsedMimeType(fileName: string): string | undefined {
  *
  * Only for formats that are already text — anything the shared parsers handle is
  * delivered to them verbatim instead, via {@link pipelineParsedMimeType}. HTML is
- * additionally reduced to plain text; everything else is a UTF-8 decode.
+ * additionally reduced to plain text; everything else is decoded with the shared
+ * BOM/UTF-8/Windows-1252 detection so a Latin-1 file never indexes as mojibake.
  */
 export function extractConnectorText(buffer: Buffer, fileName: string): string {
   const extension = connectorFileExtension(fileName)
+  const { text } = decodeTextBuffer(buffer)
 
   if (extension === 'html' || extension === 'htm') {
-    return htmlToPlainText(buffer.toString('utf8'))
+    return htmlToPlainText(text)
   }
 
-  return buffer.toString('utf8')
+  return text
 }
 
 /**

apps/sim/lib/copilot/vfs/file-reader.test.ts

@@ -6,13 +6,17 @@ import { randomFillSync } from 'node:crypto'
 import { crc32 } from 'node:zlib'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { fetchWorkspaceFileBuffer } = vi.hoisted(() => ({
+const { fetchWorkspaceFileBuffer, mockParseBuffer } = vi.hoisted(() => ({
   fetchWorkspaceFileBuffer: vi.fn(),
+  mockParseBuffer: vi.fn(),
 }))
 
 vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
   fetchWorkspaceFileBuffer,
 }))
+vi.mock('@/lib/file-parsers', () => ({
+  parseBuffer: mockParseBuffer,
+}))
 
 import {
   MAX_IMAGE_READ_BYTES,
@@ -21,6 +25,7 @@ import {
   MAX_TEXT_READ_BYTES,
   readFileRecord,
 } from '@/lib/copilot/vfs/file-reader'
+import { readPlaceholder } from '@/lib/copilot/vfs/read-placeholders'
 import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { MAX_TRANSCODE_INPUT_BYTES } from '@/lib/uploads/server/heic'
 
@@ -201,3 +206,46 @@ describe('readFileRecord', () => {
     SHARP_TEST_TIMEOUT_MS
   )
 })
+
+describe('readFileRecord parseable documents', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  function documentRecord(name: string, type: string, size: number) {
+    return { ...imageRecord(name, size, type), id: 'wf_doc' }
+  }
+
+  it('returns the parsed text of a document', async () => {
+    fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('bytes'))
+    mockParseBuffer.mockResolvedValue({
+      content: 'Quarterly review\nSecond line',
+      metadata: { extractionMethod: 'word-extractor' },
+    })
+
+    const result = await readFileRecord(documentRecord('review.doc', 'application/msword', 5))
+
+    expect(result).toEqual({ content: 'Quarterly review\nSecond line', totalLines: 2 })
+  })
+
+  /**
+   * A parser that could only scrape bytes flags the result `degraded`; that must
+   * reach the model as the could-not-parse placeholder, never as file content.
+   */
+  it('reports degraded parser output as could-not-parse instead of handing it to the model', async () => {
+    fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('bytes'))
+    mockParseBuffer.mockResolvedValue({
+      content: '[Content_Types].xml _rels/.rels theme/theme/themeManager.xml',
+      metadata: { degraded: true, warning: 'Basic text extraction used' },
+    })
+
+    const result = await readFileRecord(
+      documentRecord('deck.pptx', 'application/vnd.ms-powerpoint', 5)
+    )
+
+    expect(result).toEqual(
+      readPlaceholder.couldNotParse('deck.pptx', 'application/vnd.ms-powerpoint', 5)
+    )
+    expect(result?.content).not.toContain('[Content_Types].xml')
+  })
+})

apps/sim/lib/copilot/vfs/file-reader.ts

@@ -104,7 +104,7 @@ const TEXT_TYPES = new Set([
   'application/javascript',
 ])
 
-const PARSEABLE_EXTENSIONS = new Set(['pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx', 'ppt'])
+const PARSEABLE_EXTENSIONS = new Set(['pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx'])
 
 export function isReadableFileType(contentType: string): boolean {
   return TEXT_TYPES.has(contentType) || contentType.startsWith('text/')
@@ -587,6 +587,10 @@ export async function readFileRecord(
           try {
             const { parseBuffer } = await import('@/lib/file-parsers')
             const result = await parseBuffer(fetched.buffer, ext)
+            if (result.metadata?.degraded === true) {
+              /** Scraped ZIP internals or placeholder prose, not the document's text. */
+              throw new Error(result.metadata.warning ?? 'Parser returned degraded output')
+            }
             const content = result.content || ''
             const lines = content.split('\n').length
             span.setAttributes({

apps/sim/lib/file-parsers/csv-parser.ts

@@ -1,21 +1,34 @@
-import { createReadStream, existsSync } from 'fs'
+import { existsSync } from 'fs'
+import { readFile } from 'fs/promises'
 import { Readable } from 'stream'
 import { createLogger } from '@sim/logger'
 import { type Options, parse } from 'csv-parse'
 import { FileParserError } from '@/lib/file-parsers/errors'
 import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
-import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils'
+import {
+  type DecodedText,
+  decodeTextBuffer,
+  sanitizeTextForUTF8,
+  truncationNotice,
+} from '@/lib/file-parsers/utils'
 
 const logger = createLogger('CsvParser')
 
 const CONFIG = {
   MAX_PREVIEW_ROWS: 1000, // Only keep first 1000 rows for preview
   MAX_SAMPLE_ROWS: 100, // Sample for metadata
   MAX_ERRORS: 100, // Stop after 100 errors
-  STREAM_CHUNK_SIZE: 16384, // 16KB chunks for streaming
 }
 
 export class CsvParser implements FileParser {
+  /**
+   * Reads the whole file before parsing rather than streaming 16 KB chunks:
+   * encoding detection needs the complete byte sequence (a BOM-less UTF-16 or
+   * Windows-1252 file cannot be recognized per chunk, and a multi-byte UTF-8
+   * sequence split across chunk boundaries would be misread). The upload size
+   * caps already bound the file, and `parseBuffer` — the production path —
+   * always held the full buffer.
+   */
   async parseFile(filePath: string): Promise<FileParseResult> {
     if (!filePath) {
       throw new Error('No file path provided')
@@ -25,11 +38,7 @@ export class CsvParser implements FileParser {
       throw new Error(`File not found: ${filePath}`)
     }
 
-    const stream = createReadStream(filePath, {
-      highWaterMark: CONFIG.STREAM_CHUNK_SIZE,
-    })
-
-    return this.parseStream(stream)
+    return this.parseBuffer(await readFile(filePath))
   }
 
   async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
@@ -38,14 +47,18 @@ export class CsvParser implements FileParser {
       `Parsing CSV buffer, size: ${bufferSize} bytes (${(bufferSize / 1024 / 1024).toFixed(2)} MB)`
     )
 
+    const decoded = decodeTextBuffer(buffer)
     const stream = new Readable({ read() {} })
-    stream.push(buffer)
+    stream.push(decoded.text)
     stream.push(null)
 
-    return this.parseStream(stream)
+    return this.parseStream(stream, decoded)
   }
 
-  private parseStream(inputStream: NodeJS.ReadableStream): Promise<FileParseResult> {
+  private parseStream(
+    inputStream: NodeJS.ReadableStream,
+    decoded: DecodedText
+  ): Promise<FileParseResult> {
     return new Promise((resolve, reject) => {
       let rowCount = 0
       let errorCount = 0
@@ -145,6 +158,8 @@ export class CsvParser implements FileParser {
               errors: errors.slice(0, 10),
               truncated: rowCount > CONFIG.MAX_PREVIEW_ROWS,
               sampledData: sampledRows,
+              encoding: decoded.encoding,
+              ...(decoded.warning ? { warning: decoded.warning } : {}),
             },
           })
         }