Merge branch 'fix/parsers-input-hygiene' into eval/kb-parser-quality

Author: waleedlatif1

apps/sim/connectors/azure-devops/azure-devops.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { azureDevopsConnectorMeta } from '@/connectors/azure-devops/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -1182,7 +1183,7 @@ async function getFileDocument(
     return null
   }
 
-  const content = buffer.toString('utf8')
+  const content = decodeTextBuffer(buffer).text
   if (!content.trim()) return null
 
   const title = path.split('/').filter(Boolean).pop() || path

apps/sim/connectors/box/box.ts

@@ -1,6 +1,7 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { sleep } from '@sim/utils/helpers'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { boxConnectorMeta } from '@/connectors/box/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -319,7 +320,7 @@ async function fetchPlainTextContent(
   extension: string
 ): Promise<string> {
   const buffer = await downloadWithinLimit(`${BOX_API_BASE}/files/${fileId}/content`, accessToken)
-  const text = buffer.toString('utf8')
+  const { text } = decodeTextBuffer(buffer)
   return HTML_EXTENSIONS.has(extension) ? htmlToPlainText(text) : text
 }
 
@@ -347,7 +348,7 @@ async function fetchExtractedText(
         urlTemplate.replace('{+asset_path}', ''),
         accessToken
       )
-      return buffer.toString('utf8')
+      return decodeTextBuffer(buffer).text
     }
     if (state === 'error' || !infoUrl) return null
     if (attempt === REPRESENTATION_POLL_ATTEMPTS) break

apps/sim/connectors/dropbox/dropbox.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { dropboxConnectorMeta } from '@/connectors/dropbox/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -147,7 +148,7 @@ async function downloadFileContent(
     throw new ConnectorFileTooLargeError(MAX_FILE_SIZE)
   }
 
-  const text = buffer.toString('utf8')
+  const { text } = decodeTextBuffer(buffer)
 
   return isHtml ? htmlToPlainText(text) : text
 }

apps/sim/connectors/github/github.ts

@@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { z } from 'zod'
 import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { type RetryOptions, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { parseGitHubRepository } from '@/lib/oauth/github-repository'
 import { githubConnectorMeta } from '@/connectors/github/meta'
@@ -296,7 +297,7 @@ async function fetchBlobContent(
     throw new ConnectorFileTooLargeError(maxBytes)
   }
   if (isBinaryBuffer(buffer)) return null
-  return buffer.toString('utf8')
+  return decodeTextBuffer(buffer).text
 }
 
 /** Resolves links within one snapshot; Contents can truncate dereferenced targets at 1 MiB. */

apps/sim/connectors/gitlab/gitlab.ts

@@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
 import { isPlainRecord } from '@sim/utils/object'
 import type { SecureFetchResponse } from '@/lib/core/security/input-validation.server'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server'
 import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { gitlabConnectorMeta } from '@/connectors/gitlab/meta'
@@ -435,7 +436,7 @@ function fileToDocument(
     return skipped(sizeLimitSkipReason(MAX_FILE_SIZE), buffer.byteLength)
   }
 
-  const content = buffer.toString('utf8')
+  const content = decodeTextBuffer(buffer).text
   const body = composeBody(title, content)
   if (!body.trim()) return null
 

apps/sim/connectors/utils.test.ts

@@ -1498,19 +1498,15 @@ describe('htmlToPlainText entity decoding', () => {
 
 describe('isIndexableConnectorFile', () => {
   it('accepts the Office and PDF formats the knowledge base can parse', () => {
-    for (const name of [
-      'sop.pdf',
-      'sop.doc',
-      'sop.docx',
-      'sheet.xls',
-      'sheet.xlsx',
-      'deck.ppt',
-      'deck.pptx',
-    ]) {
+    for (const name of ['sop.pdf', 'sop.doc', 'sop.docx', 'sheet.xls', 'sheet.xlsx', 'deck.pptx']) {
       expect(isIndexableConnectorFile(name)).toBe(true)
     }
   })
 
+  it('refuses legacy .ppt up front because no parser reads it', () => {
+    expect(isIndexableConnectorFile('deck.ppt')).toBe(false)
+  })
+
   it('still accepts the plain-text formats connectors already synced', () => {
     for (const name of ['a.txt', 'a.md', 'a.html', 'a.htm', 'a.csv', 'a.log', 'a.tsv', 'a.rst']) {
       expect(isIndexableConnectorFile(name)).toBe(true)
@@ -1578,6 +1574,30 @@ describe('extractConnectorText', () => {
   it('leaves whitespace-only content alone for the caller to reject', () => {
     expect(extractConnectorText(Buffer.from('   '), 'blank.txt')).toBe('   ')
   })
+
+  it('decodes a Latin-1 file as Windows-1252 instead of indexing mojibake', () => {
+    expect(extractConnectorText(Buffer.from('Caf\xe9 \xa3 42', 'latin1'), 'notes.txt')).toBe(
+      'Café £ 42'
+    )
+  })
+
+  it('strips a UTF-8 BOM', () => {
+    expect(
+      extractConnectorText(
+        Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('a,b')]),
+        'data.csv'
+      )
+    ).toBe('a,b')
+  })
+
+  it('decodes UTF-16 with a BOM', () => {
+    expect(
+      extractConnectorText(
+        Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from('<p>Hällo</p>', 'utf16le')]),
+        'page.html'
+      )
+    ).toBe('Hällo')
+  })
 })
 
 describe('pipelineParsedMimeType', () => {

apps/sim/connectors/utils.ts

@@ -4,6 +4,7 @@ import {
   isPayloadSizeLimitError,
   readResponseToBufferWithLimit,
 } from '@/lib/core/utils/stream-limits'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { MAX_FILE_SIZE as KB_DOCUMENT_MAX_BYTES } from '@/lib/uploads/utils/validation'
 import type { ExternalDocument } from '@/connectors/types'
 
@@ -427,7 +428,6 @@ export const PIPELINE_PARSED_MIME_TYPES: ReadonlyMap<string, string> = new Map([
   ['xlsm', 'application/vnd.ms-excel.sheet.macroEnabled.12'],
   ['xlsb', 'application/vnd.ms-excel.sheet.binary.macroEnabled.12'],
   ['xltx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.template'],
-  ['ppt', 'application/vnd.ms-powerpoint'],
   ['pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'],
   ['pptm', 'application/vnd.ms-powerpoint.presentation.macroEnabled.12'],
   ['potx', 'application/vnd.openxmlformats-officedocument.presentationml.template'],
@@ -497,16 +497,18 @@ export function pipelineParsedMimeType(fileName: string): string | undefined {
  *
  * Only for formats that are already text — anything the shared parsers handle is
  * delivered to them verbatim instead, via {@link pipelineParsedMimeType}. HTML is
- * additionally reduced to plain text; everything else is a UTF-8 decode.
+ * additionally reduced to plain text; everything else is decoded with the shared
+ * BOM/UTF-8/Windows-1252 detection so a Latin-1 file never indexes as mojibake.
  */
 export function extractConnectorText(buffer: Buffer, fileName: string): string {
   const extension = connectorFileExtension(fileName)
+  const { text } = decodeTextBuffer(buffer)
 
   if (extension === 'html' || extension === 'htm') {
-    return htmlToPlainText(buffer.toString('utf8'))
+    return htmlToPlainText(text)
   }
 
-  return buffer.toString('utf8')
+  return text
 }
 
 /**

apps/sim/lib/copilot/vfs/file-reader.test.ts

@@ -6,13 +6,17 @@ import { randomFillSync } from 'node:crypto'
 import { crc32 } from 'node:zlib'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { fetchWorkspaceFileBuffer } = vi.hoisted(() => ({
+const { fetchWorkspaceFileBuffer, mockParseBuffer } = vi.hoisted(() => ({
   fetchWorkspaceFileBuffer: vi.fn(),
+  mockParseBuffer: vi.fn(),
 }))
 
 vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
   fetchWorkspaceFileBuffer,
 }))
+vi.mock('@/lib/file-parsers', () => ({
+  parseBuffer: mockParseBuffer,
+}))
 
 import {
   MAX_IMAGE_READ_BYTES,
@@ -21,6 +25,7 @@ import {
   MAX_TEXT_READ_BYTES,
   readFileRecord,
 } from '@/lib/copilot/vfs/file-reader'
+import { readPlaceholder } from '@/lib/copilot/vfs/read-placeholders'
 import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { MAX_TRANSCODE_INPUT_BYTES } from '@/lib/uploads/server/heic'
 
@@ -201,3 +206,46 @@ describe('readFileRecord', () => {
     SHARP_TEST_TIMEOUT_MS
   )
 })
+
+describe('readFileRecord parseable documents', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  function documentRecord(name: string, type: string, size: number) {
+    return { ...imageRecord(name, size, type), id: 'wf_doc' }
+  }
+
+  it('returns the parsed text of a document', async () => {
+    fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('bytes'))
+    mockParseBuffer.mockResolvedValue({
+      content: 'Quarterly review\nSecond line',
+      metadata: { extractionMethod: 'word-extractor' },
+    })
+
+    const result = await readFileRecord(documentRecord('review.doc', 'application/msword', 5))
+
+    expect(result).toEqual({ content: 'Quarterly review\nSecond line', totalLines: 2 })
+  })
+
+  /**
+   * A parser that could only scrape bytes flags the result `degraded`; that must
+   * reach the model as the could-not-parse placeholder, never as file content.
+   */
+  it('reports degraded parser output as could-not-parse instead of handing it to the model', async () => {
+    fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('bytes'))
+    mockParseBuffer.mockResolvedValue({
+      content: '[Content_Types].xml _rels/.rels theme/theme/themeManager.xml',
+      metadata: { degraded: true, warning: 'Basic text extraction used' },
+    })
+
+    const result = await readFileRecord(
+      documentRecord('deck.pptx', 'application/vnd.ms-powerpoint', 5)
+    )
+
+    expect(result).toEqual(
+      readPlaceholder.couldNotParse('deck.pptx', 'application/vnd.ms-powerpoint', 5)
+    )
+    expect(result?.content).not.toContain('[Content_Types].xml')
+  })
+})

apps/sim/lib/copilot/vfs/file-reader.ts

@@ -104,7 +104,7 @@ const TEXT_TYPES = new Set([
   'application/javascript',
 ])
 
-const PARSEABLE_EXTENSIONS = new Set(['pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx', 'ppt'])
+const PARSEABLE_EXTENSIONS = new Set(['pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx'])
 
 export function isReadableFileType(contentType: string): boolean {
   return TEXT_TYPES.has(contentType) || contentType.startsWith('text/')
@@ -587,6 +587,10 @@ export async function readFileRecord(
           try {
             const { parseBuffer } = await import('@/lib/file-parsers')
             const result = await parseBuffer(fetched.buffer, ext)
+            if (result.metadata?.degraded === true) {
+              /** Scraped ZIP internals or placeholder prose, not the document's text. */
+              throw new Error(result.metadata.warning ?? 'Parser returned degraded output')
+            }
             const content = result.content || ''
             const lines = content.split('\n').length
             span.setAttributes({

apps/sim/lib/file-parsers/csv-parser.ts

@@ -1,21 +1,34 @@
-import { createReadStream, existsSync } from 'fs'
+import { existsSync } from 'fs'
+import { readFile } from 'fs/promises'
 import { Readable } from 'stream'
 import { createLogger } from '@sim/logger'
 import { type Options, parse } from 'csv-parse'
 import { FileParserError } from '@/lib/file-parsers/errors'
 import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
-import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils'
+import {
+  type DecodedText,
+  decodeTextBuffer,
+  sanitizeTextForUTF8,
+  truncationNotice,
+} from '@/lib/file-parsers/utils'
 
 const logger = createLogger('CsvParser')
 
 const CONFIG = {
   MAX_PREVIEW_ROWS: 1000, // Only keep first 1000 rows for preview
   MAX_SAMPLE_ROWS: 100, // Sample for metadata
   MAX_ERRORS: 100, // Stop after 100 errors
-  STREAM_CHUNK_SIZE: 16384, // 16KB chunks for streaming
 }
 
 export class CsvParser implements FileParser {
+  /**
+   * Reads the whole file before parsing rather than streaming 16 KB chunks:
+   * encoding detection needs the complete byte sequence (a BOM-less UTF-16 or
+   * Windows-1252 file cannot be recognized per chunk, and a multi-byte UTF-8
+   * sequence split across chunk boundaries would be misread). The upload size
+   * caps already bound the file, and `parseBuffer` — the production path —
+   * always held the full buffer.
+   */
   async parseFile(filePath: string): Promise<FileParseResult> {
     if (!filePath) {
       throw new Error('No file path provided')
@@ -25,11 +38,7 @@ export class CsvParser implements FileParser {
       throw new Error(`File not found: ${filePath}`)
     }
 
-    const stream = createReadStream(filePath, {
-      highWaterMark: CONFIG.STREAM_CHUNK_SIZE,
-    })
-
-    return this.parseStream(stream)
+    return this.parseBuffer(await readFile(filePath))
   }
 
   async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
@@ -38,14 +47,18 @@ export class CsvParser implements FileParser {
       `Parsing CSV buffer, size: ${bufferSize} bytes (${(bufferSize / 1024 / 1024).toFixed(2)} MB)`
     )
 
+    const decoded = decodeTextBuffer(buffer)
     const stream = new Readable({ read() {} })
-    stream.push(buffer)
+    stream.push(decoded.text)
     stream.push(null)
 
-    return this.parseStream(stream)
+    return this.parseStream(stream, decoded)
   }
 
-  private parseStream(inputStream: NodeJS.ReadableStream): Promise<FileParseResult> {
+  private parseStream(
+    inputStream: NodeJS.ReadableStream,
+    decoded: DecodedText
+  ): Promise<FileParseResult> {
     return new Promise((resolve, reject) => {
       let rowCount = 0
       let errorCount = 0
@@ -145,6 +158,8 @@ export class CsvParser implements FileParser {
               errors: errors.slice(0, 10),
               truncated: rowCount > CONFIG.MAX_PREVIEW_ROWS,
               sampledData: sampledRows,
+              encoding: decoded.encoding,
+              ...(decoded.warning ? { warning: decoded.warning } : {}),
             },
           })
         }