fix(search): harden live search accuracy, provider queries, and request cost (#8226)

Author: waleedlatif1

apps/sim/connectors/google-workspace/api-errors.ts

@@ -54,6 +54,13 @@ const RATE_LIMIT_REASONS = new Set([
   'RATE_LIMIT_EXCEEDED',
 ])
 
+/** Whether a Google error reason reports an exhausted rate or usage quota rather than a denial. */
+export function isGoogleQuotaReason(reason: string): boolean {
+  return (
+    RATE_LIMIT_REASONS.has(reason) || reason === 'dailyLimitExceeded' || reason === 'quotaExceeded'
+  )
+}
+
 export function safeGoogleErrorReasons(reasons: readonly string[]): string[] {
   return [...new Set(reasons.filter((reason) => SAFE_REASONS.has(reason)))]
 }
@@ -119,13 +126,7 @@ export class GoogleApiError extends ConnectorSourceError {
     const safeReasons = safeGoogleErrorReasons(reasons)
     const suffix = safeReasons.length ? ` (${safeReasons.join(', ')})` : ''
     const category =
-      status === 429 ||
-      safeReasons.some(
-        (reason) =>
-          RATE_LIMIT_REASONS.has(reason) ||
-          reason === 'dailyLimitExceeded' ||
-          reason === 'quotaExceeded'
-      )
+      status === 429 || safeReasons.some(isGoogleQuotaReason)
         ? 'rate_limit'
         : status >= 500
           ? 'provider_unavailable'

apps/sim/connectors/utils.test.ts

@@ -72,6 +72,7 @@ import {
   isIndexableConnectorFile,
   isSkippableMicrosoftGraphFolderError,
   isSkippedDocument,
+  looksLikeHtml,
   MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES,
   MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES,
   MICROSOFT_GRAPH_MAX_PENDING_FOLDERS,
@@ -1754,3 +1755,14 @@ describe('BoundedLines', () => {
     })
   })
 })
+
+describe('looksLikeHtml', () => {
+  it('recognizes elements with valid tag syntax', () => {
+    for (const markup of ['<p>', '</p>', '<br>', '<br/>', '<a href="x">', '<td class="c">'])
+      expect(looksLikeHtml(`text ${markup} text`)).toBe(true)
+  })
+  it('does not mistake an address whose name starts like a tag for markup', () => {
+    for (const text of ['Invite <a@acme.com>', 'From <b.smith@acme.com>', '<i.e. later>'])
+      expect(looksLikeHtml(text)).toBe(false)
+  })
+})

apps/sim/connectors/utils.ts

@@ -339,11 +339,11 @@ function decodeCharacterReference(raw: string, code: number): string {
  * A false positive therefore does not merely pass text through untouched — it
  * deletes the bracketed span and flattens the document's line structure. Plain
  * text routinely contains angle brackets that are not markup: an email address
- * (`Reply from John <john@acme.com>`), a markdown autolink
+ * (`Reply from John <john@acme.com>`, or `<a@acme.com>`, whose name is a tag's), a markdown autolink
  * (`<https://acme.com>`), or a placeholder (`<redacted>`).
  */
 const HTML_TAG_PATTERN =
-  /<\/?(?:p|div|br|hr|ul|ol|li|h[1-6]|table|thead|tbody|tr|td|th|span|strong|em|b|i|u|a|code|pre|blockquote|img|figure)\b[^>]*>/i
+  /<\/?(?:p|div|br|hr|ul|ol|li|h[1-6]|table|thead|tbody|tr|td|th|span|strong|em|b|i|u|a|code|pre|blockquote|img|figure)(?=[\s/>])[^>]*>/i
 
 /**
  * Reports whether a value carries real HTML markup and is therefore worth routing
@@ -362,15 +362,25 @@ export function looksLikeHtml(value: string): boolean {
  * punctuation as numeric references, which previously reached the index verbatim.
  */
 export function htmlToPlainText(html: string): string {
-  const text = html
-    .replace(/<[^>]*>/g, ' ')
-    .replace(HTML_ENTITY_PATTERN, (raw: string, hex?: string, decimal?: string, named?: string) => {
+  return decodeHtmlEntities(html.replace(/<[^>]*>/g, ' '))
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+/**
+ * Decodes HTML character references without touching markup or whitespace. Use for text a
+ * provider HTML-escapes but does not mark up, such as Gmail message snippets.
+ */
+export function decodeHtmlEntities(text: string): string {
+  return text.replace(
+    HTML_ENTITY_PATTERN,
+    (raw: string, hex?: string, decimal?: string, named?: string) => {
       if (named !== undefined) return NAMED_ENTITIES[named] ?? raw
       if (hex !== undefined) return decodeCharacterReference(raw, Number.parseInt(hex, 16))
       if (decimal !== undefined) return decodeCharacterReference(raw, Number.parseInt(decimal, 10))
       return raw
-    })
-  return text.replace(/\s+/g, ' ').trim()
+    }
+  )
 }
 
 /**

apps/sim/lib/core/security/input-validation.server.ts

@@ -356,6 +356,19 @@ export interface SecureFetchOptions {
   proxyUrl?: string
   /** Hide credential-derived URL details from validation logs. */
   logUrlValidationDetails?: boolean
+  /**
+   * Ask for a gzip or brotli body. The body is decoded before it is returned, and
+   * `maxResponseBytes` bounds the decoded bytes, so a compression bomb still stops at the cap.
+   */
+  acceptCompressed?: boolean
+  /**
+   * Reuses keep-alive connections to the same pinned address across requests. A connection is
+   * only ever reused for the IP it was opened to, so every request keeps its DNS pinning. The
+   * owner must call {@link PinnedConnectionPool.destroy} once its requests have finished. Bun
+   * keeps sockets in its own per-address pool, so there reuse spans pools and `destroy` releases
+   * only the agents.
+   */
+  connectionPool?: PinnedConnectionPool
   /**
    * Where this request's URL came from. Carried on the options so the same
    * policy is re-applied to every redirect hop rather than re-derived — a hop
@@ -447,6 +460,43 @@ function resolveRedirectUrl(baseUrl: string, location: string): string {
   }
 }
 
+/** Keep-alive agents keyed by protocol, host, port, and the pinned address they connect to. */
+export interface PinnedConnectionPool {
+  /** Undefined once destroyed, so a late request falls back to a single-use pinned agent. */
+  agent(isHttps: boolean, host: string, port: number, resolvedIP: string): http.Agent | undefined
+  destroy(): void
+}
+
+/**
+ * Creates a request-scoped pool of pinned keep-alive agents. Reusing a connection skips the TCP
+ * and TLS handshakes that otherwise dominate short provider API calls.
+ */
+export function createPinnedConnectionPool(): PinnedConnectionPool {
+  const agents = new Map<string, http.Agent>()
+  let destroyed = false
+  return {
+    agent(isHttps, host, port, resolvedIP) {
+      if (destroyed) return undefined
+      const key = JSON.stringify([isHttps, host, port, resolvedIP])
+      let agent = agents.get(key)
+      if (!agent) {
+        const options: http.AgentOptions = {
+          keepAlive: true,
+          lookup: createPinnedLookup(resolvedIP),
+        }
+        agent = isHttps ? new https.Agent(options) : new http.Agent(options)
+        agents.set(key, agent)
+      }
+      return agent
+    },
+    destroy() {
+      destroyed = true
+      for (const agent of agents.values()) agent.destroy()
+      agents.clear()
+    },
+  }
+}
+
 /**
  * Creates a DNS lookup function that always returns a pre-resolved IP address.
  * Use this to prevent DNS rebinding (TOCTOU) attacks when connecting to
@@ -1088,6 +1138,11 @@ export async function secureFetchWithPinnedIP(
     const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort
 
     let agent: http.Agent | undefined
+    /**
+     * Bun ignores a `lookup` set on an Agent and honors one on the request, while Node honors
+     * both. A pinned direct connection sets it in both places so pinning holds in either runtime.
+     */
+    let pinnedLookup: LookupFunction | undefined
     if (outboundDispatcher) {
       agent = undefined
     } else if (options.proxyUrl) {
@@ -1096,12 +1151,17 @@ export async function secureFetchWithPinnedIP(
       // targets tunnel via CONNECT, http targets use absolute-URI forwarding.
       agent = isHttps ? new HttpsProxyAgent(options.proxyUrl) : new HttpProxyAgent(options.proxyUrl)
     } else {
-      const lookup = createPinnedLookup(resolvedIP)
-      const agentOptions: http.AgentOptions = { lookup }
-      agent = isHttps ? new https.Agent(agentOptions) : new http.Agent(agentOptions)
+      pinnedLookup = createPinnedLookup(resolvedIP)
+      agent =
+        options.connectionPool?.agent(isHttps, parsed.hostname, port, resolvedIP) ??
+        (isHttps
+          ? new https.Agent({ lookup: pinnedLookup })
+          : new http.Agent({ lookup: pinnedLookup }))
     }
 
     const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {}
+    /** Raw deflate streams are ambiguous to decode, so only gzip and brotli are requested. */
+    if (options.acceptCompressed) sanitizedHeaders['accept-encoding'] = 'gzip, br'
     if (!Object.keys(sanitizedHeaders).some((name) => name.toLowerCase() === 'user-agent')) {
       sanitizedHeaders['user-agent'] = DEFAULT_USER_AGENT
     }
@@ -1123,6 +1183,7 @@ export async function secureFetchWithPinnedIP(
       method: options.method || 'GET',
       headers: sanitizedHeaders,
       agent,
+      ...(pinnedLookup ? { lookup: pinnedLookup } : {}),
       timeout: options.timeout || 300000,
     }
 
@@ -1265,7 +1326,13 @@ export async function secureFetchWithPinnedIP(
         statusCode === 204 ||
         statusCode === 205 ||
         statusCode === 304
-      const contentLength = headersRecord['content-length']
+      /**
+       * An encoded body's Content-Length is its wire size, not the decoded size the cap bounds,
+       * so only an identity body is rejected up front; the decoded stream is capped as it reads.
+       */
+      const contentLength = headersRecord['content-encoding']
+        ? undefined
+        : headersRecord['content-length']
       if (contentLength && !isBodylessResponse) {
         const parsedLength = Number.parseInt(contentLength, 10)
         if (Number.isFinite(parsedLength) && parsedLength > maxResponseBytes) {