[dev] [carhartlewis] lewis/comp-audit-handoff-information-fixes-cs-103 (#2111)

* feat(context): resolve framework IDs to human-readable names in context entries

* refactor(auditor): exclude framework selection and auditor sections from context

---------

Co-authored-by: Lewis Carhart <lewis@trycomp.ai>

Author: github-actions[bot]

apps/app/src/app/(app)/[orgId]/settings/context-hub/data/getContextEntries.ts

@@ -4,6 +4,48 @@ import { headers } from 'next/headers';
 import { cache } from 'react';
 import 'server-only';
 
+const FRAMEWORK_ID_PATTERN = /\bfrk_[a-z0-9]+\b/g;
+
+/**
+ * Detects framework IDs (frk_xxx) in context entry answers and replaces them
+ * with human-readable framework names. Handles legacy data from before the
+ * write-time fix was applied.
+ */
+async function resolveFrameworkIdsInEntries<T extends { answer: string }>(
+  entries: T[],
+): Promise<T[]> {
+  // Collect all unique framework IDs across all entries
+  const allIds = new Set<string>();
+  for (const entry of entries) {
+    const matches = entry.answer.match(FRAMEWORK_ID_PATTERN);
+    if (matches) {
+      for (const id of matches) {
+        allIds.add(id);
+      }
+    }
+  }
+
+  if (allIds.size === 0) return entries;
+
+  // Batch-fetch framework names for all IDs
+  const frameworks = await db.frameworkEditorFramework.findMany({
+    where: { id: { in: Array.from(allIds) } },
+    select: { id: true, name: true },
+  });
+
+  const idToName = new Map(frameworks.map((f) => [f.id, f.name]));
+
+  // Replace IDs with names in each entry's answer
+  return entries.map((entry) => {
+    const resolvedAnswer = entry.answer.replace(
+      FRAMEWORK_ID_PATTERN,
+      (id) => idToName.get(id) ?? id,
+    );
+    if (resolvedAnswer === entry.answer) return entry;
+    return { ...entry, answer: resolvedAnswer };
+  });
+}
+
 export const getContextEntries = cache(
   async ({
     orgId,
@@ -42,6 +84,10 @@ export const getContextEntries = cache(
     });
     const total = await db.context.count({ where });
     const pageCount = Math.ceil(total / perPage);
-    return { data: entries, pageCount };
+
+    // Resolve any legacy framework IDs to display names
+    const resolvedEntries = await resolveFrameworkIdsInEntries(entries);
+
+    return { data: resolvedEntries, pageCount };
   },
 );

apps/app/src/app/(app)/setup/actions/create-organization-minimal.ts

@@ -2,9 +2,9 @@
 
 import { initializeOrganization } from '@/actions/organization/lib/initialize-organization';
 import { authActionClientWithoutOrg } from '@/actions/safe-action';
+import { env } from '@/env.mjs';
 import { createTrainingVideoEntries } from '@/lib/db/employee';
 import { auth } from '@/utils/auth';
-import { env } from '@/env.mjs';
 import { db } from '@db';
 import { revalidatePath } from 'next/cache';
 import { headers } from 'next/headers';
@@ -46,6 +46,13 @@ export const createOrganizationMinimal = authActionClientWithoutOrg
       // Check if self-hosted
       const isSelfHosted = env.NEXT_PUBLIC_SELF_HOSTED === 'true';
 
+      // Resolve framework IDs to display names (e.g. "SOC 2", "ISO 27001")
+      const frameworks = await db.frameworkEditorFramework.findMany({
+        where: { id: { in: parsedInput.frameworkIds } },
+        select: { name: true },
+      });
+      const frameworkNames = frameworks.map((f) => f.name).join(', ');
+
       // Create a new organization
       const newOrg = await db.organization.create({
         data: {
@@ -68,7 +75,7 @@ export const createOrganizationMinimal = authActionClientWithoutOrg
           context: {
             create: {
               question: 'Which compliance frameworks do you need?',
-              answer: parsedInput.frameworkIds.join(', '),
+              answer: frameworkNames || parsedInput.frameworkIds.join(', '),
               tags: ['onboarding'],
             },
           },

apps/app/src/app/(app)/setup/actions/create-organization.ts

@@ -41,6 +41,13 @@ export const createOrganization = authActionClientWithoutOrg
       // Create a new organization directly in the database
       const randomSuffix = Math.floor(100000 + Math.random() * 900000).toString();
 
+      // Resolve framework IDs to display names (e.g. "SOC 2", "ISO 27001")
+      const frameworks = await db.frameworkEditorFramework.findMany({
+        where: { id: { in: parsedInput.frameworkIds } },
+        select: { name: true },
+      });
+      const frameworkNames = frameworks.map((f) => f.name).join(', ');
+
       const newOrg = await db.organization.create({
         data: {
           name: parsedInput.organizationName,
@@ -62,7 +69,7 @@ export const createOrganization = authActionClientWithoutOrg
                 question: step.question,
                 answer:
                   step.key === 'frameworkIds'
-                    ? parsedInput.frameworkIds.join(', ')
+                    ? frameworkNames || parsedInput.frameworkIds.join(', ')
                     : (parsedInput[step.key as keyof typeof parsedInput] as string),
                 tags: ['onboarding'],
               })),

apps/app/src/trigger/tasks/auditor/generate-auditor-content.ts

@@ -1,5 +1,5 @@
 import { getOrganizationContext } from '@/trigger/tasks/onboarding/onboard-organization-helpers';
-import { groq } from '@ai-sdk/groq';
+import { openai } from '@ai-sdk/openai';
 import { db } from '@db';
 import { logger, metadata, schemaTask } from '@trigger.dev/sdk';
 import { generateText } from 'ai';
@@ -89,24 +89,54 @@ RULES:
 - No bullet points.
 ${TONE_RULES}`,
 
-  'critical-vendors': `List vendors in this EXACT format, one per line:
+  'critical-vendors': `Using the provided vendor/software list, narrow it down to ONLY the critical vendors from a SOC 2 perspective for the audit report.
 
+A critical vendor is one that:
+- Hosts or processes customer data (cloud infrastructure providers like AWS, GCP, Azure)
+- Provides core identity / authentication services (e.g. Okta, Google Workspace, Microsoft 365 — but ONLY if used as the primary identity provider)
+- Is essential to the company's production system or service delivery
+- Handles sensitive data (e.g. payment processors IF the company processes payments as a core service)
+
+DO NOT INCLUDE vendors that are:
+- Internal productivity / collaboration tools (e.g. Notion, Slack, Teams, Jira, Confluence, Asana)
+- General business tools (e.g. Stripe, HubSpot, Intercom, Zendesk)
+- HR / payroll tools (e.g. Rippling, Gusto, BambooHR)
+- Marketing or analytics tools
+- Version control or CI/CD tools (e.g. GitHub, GitLab) unless they host production infrastructure
+- Security monitoring tools (e.g. Vanta, Drata, CrowdStrike)
+
+Typically a SOC 2 report includes only 3-6 critical vendors. Be very selective.
+
+FORMAT — one vendor per line:
 [Vendor Name] – [Type: SaaS/IaaS/PaaS] – ([Brief description of service])
 
 EXAMPLE:
-Zoom – SaaS – (Video conferencing / collaboration)
 AWS – IaaS / PaaS – (Cloud infrastructure and hosting)
-Microsoft 365 – SaaS – (Office productivity and identity)
+Google Workspace – SaaS – (Primary identity provider and email)
+Datadog – SaaS – (Production monitoring and observability)
 
 RULES:
 - Do NOT include the section title.
 - Each vendor on its own line.
 - Follow the exact format: Name – Type – (Description)
-- Only include vendors explicitly mentioned in sources.
+- Only include vendors from the provided sources — do not add vendors not mentioned.
+- Aim for 3-6 vendors maximum.
 ${TONE_RULES}`,
 
-  'subservice-organizations': `List subservice organizations in this EXACT format:
+  'subservice-organizations': `Identify the subservice organisations from a SOC 2 perspective.
+
+A subservice organisation is an external service provider whose infrastructure or platform the company DIRECTLY RELIES ON to deliver its own services to customers. In SOC 2 terms, these are typically the main cloud infrastructure / hosting providers (IaaS/PaaS) — e.g. AWS, Google Cloud Platform, Microsoft Azure.
+
+DO NOT INCLUDE:
+- SaaS tools the company merely uses internally (e.g. Slack, Notion, Jira, GitHub, Stripe, HubSpot)
+- Communication or collaboration platforms (e.g. Teams, Zoom)
+- HR, payroll, or admin tools
+- Security or monitoring tools
+- Any tool that is NOT the primary infrastructure hosting the company's production system
+
+Typically there is only 1 (sometimes 2) subservice organisations. Be very selective.
 
+FORMAT:
 Subservice organisations: [Name1], [Name2], ...
 
 If only one: "Subservice organisations: [Name]"
@@ -118,7 +148,7 @@ RULES:
 - Do NOT include the section title.
 - Use "Subservice organisations:" prefix.
 - Just list the names, comma-separated if multiple.
-- Only include organizations explicitly mentioned as subservice providers in sources.
+- Look for where the company hosts its applications and data — that is the subservice organisation.
 ${TONE_RULES}`,
 };
 
@@ -145,7 +175,7 @@ async function scrapeWebsite(website: string): Promise<string> {
       urls: [website],
       prompt:
         'Extract all text content from this website, including company information, services, mission, vision, and any other relevant business information. Return the content as plain text or markdown.',
-      limit: 10
+      limit: 10,
     }),
   });
 
@@ -223,7 +253,7 @@ async function generateSectionContent(
   contextHubText: string,
 ): Promise<string> {
   const { text } = await generateText({
-    model: groq('openai/gpt-oss-120b'),
+    model: openai('gpt-5.2'),
     system: `You are an expert at extracting and organizing company information for audit purposes.
 
 CRITICAL RULES:
@@ -326,10 +356,16 @@ export const generateAuditorContentTask = schemaTask({
         };
       }
 
-      // Build context from organization data (excluding auditor sections to avoid circular reference)
+      // Build context from organization data, excluding:
+      // 1. Auditor sections (to avoid circular reference)
+      // 2. Framework selection (contains raw IDs like "frk_xxx" and isn't relevant to auditor content)
       const auditorQuestions = new Set(Object.values(SECTION_QUESTIONS));
+      const excludedQuestions = new Set([
+        ...auditorQuestions,
+        'Which compliance frameworks do you need?',
+      ]);
       const contextHubText = questionsAndAnswers
-        .filter((qa) => !auditorQuestions.has(qa.question))
+        .filter((qa) => !excludedQuestions.has(qa.question))
         .map((qa) => `Q: ${qa.question}\nA: ${qa.answer}`)
         .join('\n\n');