evaluator.js

/**
 * @module evaluator
 * Semantic Plausibility Gate and Advanced Evaluation module.
 * Mitigates Node Poisoning and implements LLM-as-a-Judge patterns.
 */

import { chatJson } from './llm.js';
import { envOr } from './config.js';

/**
 * Fast Plausibility Screening.
 * Mitigates Semantic Node Poisoning by verifying that generated output
 * does not introduce hallucinated variables, claims, or facts not present
 * in the source context.
 *
 * @param {string} sourceContext - The ground truth / retrieved RAG context.
 * @param {string} generatedOutput - The output generated by the previous step.
 * @param {object} config - LLM config object.
 * @param {object} [options={}] - Options, e.g. `{ useFastModel: true }`.
 * @returns {Promise<{ isPlausible: boolean, reason: string }>}
 */
export async function evaluatePlausibility(sourceContext, generatedOutput, config, options = {}) {
    const systemPrompt = `You are a strict, zero-tolerance Plausibility Evaluator.
Your ONLY job is to determine if the Generated Output hallucinates or introduces facts, variables, or claims NOT present in the Source Context.

Rules:
1. If the Output is supported by or reasonably inferred from the Source Context, it is Plausible (PASS).
2. If the Output invents names, numbers, rules, or claims not found in the Context, it is Implausible (FAIL).
3. Do not evaluate style, tone, or formatting. Only semantic truth against the source.

Return a JSON object:
{
  "reasoning": "1. List all claims/numbers/entities in the Output.\n2. Search for each one in the Source Context.\n3. Identify any claim that is missing from the Source.",
  "isPlausible": boolean // MUST be false if any claim is missing or hallucinated
}`;

    const userMessage = `## Source Context
${sourceContext}

## Generated Output
${generatedOutput}`;

    // Plausibility prefers fast models to avoid latency bloat, but defaults to config model
    const resolvedConfig = { ...config, temperature: 0.0 };
    
    // Default to qwen2.5-coder:7b for fast plausibility checking if fast model requested
    if (options.useFastModel && config.provider === 'ollama') {
        resolvedConfig.model = envOr('OLLAMA_FAST_MODEL', null, 'qwen2.5-coder:7b');
    }

    const result = await chatJson(systemPrompt, userMessage, resolvedConfig, options);
    
    return {
        isPlausible: !!result.isPlausible,
        reason: result.reasoning || "No reason provided."
    };
}

/**
 * Pairwise Preference Evaluation.
 * Compares two responses, running a dual-pass to mitigate Position Bias.
 *
 * @param {string} prompt - The original prompt/context.
 * @param {string} responseA - First response.
 * @param {string} responseB - Second response.
 * @param {Array<string>} criteria - Criteria for comparison.
 * @param {object} config - LLM config object.
 * @param {object} [options={}] - Extra options.
 * @returns {Promise<{ winner: string, confidence: number, reason: string }>}
 */
export async function evaluatePairwise(prompt, responseA, responseB, criteria, config, options = {}) {
    const systemPrompt = `You are an expert evaluator comparing two AI responses.

## Critical Instructions
- Do NOT prefer responses because they are longer.
- Do NOT prefer responses based on position.
- Focus ONLY on quality according to the specified criteria: ${criteria.join(', ')}
- Ties are acceptable when responses are genuinely equivalent.

## Output Format
Respond with JSON:
{
  "winner": "A" | "B" | "TIE",
  "confidence": 0.0 to 1.0,
  "reason": "Brief justification of why the winner was chosen based on criteria."
}`;

    // Pass 1: A vs B
    const userMessage1 = `## Original Prompt\n${prompt}\n\n## Response A\n${responseA}\n\n## Response B\n${responseB}`;
    const result1 = await chatJson(systemPrompt, userMessage1, config, options);

    // Pass 2: B vs A (Swapped)
    const userMessage2 = `## Original Prompt\n${prompt}\n\n## Response A\n${responseB}\n\n## Response B\n${responseA}`;
    const result2 = await chatJson(systemPrompt, userMessage2, config, options);

    // Normalize Pass 2 winner (since Response A was actually B)
    let mappedWinner2 = result2.winner;
    if (mappedWinner2 === 'A') mappedWinner2 = 'B';
    else if (mappedWinner2 === 'B') mappedWinner2 = 'A';

    // Check consistency
    if (result1.winner === mappedWinner2 && result1.winner !== 'TIE') {
        return {
            winner: result1.winner,
            confidence: (parseFloat(result1.confidence) + parseFloat(result2.confidence)) / 2,
            reason: `Consistent winner across both position passes. Pass 1 logic: ${result1.reason}`
        };
    } else {
        return {
            winner: 'TIE',
            confidence: 0.5,
            reason: `Position inconsistency detected. Pass 1 preferred ${result1.winner}, Pass 2 preferred ${mappedWinner2}.`
        };
    }
}

/**
 * Direct Scoring Evaluation.
 * Evaluates an output against a specific criterion, forcing chain-of-thought justification first.
 *
 * @param {string} prompt - The original prompt.
 * @param {string} response - The generated response.
 * @param {object} criterion - `{ name: string, description: string }`
 * @param {number} maxScore - Maximum score (e.g., 5).
 * @param {object} config - LLM config object.
 * @param {object} [options={}] - Extra options.
 * @returns {Promise<{ score: number, evidence: string[], justification: string, improvement: string }>}
 */
export async function evaluateDirect(prompt, response, criterion, maxScore, config, options = {}) {
    const systemPrompt = `You are an expert evaluator assessing response quality.

## Task
Evaluate the response against the following criterion.

Criterion: ${criterion.name}
Description: ${criterion.description}

## Instructions
1. Find specific evidence in the response.
2. Justify your score with evidence FIRST.
3. Score according to a 1-${maxScore} scale.
4. Suggest one specific improvement.

## Output Format
Respond with JSON:
{
  "evidence": ["evidence 1", "evidence 2"],
  "justification": "Explanation of reasoning...",
  "improvement": "Specific suggestion...",
  "score": integer between 1 and ${maxScore}
}`;

    const userMessage = `## Original Prompt\n${prompt}\n\n## Response to Evaluate\n${response}`;
    const result = await chatJson(systemPrompt, userMessage, config, options);
    
    return {
        score: parseInt(result.score) || 1,
        evidence: Array.isArray(result.evidence) ? result.evidence : [],
        justification: result.justification || "",
        improvement: result.improvement || ""
    };
}