import { AiAgentConfidence, AiAgentToolCall, AiAgentToolName, AiAgentVerificationCheck } from './ai-agent.interfaces'; const CANDIDATE_TICKER_PATTERN = /\$?[A-Za-z0-9.]{1,10}/g; const NORMALIZED_TICKER_PATTERN = /^(?=.*[A-Z])[A-Z0-9]{1,6}(?:\.[A-Z0-9]{1,4})?$/; const SYMBOL_STOP_WORDS = new Set([ 'AND', 'FOR', 'GIVE', 'HELP', 'I', 'IS', 'MARKET', 'OF', 'PLEASE', 'PORTFOLIO', 'PRICE', 'QUOTE', 'RISK', 'SHOW', 'SYMBOL', 'THE', 'TICKER', 'WHAT', 'WITH' ]); const INVESTMENT_INTENT_KEYWORDS = [ 'add', 'allocat', 'buy', 'invest', 'next', 'rebalanc', 'sell', 'trim' ]; const REBALANCE_KEYWORDS = [ 'rebalanc', 'reduce', 'trim', 'underweight', 'overweight' ]; const STRESS_TEST_KEYWORDS = ['crash', 'drawdown', 'shock', 'stress']; const ANSWER_NUMERIC_INTENT_KEYWORDS = [ 'allocat', 'drawdown', 'hhi', 'market', 'performance', 'price', 'quote', 'return', 'risk', 'shock', 'stress', 'trim' ]; const ANSWER_ACTIONABLE_KEYWORDS = [ 'add', 'allocate', 'buy', 'hedge', 'increase', 'monitor', 'rebalance', 'reduce', 'sell', 'trim' ]; const DISALLOWED_RESPONSE_PATTERNS = [ /\bas an ai\b/i, /\bi am not (?:a|your) financial advisor\b/i, /\bi can(?:not|'t) provide financial advice\b/i, /\bconsult (?:a|your) financial advisor\b/i ]; const MINIMUM_GENERATED_ANSWER_WORDS = 12; interface AnswerQualitySignals { disallowedPhraseDetected: boolean; hasActionableGuidance: boolean; hasInvestmentIntent: boolean; hasNumericIntent: boolean; hasNumericSignal: boolean; sentenceCount: number; wordCount: number; } function getAnswerQualitySignals({ answer, query }: { answer: string; query: string; }): AnswerQualitySignals { const normalizedAnswer = answer.trim(); const normalizedAnswerLowerCase = normalizedAnswer.toLowerCase(); const normalizedQueryLowerCase = query.toLowerCase(); const words = normalizedAnswer.split(/\s+/).filter(Boolean); const sentenceCount = normalizedAnswer .split(/[.!?](?:\s+|$)/) .map((sentence) => sentence.trim()) .filter(Boolean).length; const hasInvestmentIntent = INVESTMENT_INTENT_KEYWORDS.some((keyword) => { return normalizedQueryLowerCase.includes(keyword); }); const hasNumericIntent = ANSWER_NUMERIC_INTENT_KEYWORDS.some((keyword) => { return normalizedQueryLowerCase.includes(keyword); }); const hasActionableGuidance = ANSWER_ACTIONABLE_KEYWORDS.some((keyword) => { return normalizedAnswerLowerCase.includes(keyword); }); const hasNumericSignal = /\d/.test(normalizedAnswer); const disallowedPhraseDetected = DISALLOWED_RESPONSE_PATTERNS.some((pattern) => { return pattern.test(normalizedAnswer); }); return { disallowedPhraseDetected, hasActionableGuidance, hasInvestmentIntent, hasNumericIntent, hasNumericSignal, sentenceCount, wordCount: words.length }; } export function isGeneratedAnswerReliable({ answer, query }: { answer: string; query: string; }) { const qualitySignals = getAnswerQualitySignals({ answer, query }); if (qualitySignals.disallowedPhraseDetected) { return false; } if (qualitySignals.wordCount < MINIMUM_GENERATED_ANSWER_WORDS) { return false; } if (qualitySignals.hasInvestmentIntent && !qualitySignals.hasActionableGuidance) { return false; } if (qualitySignals.hasNumericIntent && !qualitySignals.hasNumericSignal) { return false; } return true; } export function evaluateAnswerQuality({ answer, query }: { answer: string; query: string; }): AiAgentVerificationCheck { const qualitySignals = getAnswerQualitySignals({ answer, query }); const issues: string[] = []; if (qualitySignals.disallowedPhraseDetected) { issues.push('Response contains a generic AI disclaimer'); } if (qualitySignals.wordCount < MINIMUM_GENERATED_ANSWER_WORDS) { issues.push( `Response length is short (${qualitySignals.wordCount} words; target >= ${MINIMUM_GENERATED_ANSWER_WORDS})` ); } if (qualitySignals.sentenceCount < 2) { issues.push( `Response uses limited structure (${qualitySignals.sentenceCount} sentence)` ); } if (qualitySignals.hasInvestmentIntent && !qualitySignals.hasActionableGuidance) { issues.push('Investment request lacks explicit action guidance'); } if (qualitySignals.hasNumericIntent && !qualitySignals.hasNumericSignal) { issues.push('Quantitative query response lacks numeric support'); } if (qualitySignals.disallowedPhraseDetected) { return { check: 'response_quality', details: issues.join('; '), status: 'failed' }; } return { check: 'response_quality', details: issues.length > 0 ? issues.join('; ') : 'Response passed structure, actionability, and evidence heuristics', status: issues.length === 0 ? 'passed' : 'warning' }; } function normalizeSymbolCandidate(rawCandidate: string) { const hasDollarPrefix = rawCandidate.startsWith('$'); const candidate = hasDollarPrefix ? rawCandidate.slice(1) : rawCandidate; if (!candidate) { return null; } const normalized = candidate.toUpperCase(); if (SYMBOL_STOP_WORDS.has(normalized)) { return null; } if (!NORMALIZED_TICKER_PATTERN.test(normalized)) { return null; } // Conservative mode for non-prefixed symbols avoids false positives from // natural language words such as WHAT/THE/AND. if (!hasDollarPrefix && candidate !== candidate.toUpperCase()) { return null; } return normalized; } export function extractSymbolsFromQuery(query: string) { const matches = query.match(CANDIDATE_TICKER_PATTERN) ?? []; return Array.from( new Set( matches .map((candidate) => normalizeSymbolCandidate(candidate)) .filter(Boolean) ) ); } export function determineToolPlan({ query, symbols }: { query: string; symbols?: string[]; }): AiAgentToolName[] { const normalizedQuery = query.toLowerCase(); const selectedTools = new Set(); const extractedSymbols = symbols?.length ? symbols : extractSymbolsFromQuery(query); const hasInvestmentIntent = INVESTMENT_INTENT_KEYWORDS.some((keyword) => { return normalizedQuery.includes(keyword); }); const hasRebalanceIntent = REBALANCE_KEYWORDS.some((keyword) => { return normalizedQuery.includes(keyword); }); const hasStressTestIntent = STRESS_TEST_KEYWORDS.some((keyword) => { return normalizedQuery.includes(keyword); }); if ( normalizedQuery.includes('portfolio') || normalizedQuery.includes('holding') || normalizedQuery.includes('allocation') || normalizedQuery.includes('performance') || normalizedQuery.includes('return') ) { selectedTools.add('portfolio_analysis'); } if ( normalizedQuery.includes('risk') || normalizedQuery.includes('concentration') || normalizedQuery.includes('diversif') ) { selectedTools.add('portfolio_analysis'); selectedTools.add('risk_assessment'); } if (hasInvestmentIntent || hasRebalanceIntent) { selectedTools.add('portfolio_analysis'); selectedTools.add('risk_assessment'); selectedTools.add('rebalance_plan'); } if (hasStressTestIntent) { selectedTools.add('portfolio_analysis'); selectedTools.add('risk_assessment'); selectedTools.add('stress_test'); } if ( normalizedQuery.includes('quote') || normalizedQuery.includes('price') || normalizedQuery.includes('market') || normalizedQuery.includes('ticker') || extractedSymbols.length > 0 ) { selectedTools.add('market_data_lookup'); } return Array.from(selectedTools); } export function calculateConfidence({ toolCalls, verification }: { toolCalls: AiAgentToolCall[]; verification: AiAgentVerificationCheck[]; }): AiAgentConfidence { const successfulToolCalls = toolCalls.filter(({ status }) => { return status === 'success'; }).length; const passedVerification = verification.filter(({ status }) => { return status === 'passed'; }).length; const failedVerification = verification.filter(({ status }) => { return status === 'failed'; }).length; const toolSuccessRate = toolCalls.length > 0 ? successfulToolCalls / toolCalls.length : 0; const verificationPassRate = verification.length > 0 ? passedVerification / verification.length : 0; let score = 0.4 + toolSuccessRate * 0.35 + verificationPassRate * 0.25; score -= failedVerification * 0.1; score = Math.max(0, Math.min(1, score)); let band: AiAgentConfidence['band'] = 'low'; if (score >= 0.8) { band = 'high'; } else if (score >= 0.6) { band = 'medium'; } return { band, score: Number(score.toFixed(2)) }; }