Browse Source

fix(ai): wrap provider calls in langchain and restore arithmetic direct replies

pull/6395/head
Max P 1 month ago
parent
commit
b5dbfd6081
  1. 2
      Tasks.md
  2. 208
      apps/api/src/app/endpoints/ai/ai-agent.policy.utils.ts
  3. 13
      apps/api/src/app/endpoints/ai/ai-agent.utils.spec.ts
  4. 17
      apps/api/src/app/endpoints/ai/ai.service.spec.ts
  5. 80
      apps/api/src/app/endpoints/ai/ai.service.ts
  6. 170
      package-lock.json
  7. 1
      package.json
  8. 11
      tasks/tasks.md

2
Tasks.md

@ -17,6 +17,7 @@ Last updated: 2026-02-24
| T-009 | Open source eval framework contribution | In Review | `@ghostfolio/finance-agent-evals` package scaffold + dataset export + smoke/pack checks | openai/evals PR #1625 + langchain PR #35421 |
| T-010 | Chat history persistence and simple direct-query handling | Complete | `apps/client/src/app/pages/portfolio/analysis/ai-chat-panel/ai-chat-panel.component.spec.ts`, `apps/api/src/app/endpoints/ai/ai-agent.utils.spec.ts`, `apps/api/src/app/endpoints/ai/ai.service.spec.ts` | Local implementation |
| T-011 | Per-LLM LangSmith invocation tracing + production tracing env enablement | Complete | `apps/api/src/app/endpoints/ai/ai-observability.service.spec.ts`, `apps/api/src/app/endpoints/ai/ai.service.spec.ts`, `apps/api/src/app/endpoints/ai/ai-performance.spec.ts`, `apps/api/src/app/endpoints/ai/evals/mvp-eval.runner.spec.ts`, `apps/api/src/app/endpoints/ai/evals/ai-quality-eval.spec.ts` | Local implementation + Railway variable update |
| T-012 | LangChain wrapper enforcement for provider calls + arithmetic direct-response correction | Complete | `apps/api/src/app/endpoints/ai/ai.service.spec.ts`, `apps/api/src/app/endpoints/ai/ai-agent.utils.spec.ts`, `npm run test:ai` | Local implementation |
## Notes
@ -37,3 +38,4 @@ Last updated: 2026-02-24
- Tool gating hardening (2026-02-24): planner unknown-intent fallback changed to no-tools, executor policy gate added (`direct|tools|clarify`), and policy metrics emitted via verification and observability logs.
- Chat persistence + simple direct-query handling (2026-02-24): client chat panel now restores/persists session + bounded message history via localStorage and policy no-tool prompts now return assistant capability guidance for queries like "Who are you?".
- Per-LLM LangSmith invocation tracing (2026-02-24): each provider call now records an explicit LangSmith `llm` run (provider/model/query/session/response metadata), and production Railway env now has tracing variables enabled.
- Direct arithmetic no-tool behavior fix (2026-02-24): simple arithmetic prompts now return computed answers (for example `2+2 = 4`) instead of generic capability guidance.

208
apps/api/src/app/endpoints/ai/ai-agent.policy.utils.ts

@ -28,6 +28,7 @@ const GREETING_ONLY_PATTERN =
const SIMPLE_ARITHMETIC_QUERY_PATTERN =
/^\s*(?:what(?:'s| is)\s+)?[-+*/().\d\s%=]+\??\s*$/i;
const SIMPLE_ARITHMETIC_OPERATOR_PATTERN = /[+\-*/]/;
const SIMPLE_ARITHMETIC_PREFIX_PATTERN = /^\s*(?:what(?:'s| is)\s+)?/i;
const SIMPLE_ASSISTANT_QUERY_PATTERNS = [
/^\s*(?:who are you|what are you|what can you do)\s*[!.?]*\s*$/i,
/^\s*(?:how do you work|how (?:can|do) i use (?:you|this))\s*[!.?]*\s*$/i,
@ -94,6 +95,203 @@ function isNoToolDirectQuery(query: string) {
);
}
function formatNumericResult(value: number) {
if (Math.abs(value) < Number.EPSILON) {
return '0';
}
if (Number.isInteger(value)) {
return value.toString();
}
return value.toFixed(6).replace(/\.?0+$/, '');
}
function evaluateArithmeticExpression(expression: string) {
let cursor = 0;
const skipWhitespace = () => {
while (cursor < expression.length && /\s/.test(expression[cursor])) {
cursor += 1;
}
};
const parseNumber = () => {
skipWhitespace();
const start = cursor;
let dotCount = 0;
while (cursor < expression.length) {
const token = expression[cursor];
if (token >= '0' && token <= '9') {
cursor += 1;
continue;
}
if (token === '.') {
dotCount += 1;
if (dotCount > 1) {
return undefined;
}
cursor += 1;
continue;
}
break;
}
if (start === cursor) {
return undefined;
}
const parsed = Number(expression.slice(start, cursor));
return Number.isFinite(parsed) ? parsed : undefined;
};
const parseFactor = (): number | undefined => {
skipWhitespace();
if (expression[cursor] === '+') {
cursor += 1;
return parseFactor();
}
if (expression[cursor] === '-') {
cursor += 1;
const nested = parseFactor();
return nested === undefined ? undefined : -nested;
}
if (expression[cursor] === '(') {
cursor += 1;
const nested = parseExpression();
skipWhitespace();
if (nested === undefined || expression[cursor] !== ')') {
return undefined;
}
cursor += 1;
return nested;
}
return parseNumber();
};
const parseTerm = (): number | undefined => {
let value = parseFactor();
if (value === undefined) {
return undefined;
}
while (true) {
skipWhitespace();
const operator = expression[cursor];
if (operator !== '*' && operator !== '/') {
break;
}
cursor += 1;
const right = parseFactor();
if (right === undefined) {
return undefined;
}
if (operator === '*') {
value *= right;
} else {
if (Math.abs(right) < Number.EPSILON) {
return undefined;
}
value /= right;
}
}
return value;
};
const parseExpression = (): number | undefined => {
let value = parseTerm();
if (value === undefined) {
return undefined;
}
while (true) {
skipWhitespace();
const operator = expression[cursor];
if (operator !== '+' && operator !== '-') {
break;
}
cursor += 1;
const right = parseTerm();
if (right === undefined) {
return undefined;
}
if (operator === '+') {
value += right;
} else {
value -= right;
}
}
return value;
};
const result = parseExpression();
skipWhitespace();
if (result === undefined || cursor !== expression.length || !Number.isFinite(result)) {
return undefined;
}
return result;
}
function evaluateSimpleArithmetic(query: string) {
const normalized = query.trim();
if (
!SIMPLE_ARITHMETIC_QUERY_PATTERN.test(normalized) ||
!SIMPLE_ARITHMETIC_OPERATOR_PATTERN.test(normalized) ||
!/\d/.test(normalized)
) {
return undefined;
}
const expression = normalized
.replace(SIMPLE_ARITHMETIC_PREFIX_PATTERN, '')
.replace(/\?+$/, '')
.replace(/=/g, '')
.trim();
if (!expression) {
return undefined;
}
const result = evaluateArithmeticExpression(expression);
if (result === undefined) {
return undefined;
}
return `${expression} = ${formatNumericResult(result)}`;
}
export function applyToolExecutionPolicy({
plannedTools,
query
@ -188,9 +386,11 @@ export function applyToolExecutionPolicy({
}
export function createPolicyRouteResponse({
policyDecision
policyDecision,
query
}: {
policyDecision: AiAgentToolPolicyDecision;
query?: string;
}) {
if (policyDecision.route === 'clarify') {
if (policyDecision.blockReason === 'needs_confirmation') {
@ -204,6 +404,12 @@ export function createPolicyRouteResponse({
policyDecision.route === 'direct' &&
policyDecision.blockReason === 'no_tool_query'
) {
const arithmeticResult = query ? evaluateSimpleArithmetic(query) : undefined;
if (arithmeticResult) {
return arithmeticResult;
}
return `I am your Ghostfolio AI assistant. I can help with portfolio analysis, concentration risk, market prices, rebalancing ideas, and stress scenarios. Try: "Show my top holdings" or "What is my concentration risk?".`;
}

13
apps/api/src/app/endpoints/ai/ai-agent.utils.spec.ts

@ -83,6 +83,19 @@ describe('AiAgentUtils', () => {
);
});
it('returns deterministic arithmetic result for direct no-tool arithmetic query', () => {
const decision = applyToolExecutionPolicy({
plannedTools: [],
query: '2+2'
});
expect(decision.route).toBe('direct');
expect(decision.toolsToExecute).toEqual([]);
expect(createPolicyRouteResponse({ policyDecision: decision, query: '2+2' })).toBe(
'2+2 = 4'
);
});
it('keeps finance-intent prompts on clarify route even with capability phrasing', () => {
const decision = applyToolExecutionPolicy({
plannedTools: [],

17
apps/api/src/app/endpoints/ai/ai.service.spec.ts

@ -274,6 +274,23 @@ describe('AiService', () => {
);
});
it('returns arithmetic response on direct no-tool arithmetic query', async () => {
redisCacheService.get.mockResolvedValue(undefined);
const generateTextSpy = jest.spyOn(subject, 'generateText');
const result = await subject.chat({
languageCode: 'en',
query: '2+2',
sessionId: 'session-arithmetic-route',
userCurrency: 'USD',
userId: 'user-arithmetic-route'
});
expect(result.answer).toBe('2+2 = 4');
expect(result.toolCalls).toEqual([]);
expect(generateTextSpy).not.toHaveBeenCalled();
});
it('runs rebalance and stress test tools for portfolio scenario prompts', async () => {
portfolioService.getDetails.mockResolvedValue({
holdings: {

80
apps/api/src/app/endpoints/ai/ai.service.ts

@ -10,6 +10,7 @@ import { Filter } from '@ghostfolio/common/interfaces';
import type { AiPromptMode } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { createOpenRouter } from '@openrouter/ai-sdk-provider';
import { RunnableLambda } from '@langchain/core/runnables';
import { generateText } from 'ai';
import { randomUUID } from 'node:crypto';
import {
@ -85,31 +86,71 @@ export class AiService {
provider: string;
run: () => Promise<{ text?: string }>;
}) => {
const startedAt = Date.now();
let invocationError: unknown;
let responseText: string | undefined;
try {
const response = await run();
responseText = response?.text;
const invocationRunnable = RunnableLambda.from(
async ({
model: runnableModel,
prompt: runnablePrompt,
provider: runnableProvider,
query,
sessionId,
userId
}: {
model: string;
prompt: string;
provider: string;
query?: string;
sessionId?: string;
userId?: string;
}) => {
const startedAt = Date.now();
let invocationError: unknown;
let responseText: string | undefined;
try {
const response = await run();
responseText = response?.text;
return response;
} catch (error) {
invocationError = error;
throw error;
} finally {
void this.aiObservabilityService.recordLlmInvocation({
durationInMs: Date.now() - startedAt,
error: invocationError,
model: runnableModel,
prompt: runnablePrompt,
provider: runnableProvider,
query,
responseText,
sessionId,
userId
});
}
}
);
return response;
} catch (error) {
invocationError = error;
throw error;
} finally {
void this.aiObservabilityService.recordLlmInvocation({
durationInMs: Date.now() - startedAt,
error: invocationError,
return invocationRunnable.invoke(
{
model,
prompt,
provider,
query: traceContext?.query,
responseText,
sessionId: traceContext?.sessionId,
userId: traceContext?.userId
});
}
},
{
metadata: {
model,
provider,
query: traceContext?.query ?? '',
sessionId: traceContext?.sessionId ?? '',
userId: traceContext?.userId ?? ''
},
runName: `ghostfolio_ai_llm_${provider}`,
tags: ['ghostfolio-ai', 'llm-invocation', provider]
}
);
};
if (zAiGlmApiKey) {
@ -393,7 +434,8 @@ export class AiService {
});
let answer = createPolicyRouteResponse({
policyDecision
policyDecision,
query: normalizedQuery
});
if (policyDecision.route === 'tools') {

170
package-lock.json

@ -26,6 +26,7 @@
"@internationalized/number": "3.6.5",
"@ionic/angular": "8.7.8",
"@keyv/redis": "4.4.0",
"@langchain/core": "^0.3.80",
"@nestjs/bull": "11.0.4",
"@nestjs/cache-manager": "3.0.1",
"@nestjs/common": "11.1.8",
@ -3633,6 +3634,12 @@
"devOptional": true,
"license": "(Apache-2.0 AND BSD-3-Clause)"
},
"node_modules/@cfworker/json-schema": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz",
"integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==",
"license": "MIT"
},
"node_modules/@chevrotain/cst-dts-gen": {
"version": "11.0.3",
"resolved": "https://registry.npmjs.org/@chevrotain/cst-dts-gen/-/cst-dts-gen-11.0.3.tgz",
@ -6292,6 +6299,150 @@
"integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==",
"license": "MIT"
},
"node_modules/@langchain/core": {
"version": "0.3.80",
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.80.tgz",
"integrity": "sha512-vcJDV2vk1AlCwSh3aBm/urQ1ZrlXFFBocv11bz/NBUfLWD5/UDNMzwPdaAd2dKvNmTWa9FM2lirLU3+JCf4cRA==",
"license": "MIT",
"dependencies": {
"@cfworker/json-schema": "^4.0.2",
"ansi-styles": "^5.0.0",
"camelcase": "6",
"decamelize": "1.2.0",
"js-tiktoken": "^1.0.12",
"langsmith": "^0.3.67",
"mustache": "^4.2.0",
"p-queue": "^6.6.2",
"p-retry": "4",
"uuid": "^10.0.0",
"zod": "^3.25.32",
"zod-to-json-schema": "^3.22.3"
},
"engines": {
"node": ">=18"
}
},
"node_modules/@langchain/core/node_modules/@types/retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
"integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==",
"license": "MIT"
},
"node_modules/@langchain/core/node_modules/ansi-styles": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
"integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/@langchain/core/node_modules/camelcase": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
"integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@langchain/core/node_modules/chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"license": "MIT",
"dependencies": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/@langchain/core/node_modules/chalk/node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"license": "MIT",
"dependencies": {
"color-convert": "^2.0.1"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/@langchain/core/node_modules/langsmith": {
"version": "0.3.87",
"resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.87.tgz",
"integrity": "sha512-XXR1+9INH8YX96FKWc5tie0QixWz6tOqAsAKfcJyPkE0xPep+NDz0IQLR32q4bn10QK3LqD2HN6T3n6z1YLW7Q==",
"license": "MIT",
"dependencies": {
"@types/uuid": "^10.0.0",
"chalk": "^4.1.2",
"console-table-printer": "^2.12.1",
"p-queue": "^6.6.2",
"semver": "^7.6.3",
"uuid": "^10.0.0"
},
"peerDependencies": {
"@opentelemetry/api": "*",
"@opentelemetry/exporter-trace-otlp-proto": "*",
"@opentelemetry/sdk-trace-base": "*",
"openai": "*"
},
"peerDependenciesMeta": {
"@opentelemetry/api": {
"optional": true
},
"@opentelemetry/exporter-trace-otlp-proto": {
"optional": true
},
"@opentelemetry/sdk-trace-base": {
"optional": true
},
"openai": {
"optional": true
}
}
},
"node_modules/@langchain/core/node_modules/p-retry": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz",
"integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==",
"license": "MIT",
"dependencies": {
"@types/retry": "0.12.0",
"retry": "^0.13.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@langchain/core/node_modules/uuid": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
"integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/@leichtgewicht/ip-codec": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz",
@ -24650,6 +24801,15 @@
"license": "MIT",
"peer": true
},
"node_modules/js-tiktoken": {
"version": "1.0.21",
"resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz",
"integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==",
"license": "MIT",
"dependencies": {
"base64-js": "^1.5.1"
}
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@ -26735,6 +26895,15 @@
"multicast-dns": "cli.js"
}
},
"node_modules/mustache": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz",
"integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==",
"license": "MIT",
"bin": {
"mustache": "bin/mustache"
}
},
"node_modules/mute-stream": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz",
@ -31127,7 +31296,6 @@
"version": "0.13.1",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
"integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 4"

1
package.json

@ -85,6 +85,7 @@
"@internationalized/number": "3.6.5",
"@ionic/angular": "8.7.8",
"@keyv/redis": "4.4.0",
"@langchain/core": "^0.3.80",
"@nestjs/bull": "11.0.4",
"@nestjs/cache-manager": "3.0.1",
"@nestjs/common": "11.1.8",

11
tasks/tasks.md

@ -209,6 +209,13 @@ Last updated: 2026-02-24
- [x] Update AI and observability unit tests to assert LLM invocation trace behavior and keep provider fallback behavior stable.
- [x] Run focused verification for touched AI suites and update task tracking notes.
## Session Plan (2026-02-24, LangChain Wrapper + Arithmetic Direct Reply Correction)
- [x] Enforce provider invocation through LangChain runnable wrapper in `AiService.generateText`.
- [x] Fix no-tool arithmetic prompts to return computed deterministic replies instead of capability fallback text.
- [x] Add/update unit tests for arithmetic direct replies and provider tracing/fallback behavior.
- [x] Run focused verification (`test:ai` and `api:lint`) and update tracker notes.
## Verification Notes
- `nx run api:lint` completed successfully (existing workspace warnings only).
@ -282,3 +289,7 @@ Last updated: 2026-02-24
- `npx nx run api:lint` (passes with existing workspace warnings)
- `railway variable set -s ghostfolio-api --skip-deploys LANGCHAIN_API_KEY=... LANGSMITH_API_KEY=... LANGCHAIN_TRACING_V2=true LANGSMITH_TRACING=true LANGSMITH_PROJECT=ghostfolio-ai-agent`
- `railway variable list -s ghostfolio-api --kv` confirms: `LANGCHAIN_API_KEY`, `LANGSMITH_API_KEY`, `LANGCHAIN_TRACING_V2`, `LANGSMITH_TRACING`, `LANGSMITH_PROJECT`
- LangChain wrapper + arithmetic direct reply verification (local, 2026-02-24):
- `npx jest apps/api/src/app/endpoints/ai/ai-agent.utils.spec.ts apps/api/src/app/endpoints/ai/ai.service.spec.ts apps/api/src/app/endpoints/ai/ai-observability.service.spec.ts --config apps/api/jest.config.ts` (36/36 tests passed)
- `npm run test:ai` (9/9 suites passed, 49/49 tests)
- `npx nx run api:lint --verbose` (passes with existing workspace warnings)

Loading…
Cancel
Save