From 4cf70e2a468bba6fe0d862f064269a70ad36505c Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:51:10 +0200 Subject: [PATCH] Fix import of activities with Ghostfolio AI --- .../app/endpoints/ai/ai.controller.spec.ts | 14 +- .../api/src/app/endpoints/ai/ai.controller.ts | 6 +- .../src/app/endpoints/ai/ai.schemas.spec.ts | 54 +------ apps/api/src/app/endpoints/ai/ai.schemas.ts | 30 +--- .../src/app/endpoints/ai/ai.service.spec.ts | 123 +++++++++++++-- apps/api/src/app/endpoints/ai/ai.service.ts | 148 +++++++++++++++--- .../import-activities-dialog.component.ts | 62 ++++++-- .../import-activities-dialog.html | 10 ++ .../ai-activities-response.interface.ts | 6 +- 9 files changed, 321 insertions(+), 132 deletions(-) diff --git a/apps/api/src/app/endpoints/ai/ai.controller.spec.ts b/apps/api/src/app/endpoints/ai/ai.controller.spec.ts index 3e4fc7abe5..69a396e30d 100644 --- a/apps/api/src/app/endpoints/ai/ai.controller.spec.ts +++ b/apps/api/src/app/endpoints/ai/ai.controller.spec.ts @@ -73,9 +73,11 @@ describe('AiController', () => { jest.spyOn(Logger.prototype, 'warn').mockImplementation(); aiService = { - getActivitiesFromFile: jest - .fn() - .mockResolvedValue({ activities: [], rowsWithoutMatch: [] }) + getActivitiesFromFile: jest.fn().mockResolvedValue({ + activities: [], + assetProfiles: [], + rowsWithoutMatch: [] + }) } as unknown as AiService; aiController = new AiController( @@ -158,7 +160,11 @@ describe('AiController', () => { it('Passes the file with its media type and the user of the request', async () => { await expect( aiController.getActivitiesFromFile(createResponse(), pdfFile) - ).resolves.toEqual({ activities: [], rowsWithoutMatch: [] }); + ).resolves.toEqual({ + activities: [], + assetProfiles: [], + rowsWithoutMatch: [] + }); expect(aiService.getActivitiesFromFile).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/apps/api/src/app/endpoints/ai/ai.controller.ts b/apps/api/src/app/endpoints/ai/ai.controller.ts index e23aae4812..538f102793 100644 --- a/apps/api/src/app/endpoints/ai/ai.controller.ts +++ b/apps/api/src/app/endpoints/ai/ai.controller.ts @@ -66,14 +66,15 @@ export class AiController { /** * Reads the activities of a CSV, JPEG, PDF or PNG file with Ghostfolio AI, * without saving anything. The client imports the activities which the user - * selects with POST api/v1/import. + * selects with POST api/v1/import. The daily limit applies to each user + * instead of each IP address. The models stop if the client closes the + * request before the response. */ @HasPermission(permissions.importActivitiesWithAi) @HttpCode(StatusCodes.OK) @Post('activities') @Throttle({ default: { - // The limit applies to each user instead of each IP address getTracker: ({ user }: RequestWithUser) => { return user.id; }, @@ -128,7 +129,6 @@ export class AiController { } } - // Stops the models if the client closes the request before the response const abortController = new AbortController(); response.on('close', () => { diff --git a/apps/api/src/app/endpoints/ai/ai.schemas.spec.ts b/apps/api/src/app/endpoints/ai/ai.schemas.spec.ts index cec4104e64..fc01e0c55f 100644 --- a/apps/api/src/app/endpoints/ai/ai.schemas.spec.ts +++ b/apps/api/src/app/endpoints/ai/ai.schemas.spec.ts @@ -34,25 +34,6 @@ describe('ACTIVITY_ROWS_OUTPUT', () => { expect(ACTIVITY_ROWS_OUTPUT.safeParse({ rows: [row] }).success).toBe(false); }); - it('Refuses a currency which is not an ISO 4217 code in upper case', () => { - for (const currency of ['usd', 'USDOLLAR', '$']) { - expect( - ACTIVITY_ROWS_OUTPUT.safeParse({ - rows: [createActivityRow({ currency })] - }).success - ).toBe(false); - } - }); - - it('Refuses a date which is not an ISO 8601 date after 1970', () => { - for (const date of ['15.01.2026', '1969-12-31']) { - expect( - ACTIVITY_ROWS_OUTPUT.safeParse({ rows: [createActivityRow({ date })] }) - .success - ).toBe(false); - } - }); - it('Refuses a negative fee, quantity or unit price', () => { for (const row of [{ fee: -1 }, { quantity: -1 }, { unitPrice: -1 }]) { expect( @@ -62,24 +43,21 @@ describe('ACTIVITY_ROWS_OUTPUT', () => { } }); - it('Refuses a printed text which is longer than the maximum length', () => { - expect( - ACTIVITY_ROWS_OUTPUT.safeParse({ - rows: [ - createActivityRow({ name: 'A'.repeat(SEARCH_QUERY_MAXIMUM_LENGTH) }) - ] - }).success - ).toBe(true); - + it('Accepts a currency, a date, a printed text and a number of rows which the join checks, so that one value does not refuse all rows', () => { expect( ACTIVITY_ROWS_OUTPUT.safeParse({ rows: [ createActivityRow({ + currency: 'GBX', + date: '15.01.2026', name: 'A'.repeat(SEARCH_QUERY_MAXIMUM_LENGTH + 1) + }), + ...Array.from({ length: MCP_MAX_ACTIVITIES }, () => { + return createActivityRow(); }) ] }).success - ).toBe(false); + ).toBe(true); }); it('Trims a printed text', () => { @@ -89,24 +67,6 @@ describe('ACTIVITY_ROWS_OUTPUT', () => { expect(data.rows[0].name).toBe('Apple Inc.'); }); - - it(`Refuses more than ${MCP_MAX_ACTIVITIES} rows`, () => { - expect( - ACTIVITY_ROWS_OUTPUT.safeParse({ - rows: Array.from({ length: MCP_MAX_ACTIVITIES }, () => { - return createActivityRow(); - }) - }).success - ).toBe(true); - - expect( - ACTIVITY_ROWS_OUTPUT.safeParse({ - rows: Array.from({ length: MCP_MAX_ACTIVITIES + 1 }, () => { - return createActivityRow(); - }) - }).success - ).toBe(false); - }); }); describe.each([ diff --git a/apps/api/src/app/endpoints/ai/ai.schemas.ts b/apps/api/src/app/endpoints/ai/ai.schemas.ts index 2bec1637f8..24a20a4a12 100644 --- a/apps/api/src/app/endpoints/ai/ai.schemas.ts +++ b/apps/api/src/app/endpoints/ai/ai.schemas.ts @@ -1,29 +1,14 @@ -import { - MCP_MAX_ACTIVITIES, - SEARCH_QUERY_MAXIMUM_LENGTH -} from '@ghostfolio/common/config'; -import { - isValidCurrencyCode, - isValidDateAfter1970 -} from '@ghostfolio/common/helper'; +import { MCP_MAX_ACTIVITIES } from '@ghostfolio/common/config'; import { Type as ActivityType } from '@prisma/client'; import { z } from 'zod'; // The strict mode of the structured outputs requires every property, hence a // value which the file does not show is null instead of being left out. The -// maximum length is checked by a refinement, because the strict mode does not -// support every keyword of JSON Schema -const PRINTED_TEXT = z - .string() - .trim() - .refine( - (value) => { - return value.length <= SEARCH_QUERY_MAXIMUM_LENGTH; - }, - { error: `Too long: expected at most ${SEARCH_QUERY_MAXIMUM_LENGTH}` } - ) - .nullable(); +// values are checked when the rows and the matches are joined to the +// activities to import, so that a wrong value or a wrong match skips its rows +// only +const PRINTED_TEXT = z.string().trim().nullable(); export const ACTIVITY_ROWS_OUTPUT = z.object({ rows: z @@ -34,14 +19,12 @@ export const ACTIVITY_ROWS_OUTPUT = z.object({ ), currency: z .string() - .refine(isValidCurrencyCode) .nullable() .describe( 'The currency of the fee and of the unit price, as an ISO 4217 code in upper case' ), date: z .string() - .refine(isValidDateAfter1970) .describe('The date of the activity, as an ISO 8601 date'), fee: z.number().min(0).describe('The fee of the activity'), isin: PRINTED_TEXT.describe('The ISIN of the asset, as printed'), @@ -56,12 +39,9 @@ export const ACTIVITY_ROWS_OUTPUT = z.object({ unitPrice: z.number().min(0).describe('The unit price of the activity') }) ) - .max(MCP_MAX_ACTIVITIES) .describe(`The activities of the file, at most ${MCP_MAX_ACTIVITIES}`) }); -// The values are checked when the rows and the matches are joined to the -// activities to import, so that a wrong match skips its rows only export const ROW_MATCHES_OUTPUT = z.object({ accounts: z .array( diff --git a/apps/api/src/app/endpoints/ai/ai.service.spec.ts b/apps/api/src/app/endpoints/ai/ai.service.spec.ts index 747d6ec3b2..a7f99c068e 100644 --- a/apps/api/src/app/endpoints/ai/ai.service.spec.ts +++ b/apps/api/src/app/endpoints/ai/ai.service.spec.ts @@ -9,11 +9,12 @@ import { PROPERTY_OPENROUTER_MODEL_AGENT, PROPERTY_OPENROUTER_MODEL_VISION } from '@ghostfolio/common/config'; +import { isValidCustomAssetProfileSymbol } from '@ghostfolio/common/helper'; import type { UserWithSettings } from '@ghostfolio/common/types'; import { Logger } from '@nestjs/common'; import { createOpenRouter } from '@openrouter/ai-sdk-provider'; -import { Type as ActivityType } from '@prisma/client'; +import { DataSource, Type as ActivityType } from '@prisma/client'; import { generateText, isStepCount, ToolSet } from 'ai'; import { z } from 'zod'; @@ -157,7 +158,8 @@ describe('AiService', () => { expect(chat).toHaveBeenCalledWith('vision-model', { plugins: [{ id: 'file-parser', pdf: { engine: 'native' } }], - provider: { zdr: true } + provider: { zdr: true }, + usage: { include: true } }); }); @@ -228,7 +230,11 @@ describe('AiService', () => { await expect( aiService.getActivitiesFromFile({ abortSignal, file, user }) - ).resolves.toEqual({ activities, rowsWithoutMatch: [] }); + ).resolves.toEqual({ + activities, + assetProfiles: [], + rowsWithoutMatch: [] + }); expect(aiService.getActivityRowsFromFile).toHaveBeenCalledWith({ abortSignal, @@ -245,6 +251,46 @@ describe('AiService', () => { }); }); + it('Gives a fee, an interest or a liability a custom asset profile with its description as name', async () => { + const fee = { + currency: 'CHF', + date: '2026-01-15', + fee: 5, + quantity: 0, + symbol: 'Custody fee', + type: ActivityType.FEE, + unitPrice: 0 + }; + + jest.spyOn(aiService, 'getActivityRowsFromFile').mockResolvedValue(rows); + + jest.spyOn(aiService, 'getActivitiesFromRows').mockResolvedValue({ + activities: [fee, { ...fee, date: '2026-02-15' }], + rowsWithoutMatch: [] + }); + + const { activities, assetProfiles } = + await aiService.getActivitiesFromFile({ file, user }); + + const [{ symbol }] = assetProfiles; + + expect(isValidCustomAssetProfileSymbol(symbol)).toBe(true); + + expect(assetProfiles).toEqual([ + { + symbol, + currency: 'CHF', + dataSource: DataSource.MANUAL, + name: 'Custody fee' + } + ]); + + expect(activities).toEqual([ + { ...fee, symbol, dataSource: DataSource.MANUAL }, + { ...fee, symbol, dataSource: DataSource.MANUAL, date: '2026-02-15' } + ]); + }); + it('Reads the file in the default locale if the user has no locale', async () => { jest.spyOn(aiService, 'getActivityRowsFromFile').mockResolvedValue([]); @@ -276,13 +322,34 @@ describe('AiService', () => { function mockRowMatches({ accounts: accountMatches = [], + assetProfilesOfSearch, assets: assetMatches = [] - }: Partial>) { - jest.mocked(generateText).mockResolvedValueOnce({ - output: { accounts: accountMatches, assets: assetMatches }, - steps: [{}, {}], - totalUsage: { totalTokens: 2000 } - } as unknown as Awaited>); + }: Partial> & { + assetProfilesOfSearch?: { dataSource: string; symbol: string }[]; + }) { + jest.mocked(mcpService.searchAssetProfiles).mockResolvedValueOnce({ + content: [ + { + text: JSON.stringify({ + assetProfiles: assetProfilesOfSearch ?? assetMatches + }), + type: 'text' + } + ] + }); + + jest.mocked(generateText).mockImplementationOnce(async ({ tools }) => { + await tools['search-asset-profiles'].execute( + { query: 'US0378331005' }, + undefined + ); + + return { + output: { accounts: accountMatches, assets: assetMatches }, + steps: [{}, {}], + totalUsage: { totalTokens: 2000 } + } as unknown as Awaited>; + }); } it('Gives an activity for each row whose asset matches an asset profile', async () => { @@ -373,6 +440,26 @@ describe('AiService', () => { ).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); }); + it('Gives a row whose asset profile no search gave as a row without a match', async () => { + const row = createActivityRow({ accountName: null }); + + mockRowMatches({ + assetProfilesOfSearch: [{ dataSource: 'YAHOO', symbol: 'AAPL' }], + assets: [ + { currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL.XX' } + ] + }); + + await expect( + aiService.getActivitiesFromRows({ + accounts, + userCurrency, + userId, + rows: [row] + }) + ).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); + }); + it('Ignores an account id which is not the id of an account of the user', async () => { mockRowMatches({ accounts: [{ accountId: 'unknown-account-id', name: 'Brokerage' }], @@ -547,7 +634,7 @@ describe('AiService', () => { }); }); - it('Restricts the routing to endpoints with zero data retention and limits the number of steps', async () => { + it('Restricts the routing to endpoints with zero data retention, limits the number of steps and allows no tool call in the last step', async () => { mockRowMatches({}); await aiService.getActivitiesFromRows({ @@ -559,7 +646,8 @@ describe('AiService', () => { }); expect(chat).toHaveBeenCalledWith('agent-model', { - provider: { zdr: true } + provider: { zdr: true }, + usage: { include: true } }); expect(isStepCount).toHaveBeenCalled(); @@ -572,9 +660,20 @@ describe('AiService', () => { }) ); - const { stopWhen } = jest.mocked(generateText).mock.calls[0][0]; + const { prepareStep, stopWhen } = jest.mocked(generateText).mock + .calls[0][0] as { + prepareStep: (options: { stepNumber: number }) => unknown; + stopWhen: unknown; + }; expect(stopWhen).toBe(jest.mocked(isStepCount).mock.results[0].value); + + const [stepCount] = jest.mocked(isStepCount).mock.calls[0]; + + expect(prepareStep({ stepNumber: stepCount - 2 })).toBeUndefined(); + expect(prepareStep({ stepNumber: stepCount - 1 })).toEqual({ + toolChoice: 'none' + }); }); it('Falls back to the general model if no agent model is set', async () => { diff --git a/apps/api/src/app/endpoints/ai/ai.service.ts b/apps/api/src/app/endpoints/ai/ai.service.ts index 8a2f4219f1..dccaa5baf4 100644 --- a/apps/api/src/app/endpoints/ai/ai.service.ts +++ b/apps/api/src/app/endpoints/ai/ai.service.ts @@ -16,8 +16,16 @@ import { PROPERTY_OPENROUTER_MODEL_AGENT, PROPERTY_OPENROUTER_MODEL_VISION } from '@ghostfolio/common/config'; -import { DATE_FORMAT } from '@ghostfolio/common/helper'; -import { AiActivitiesResponse, Filter } from '@ghostfolio/common/interfaces'; +import { CreateAssetProfileWithMarketDataDto } from '@ghostfolio/common/dtos'; +import { + DATE_FORMAT, + getAssetProfileIdentifier +} from '@ghostfolio/common/helper'; +import { + AiActivitiesResponse, + AssetProfileIdentifier, + Filter +} from '@ghostfolio/common/interfaces'; import type { AiPromptMode, PropertyKey, @@ -27,7 +35,8 @@ import type { import { Injectable, Logger } from '@nestjs/common'; import { createOpenRouter, - OpenRouterChatSettings + OpenRouterChatSettings, + OpenRouterUsageAccounting } from '@openrouter/ai-sdk-provider'; import { DataSource } from '@prisma/client'; import { @@ -35,11 +44,14 @@ import { generateText, isStepCount, Output, + ProviderMetadata, TextPart, tool } from 'ai'; +import { Big } from 'big.js'; import { format } from 'date-fns'; import { keyBy, uniq } from 'lodash-es'; +import { randomUUID } from 'node:crypto'; import { z } from 'zod'; import { ACTIVITY_ROWS_OUTPUT, ROW_MATCHES_OUTPUT } from './ai.schemas'; @@ -50,6 +62,7 @@ import { AssetToMatch } from './types/asset-to-match.type'; @Injectable() export class AiService { private static readonly AGENT_MAXIMUM_STEP_COUNT = 10; + private static readonly VISION_MAXIMUM_OUTPUT_TOKEN_COUNT = 16384; private readonly logger = new Logger(AiService.name); @@ -114,12 +127,8 @@ export class AiService { }); return { - rowsWithoutMatch, - activities: activities.map(({ dataSource, ...activity }) => { - // The data source is given as search-asset-profiles gives it, which - // POST api/v1/import transforms back - return { ...activity, dataSource: dataSource as DataSource }; - }) + ...this.getActivitiesWithCustomAssetProfiles(activities), + rowsWithoutMatch }; } @@ -127,7 +136,8 @@ export class AiService { * Matches the rows of a file to the asset profiles with the agent model and * the tool search-asset-profiles, and to the accounts of the user. A row * which does not give a valid activity, for example because its asset has - * no match, is a row without a match. + * no match, is a row without a match. An account id which is not the id of + * an account of the user is ignored. */ public async getActivitiesFromRows({ abortSignal, @@ -203,7 +213,6 @@ export class AiService { unitPrice } = row; - // An id which is not the id of an account of the user is ignored const account = accounts.find(({ id }) => { return id === accountIdByName.get(accountName); }); @@ -251,9 +260,12 @@ export class AiService { /** * Reads the activities of a file (CSV, JPEG, PNG or PDF) with the vision - * model. The model gets no tools and copies the asset and the account as - * printed, hence the rows still have to be matched to asset profiles and - * accounts. + * model. A CSV file is passed as text, every other file as a file part. The + * native engine passes a PDF file to the model itself, as a parser plugin of + * OpenRouter is not covered by the zero data retention. The model gets no + * tools and copies the asset and the account as printed, hence the rows + * still have to be matched to asset profiles and accounts. A row above the + * maximum number of activities is left out. */ public async getActivityRowsFromFile({ abortSignal, @@ -266,13 +278,12 @@ export class AiService { }): Promise { const startTime = performance.now(); - // A CSV file is passed as text, every other file as a file part const content: (FilePart | TextPart)[] = file.mediaType === 'text/csv' ? [{ text: file.buffer.toString('utf8'), type: 'text' }] : [{ data: file.buffer, mediaType: file.mediaType, type: 'file' }]; - const { output, totalUsage } = await generateText({ + const { output, steps, totalUsage } = await generateText({ abortSignal, instructions: [ 'You read the activities of a portfolio from a file, for example a CSV export or a statement of a broker, a confirmation of a trade or a screenshot.', @@ -288,24 +299,24 @@ export class AiService { `If the format of a date or of a number is ambiguous and the file gives no other hint, read it as in the locale ${locale}.`, `Give at most ${MCP_MAX_ACTIVITIES} rows.` ].join('\n'), + maxOutputTokens: AiService.VISION_MAXIMUM_OUTPUT_TOKEN_COUNT, messages: [{ content, role: 'user' }], model: await this.getOpenRouterChatModel({ modelPropertyKey: PROPERTY_OPENROUTER_MODEL_VISION, settings: { - // The native engine passes a PDF file to the model itself. A parser - // plugin of OpenRouter is not covered by the zero data retention. plugins: [{ id: 'file-parser', pdf: { engine: 'native' } }], - provider: { zdr: true } + provider: { zdr: true }, + usage: { include: true } } }), output: Output.object({ schema: ACTIVITY_ROWS_OUTPUT }) }); this.logger.log( - `Read ${output.rows.length} rows from a file in ${this.getSecondsSince(startTime)} seconds with ${totalUsage.totalTokens} tokens` + `Read ${output.rows.length} rows from a file in ${this.getSecondsSince(startTime)} seconds with ${totalUsage.totalTokens} tokens for ${this.getCost(steps)} USD` ); - return output.rows; + return output.rows.slice(0, MCP_MAX_ACTIVITIES); } public async getPrompt({ @@ -346,10 +357,67 @@ export class AiService { ].join('\n'); } + /** + * Gives each fee, interest and liability a custom asset profile with its + * description as name, as the import after the dry run requires a valid + * symbol for the data source MANUAL. The data source of every other activity + * is given as search-asset-profiles gives it, which POST api/v1/import + * transforms back. + */ + private getActivitiesWithCustomAssetProfiles( + activities: ActivityToImport[] + ): Omit { + const assetProfiles: CreateAssetProfileWithMarketDataDto[] = []; + + const activitiesWithCustomAssetProfiles = activities.map((activity) => { + if (activity.dataSource) { + return { ...activity, dataSource: activity.dataSource as DataSource }; + } + + let assetProfile = assetProfiles.find(({ name }) => { + return name === activity.symbol; + }); + + if (!assetProfile) { + assetProfile = { + currency: activity.currency, + dataSource: DataSource.MANUAL, + name: activity.symbol, + symbol: randomUUID() + }; + + assetProfiles.push(assetProfile); + } + + return { + ...activity, + dataSource: DataSource.MANUAL, + symbol: assetProfile.symbol + }; + }); + + return { assetProfiles, activities: activitiesWithCustomAssetProfiles }; + } + private getAssetKey({ currency, isin, name, symbol }: ActivityRow) { return JSON.stringify([currency, isin, name, symbol]); } + /** + * Gives the cost in USD of the steps, as OpenRouter gives it + */ + private getCost(steps: { providerMetadata: ProviderMetadata | undefined }[]) { + return steps + .reduce((cost, { providerMetadata }) => { + const { usage } = (providerMetadata?.openrouter ?? {}) as { + usage?: OpenRouterUsageAccounting; + }; + + return cost.plus(usage?.cost ?? 0); + }, new Big(0)) + .toNumber(); + } + /** * Gives the chat model of the property, with the general model as fallback */ @@ -377,6 +445,12 @@ export class AiService { ); } + /** + * Matches the assets and the account names with the agent model. The last + * step allows no tool call, so that the model always gives the matches. An + * asset match whose asset profile no search gave is left out, as the import + * refuses a symbol which the data source cannot resolve. + */ private async getRowMatches({ abortSignal, accountNames, @@ -390,6 +464,7 @@ export class AiService { assets: AssetToMatch[]; userId: string; }) { + const assetProfileIdentifiersOfSearches = new Set(); const startTime = performance.now(); const { output, steps, totalUsage } = await generateText({ @@ -404,9 +479,14 @@ export class AiService { ].join('\n'), model: await this.getOpenRouterChatModel({ modelPropertyKey: PROPERTY_OPENROUTER_MODEL_AGENT, - settings: { provider: { zdr: true } } + settings: { provider: { zdr: true }, usage: { include: true } } }), output: Output.object({ schema: ROW_MATCHES_OUTPUT }), + prepareStep: ({ stepNumber }) => { + return stepNumber === AiService.AGENT_MAXIMUM_STEP_COUNT - 1 + ? { toolChoice: 'none' } + : undefined; + }, prompt: JSON.stringify({ accountNames, assets, @@ -427,6 +507,16 @@ export class AiService { userId }); + const { assetProfiles } = JSON.parse(content[0].text) as { + assetProfiles: AssetProfileIdentifier[]; + }; + + for (const assetProfile of assetProfiles) { + assetProfileIdentifiersOfSearches.add( + getAssetProfileIdentifier(assetProfile) + ); + } + return content[0].text; }, inputSchema: SEARCH_ASSET_PROFILES_PARAMETERS @@ -435,10 +525,20 @@ export class AiService { }); this.logger.log( - `Matched ${assets.length} assets and ${accountNames.length} account names in ${steps.length} steps and ${this.getSecondsSince(startTime)} seconds with ${totalUsage.totalTokens} tokens` + `Matched ${assets.length} assets and ${accountNames.length} account names in ${steps.length} steps and ${this.getSecondsSince(startTime)} seconds with ${totalUsage.totalTokens} tokens for ${this.getCost(steps)} USD` ); - return output; + return { + accounts: output.accounts, + assets: output.assets.filter(({ dataSource, symbol }) => { + return assetProfileIdentifiersOfSearches.has( + getAssetProfileIdentifier({ + symbol, + dataSource: dataSource as DataSource + }) + ); + }) + }; } private getSecondsSince(startTime: number) { diff --git a/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts b/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts index 11aebcd19d..bcef62f67d 100644 --- a/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts +++ b/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts @@ -1,6 +1,7 @@ import { GfFileDropDirective } from '@ghostfolio/client/directives/file-drop/file-drop.directive'; import { ImportActivitiesService } from '@ghostfolio/client/services/import-activities.service'; import { + AI_ACTIVITIES_FILE_MAXIMUM_SIZE, DEFAULT_DATE_RANGE, MCP_MAX_ACTIVITIES } from '@ghostfolio/common/config'; @@ -133,6 +134,7 @@ export class GfImportActivitiesDialogComponent { protected importStep: ImportStep = ImportStep.UPLOAD_FILE; protected isGhostfolioAiEnabled = false; protected isLoading = false; + protected isMaxActivitiesOfGhostfolioAiReached = false; protected readonly maxActivitiesOfGhostfolioAi = MCP_MAX_ACTIVITIES; protected mode: 'DIVIDEND'; protected pageIndex = 0; @@ -308,6 +310,7 @@ export class GfImportActivitiesDialogComponent { this.details = []; this.errorMessages = []; this.importStep = ImportStep.SELECT_ACTIVITIES; + this.isMaxActivitiesOfGhostfolioAiReached = false; this.pageIndex = 0; this.rowsWithoutMatch = []; this.assetProfileForm.controls.assetProfileIdentifier.enable(); @@ -348,6 +351,32 @@ export class GfImportActivitiesDialogComponent { }); } + /** + * Gives the error message of a file which the server rejects, as a rejected + * file counts towards the daily limit of Ghostfolio AI + */ + private async getErrorMessageOfFileForGhostfolioAi({ + file, + fileExtension + }: { + file: File; + fileExtension?: string; + }) { + if (file.size > AI_ACTIVITIES_FILE_MAXIMUM_SIZE) { + return $localize`The file is larger than ${AI_ACTIVITIES_FILE_MAXIMUM_SIZE / 1024 / 1024}:maxFileSize: MB`; + } + + if ( + fileExtension === 'csv' && + this.importActivitiesService.getDataRowCountOfCsv(await file.text()) > + MCP_MAX_ACTIVITIES + ) { + return $localize`The CSV file has more than ${MCP_MAX_ACTIVITIES}:maxActivities: rows of activities`; + } + + return undefined; + } + private handleFile({ file, stepper }: { file: File; stepper: MatStepper }) { const fileExtension = file.name.split('.').pop()?.toLowerCase(); @@ -468,6 +497,10 @@ export class GfImportActivitiesDialogComponent { }; } + /** + * Reads the file with Ghostfolio AI. The request is canceled if the dialog + * is closed, which stops the models on the server. + */ private async handleFileWithGhostfolioAi({ file, fileExtension, @@ -480,25 +513,19 @@ export class GfImportActivitiesDialogComponent { this.accounts = []; this.activities = []; this.assetProfiles = []; + this.isMaxActivitiesOfGhostfolioAiReached = false; this.platforms = []; this.tags = []; - // A CSV file is checked before the upload, because a file which the - // server rejects counts towards the daily limit of Ghostfolio AI - if ( - fileExtension === 'csv' && - this.importActivitiesService.getDataRowCountOfCsv(await file.text()) > - MCP_MAX_ACTIVITIES - ) { + const errorMessage = await this.getErrorMessageOfFileForGhostfolioAi({ + file, + fileExtension + }); + + if (errorMessage) { this.handleImportError({ activities: [], - error: { - error: { - message: [ - $localize`The CSV file has more than ${MCP_MAX_ACTIVITIES}:maxActivities: rows of activities` - ] - } - } + error: { error: { message: [errorMessage] } } }); this.showSelectActivitiesStep(stepper); @@ -511,17 +538,20 @@ export class GfImportActivitiesDialogComponent { let activitiesOfFile: CreateOrderDto[] = []; - // The request is canceled if the dialog is closed this.dataService .postAiActivities(file) .pipe( - switchMap(({ activities, rowsWithoutMatch }) => { + switchMap(({ activities, assetProfiles, rowsWithoutMatch }) => { activitiesOfFile = activities; + this.assetProfiles = assetProfiles; + this.isMaxActivitiesOfGhostfolioAiReached = + activities.length + rowsWithoutMatch.length >= MCP_MAX_ACTIVITIES; this.rowsWithoutMatch = rowsWithoutMatch; return from( this.importActivitiesService.importJson({ activities, + assetProfiles, isDryRun: true }) ); diff --git a/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.html b/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.html index 7eb0550162..a068f2b257 100644 --- a/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.html +++ b/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.html @@ -185,6 +185,16 @@ } + @if (isMaxActivitiesOfGhostfolioAiReached) { +
+ + Ghostfolio AI has read the maximum of + {{ maxActivitiesOfGhostfolioAi }} activities. The file can + have more activities. +
+ }