From 82179076fb0a0a7834fa83ee5c07dafd87cb579e Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:45:05 +0200 Subject: [PATCH] Fix import of activities with Ghostfolio AI --- .../app/endpoints/ai/ai.controller.spec.ts | 33 +++++++++++++++++++ .../api/src/app/endpoints/ai/ai.controller.ts | 26 +++++++++++---- .../src/app/endpoints/ai/ai.helper.spec.ts | 22 +++++++++++-- apps/api/src/app/endpoints/ai/ai.helper.ts | 15 ++++++--- .../app/services/import-activities.service.ts | 6 ++-- 5 files changed, 86 insertions(+), 16 deletions(-) diff --git a/apps/api/src/app/endpoints/ai/ai.controller.spec.ts b/apps/api/src/app/endpoints/ai/ai.controller.spec.ts index 4ffe7fc558..2880c1aa85 100644 --- a/apps/api/src/app/endpoints/ai/ai.controller.spec.ts +++ b/apps/api/src/app/endpoints/ai/ai.controller.spec.ts @@ -4,6 +4,7 @@ import { CustomThrottlerGuard } from '@ghostfolio/api/guards/custom-throttler.gu import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'; import { ApiService } from '@ghostfolio/api/services/api/api.service'; import { + AI_ACTIVITIES_TIMEOUT, AI_MAX_ACTIVITIES, THROTTLE_AI_ACTIVITIES_LIMIT, THROTTLE_DAILY_TTL @@ -233,6 +234,38 @@ describe('AiController', () => { }); }); + it('Gives the status 400 with a message for the caller at the timeout', async () => { + const timeoutSignal = AbortSignal.abort( + new DOMException( + 'The operation was aborted due to timeout', + 'TimeoutError' + ) + ); + + jest.spyOn(AbortSignal, 'timeout').mockReturnValueOnce(timeoutSignal); + + jest + .mocked(aiService.getActivitiesFromFile) + .mockRejectedValueOnce(timeoutSignal.reason); + + await expect( + aiController.getActivitiesFromFile(createResponse(), pdfFile) + ).rejects.toMatchObject({ + response: { + message: [ + 'Ghostfolio AI needs too much time for the file. Split it into smaller files.' + ] + }, + status: StatusCodes.BAD_REQUEST + }); + + expect(AbortSignal.timeout).toHaveBeenCalledWith(AI_ACTIVITIES_TIMEOUT); + + expect(Logger.prototype.warn).toHaveBeenCalledWith( + `Stopped reading a file of the type application/pdf at the timeout of ${AI_ACTIVITIES_TIMEOUT} ms` + ); + }); + it('Passes on every other error', async () => { const error = new Error('Model error'); diff --git a/apps/api/src/app/endpoints/ai/ai.controller.ts b/apps/api/src/app/endpoints/ai/ai.controller.ts index 6db47d79d6..49d26b3427 100644 --- a/apps/api/src/app/endpoints/ai/ai.controller.ts +++ b/apps/api/src/app/endpoints/ai/ai.controller.ts @@ -69,8 +69,9 @@ export class AiController { * without saving anything. The client imports the activities which the user * selects with POST api/v1/import. The daily limit applies to each user * instead of each IP address. The models stop if the client closes the - * request before the response. An error whose message is written for the - * caller gives the status 400 with this message. + * request before the response or at the timeout. An error whose message is + * written for the caller gives the status 400 with this message, the + * timeout gives the status 400 with a message for the caller as well. */ @HasPermission(permissions.importActivitiesWithAi) @HttpCode(StatusCodes.OK) @@ -87,6 +88,7 @@ export class AiController { @UseGuards(AuthGuard('jwt'), HasPermissionGuard, CustomThrottlerGuard) @UseInterceptors( FileInterceptor('file', { + defParamCharset: 'utf8', limits: { fileSize: AI_ACTIVITIES_FILE_MAXIMUM_SIZE } }) ) @@ -132,6 +134,7 @@ export class AiController { } const abortController = new AbortController(); + const timeoutSignal = AbortSignal.timeout(AI_ACTIVITIES_TIMEOUT); response.on('close', () => { if (!response.writableFinished) { @@ -141,10 +144,7 @@ export class AiController { try { return await this.aiService.getActivitiesFromFile({ - abortSignal: AbortSignal.any([ - abortController.signal, - AbortSignal.timeout(AI_ACTIVITIES_TIMEOUT) - ]), + abortSignal: AbortSignal.any([abortController.signal, timeoutSignal]), file: { mediaType, buffer: file.buffer, name: file.originalname }, user: this.request.user }); @@ -157,6 +157,20 @@ export class AiController { }, StatusCodes.BAD_REQUEST ); + } else if (timeoutSignal.aborted) { + this.logger.warn( + `Stopped reading a file of the type ${mediaType} at the timeout of ${AI_ACTIVITIES_TIMEOUT} ms` + ); + + throw new HttpException( + { + error: getReasonPhrase(StatusCodes.BAD_REQUEST), + message: [ + 'Ghostfolio AI needs too much time for the file. Split it into smaller files.' + ] + }, + StatusCodes.BAD_REQUEST + ); } throw error; diff --git a/apps/api/src/app/endpoints/ai/ai.helper.spec.ts b/apps/api/src/app/endpoints/ai/ai.helper.spec.ts index 7bd004361f..f4b9d5707b 100644 --- a/apps/api/src/app/endpoints/ai/ai.helper.spec.ts +++ b/apps/api/src/app/endpoints/ai/ai.helper.spec.ts @@ -12,12 +12,28 @@ describe('getDataRowCountOfCsvFile', () => { ).toBe(2); }); - it('Counts a line with a line break in quotes as one line', () => { + it('Counts each line of a value with a line break in quotes', () => { expect( getDataRowCountOfCsvFile( - Buffer.from('Date,Comment\n2026-01-15,"First\nsecond"\n') + Buffer.from('Date,Comment\r\n2026-01-15,"First\r\nsecond"\r\n') ) - ).toBe(1); + ).toBe(2); + }); + + it('Counts each line after a quote which is not closed', () => { + expect( + getDataRowCountOfCsvFile( + Buffer.from( + [ + 'Date,Symbol,Quantity', + '2026-01-15,"AAPL,1', + ...Array.from({ length: 200 }, () => { + return '2026-01-16,MSFT,1'; + }) + ].join('\n') + ) + ) + ).toBe(201); }); it('Gives 0 for a file with a header only or without content', () => { diff --git a/apps/api/src/app/endpoints/ai/ai.helper.ts b/apps/api/src/app/endpoints/ai/ai.helper.ts index 8ca4fde61c..f229542aed 100644 --- a/apps/api/src/app/endpoints/ai/ai.helper.ts +++ b/apps/api/src/app/endpoints/ai/ai.helper.ts @@ -1,5 +1,4 @@ import { isUtf8 } from 'node:buffer'; -import { parse as parseCsv } from 'papaparse'; const JPEG_SIGNATURE = [0xff, 0xd8, 0xff]; const PDF_SIGNATURE = [0x25, 0x50, 0x44, 0x46, 0x2d]; @@ -7,13 +6,19 @@ const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; /** * Gives the number of the lines of a CSV file which are not empty, without - * the first line. A line before the header is counted as a data row, hence - * the number is never too low. + * the first line. A line before the header and each line of a value with a + * line break are counted as data rows, hence the number is never too low, + * also for a file with a quote which is not closed. */ export function getDataRowCountOfCsvFile(buffer: Buffer) { - const { data } = parseCsv(buffer.toString('utf8'), { skipEmptyLines: true }); + const lineCount = buffer + .toString('utf8') + .split(/\r\n|\r|\n/) + .filter((line) => { + return line.trim().length > 0; + }).length; - return Math.max(data.length - 1, 0); + return Math.max(lineCount - 1, 0); } /** diff --git a/apps/client/src/app/services/import-activities.service.ts b/apps/client/src/app/services/import-activities.service.ts index af21f6c1c2..06ca922c1f 100644 --- a/apps/client/src/app/services/import-activities.service.ts +++ b/apps/client/src/app/services/import-activities.service.ts @@ -46,9 +46,11 @@ export class ImportActivitiesService { * the first line */ public getDataRowCountOfCsv(fileContent: string) { - const { data } = csvToJson(fileContent, { skipEmptyLines: true }); + const lineCount = fileContent.split(/\r\n|\r|\n/).filter((line) => { + return line.trim().length > 0; + }).length; - return Math.max(data.length - 1, 0); + return Math.max(lineCount - 1, 0); } public async importCsv({