Fix import of activities with Ghostfolio AI

apps/api/src/app/endpoints/ai/ai.controller.spec.ts

@@ -4,6 +4,7 @@ import { CustomThrottlerGuard } from '@ghostfolio/api/guards/custom-throttler.gu
 import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard';
 import { ApiService } from '@ghostfolio/api/services/api/api.service';
 import {
+  AI_ACTIVITIES_TIMEOUT,
   AI_MAX_ACTIVITIES,
   THROTTLE_AI_ACTIVITIES_LIMIT,
   THROTTLE_DAILY_TTL
@@ -233,6 +234,38 @@ describe('AiController', () => {
       });
     });
 
+    it('Gives the status 400 with a message for the caller at the timeout', async () => {
+      const timeoutSignal = AbortSignal.abort(
+        new DOMException(
+          'The operation was aborted due to timeout',
+          'TimeoutError'
+        )
+      );
+
+      jest.spyOn(AbortSignal, 'timeout').mockReturnValueOnce(timeoutSignal);
+
+      jest
+        .mocked(aiService.getActivitiesFromFile)
+        .mockRejectedValueOnce(timeoutSignal.reason);
+
+      await expect(
+        aiController.getActivitiesFromFile(createResponse(), pdfFile)
+      ).rejects.toMatchObject({
+        response: {
+          message: [
+            'Ghostfolio AI needs too much time for the file. Split it into smaller files.'
+          ]
+        },
+        status: StatusCodes.BAD_REQUEST
+      });
+
+      expect(AbortSignal.timeout).toHaveBeenCalledWith(AI_ACTIVITIES_TIMEOUT);
+
+      expect(Logger.prototype.warn).toHaveBeenCalledWith(
+        `Stopped reading a file of the type application/pdf at the timeout of ${AI_ACTIVITIES_TIMEOUT} ms`
+      );
+    });
+
     it('Passes on every other error', async () => {
       const error = new Error('Model error');
 

apps/api/src/app/endpoints/ai/ai.controller.ts

@@ -69,8 +69,9 @@ export class AiController {
    * without saving anything. The client imports the activities which the user
    * selects with POST api/v1/import. The daily limit applies to each user
    * instead of each IP address. The models stop if the client closes the
-   * request before the response. An error whose message is written for the
-   * caller gives the status 400 with this message.
+   * request before the response or at the timeout. An error whose message is
+   * written for the caller gives the status 400 with this message, the
+   * timeout gives the status 400 with a message for the caller as well.
    */
   @HasPermission(permissions.importActivitiesWithAi)
   @HttpCode(StatusCodes.OK)
@@ -87,6 +88,7 @@ export class AiController {
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard, CustomThrottlerGuard)
   @UseInterceptors(
     FileInterceptor('file', {
+      defParamCharset: 'utf8',
       limits: { fileSize: AI_ACTIVITIES_FILE_MAXIMUM_SIZE }
     })
   )
@@ -132,6 +134,7 @@ export class AiController {
     }
 
     const abortController = new AbortController();
+    const timeoutSignal = AbortSignal.timeout(AI_ACTIVITIES_TIMEOUT);
 
     response.on('close', () => {
       if (!response.writableFinished) {
@@ -141,10 +144,7 @@ export class AiController {
 
     try {
       return await this.aiService.getActivitiesFromFile({
-        abortSignal: AbortSignal.any([
-          abortController.signal,
-          AbortSignal.timeout(AI_ACTIVITIES_TIMEOUT)
-        ]),
+        abortSignal: AbortSignal.any([abortController.signal, timeoutSignal]),
         file: { mediaType, buffer: file.buffer, name: file.originalname },
         user: this.request.user
       });
@@ -157,6 +157,20 @@ export class AiController {
           },
           StatusCodes.BAD_REQUEST
         );
+      } else if (timeoutSignal.aborted) {
+        this.logger.warn(
+          `Stopped reading a file of the type ${mediaType} at the timeout of ${AI_ACTIVITIES_TIMEOUT} ms`
+        );
+
+        throw new HttpException(
+          {
+            error: getReasonPhrase(StatusCodes.BAD_REQUEST),
+            message: [
+              'Ghostfolio AI needs too much time for the file. Split it into smaller files.'
+            ]
+          },
+          StatusCodes.BAD_REQUEST
+        );
       }
 
       throw error;

apps/api/src/app/endpoints/ai/ai.helper.spec.ts

@@ -12,12 +12,28 @@ describe('getDataRowCountOfCsvFile', () => {
     ).toBe(2);
   });
 
-  it('Counts a line with a line break in quotes as one line', () => {
+  it('Counts each line of a value with a line break in quotes', () => {
     expect(
       getDataRowCountOfCsvFile(
-        Buffer.from('Date,Comment\n2026-01-15,"First\nsecond"\n')
+        Buffer.from('Date,Comment\r\n2026-01-15,"First\r\nsecond"\r\n')
       )
-    ).toBe(1);
+    ).toBe(2);
+  });
+
+  it('Counts each line after a quote which is not closed', () => {
+    expect(
+      getDataRowCountOfCsvFile(
+        Buffer.from(
+          [
+            'Date,Symbol,Quantity',
+            '2026-01-15,"AAPL,1',
+            ...Array.from({ length: 200 }, () => {
+              return '2026-01-16,MSFT,1';
+            })
+          ].join('\n')
+        )
+      )
+    ).toBe(201);
   });
 
   it('Gives 0 for a file with a header only or without content', () => {

apps/api/src/app/endpoints/ai/ai.helper.ts

@@ -1,5 +1,4 @@
 import { isUtf8 } from 'node:buffer';
-import { parse as parseCsv } from 'papaparse';
 
 const JPEG_SIGNATURE = [0xff, 0xd8, 0xff];
 const PDF_SIGNATURE = [0x25, 0x50, 0x44, 0x46, 0x2d];
@@ -7,13 +6,19 @@ const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
 
 /**
  * Gives the number of the lines of a CSV file which are not empty, without
- * the first line. A line before the header is counted as a data row, hence
- * the number is never too low.
+ * the first line. A line before the header and each line of a value with a
+ * line break are counted as data rows, hence the number is never too low,
+ * also for a file with a quote which is not closed.
  */
 export function getDataRowCountOfCsvFile(buffer: Buffer) {
-  const { data } = parseCsv(buffer.toString('utf8'), { skipEmptyLines: true });
+  const lineCount = buffer
+    .toString('utf8')
+    .split(/\r\n|\r|\n/)
+    .filter((line) => {
+      return line.trim().length > 0;
+    }).length;
 
-  return Math.max(data.length - 1, 0);
+  return Math.max(lineCount - 1, 0);
 }
 
 /**

apps/client/src/app/services/import-activities.service.ts

@@ -46,9 +46,11 @@ export class ImportActivitiesService {
    * the first line
    */
   public getDataRowCountOfCsv(fileContent: string) {
-    const { data } = csvToJson(fileContent, { skipEmptyLines: true });
+    const lineCount = fileContent.split(/\r\n|\r|\n/).filter((line) => {
+      return line.trim().length > 0;
+    }).length;
 
-    return Math.max(data.length - 1, 0);
+    return Math.max(lineCount - 1, 0);
   }
 
   public async importCsv({