Bugfix/invalid output of Ghostfolio AI (#8299)

* Request output of Ghostfolio AI one more time if it does not agree with schema

* Update changelog

CHANGELOG.md

@@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Improved the style of the menu button in the user detail dialog of the admin control panel
 - Improved the consistency of the import of activities with _Ghostfolio AI_ (experimental)
 
+### Fixed
+
+- Fixed the handling of an output of the model which does not agree with the schema in the import of activities with _Ghostfolio AI_ (experimental)
+
 ## 3.92.0 - 2026-10-24
 
 ### Changed

apps/api/src/app/endpoints/ai/ai.service.spec.ts

@@ -37,11 +37,23 @@ jest.mock('ai', () => {
     generateText: jest.fn(),
     NoObjectGeneratedError: class MockNoObjectGeneratedError extends Error {
       public readonly finishReason: string;
-
-      public constructor({ finishReason }: { finishReason: string }) {
+      public readonly response?: { modelId: string };
+      public readonly usage?: { totalTokens: number };
+
+      public constructor({
+        finishReason,
+        response,
+        usage
+      }: {
+        finishReason: string;
+        response?: { modelId: string };
+        usage?: { totalTokens: number };
+      }) {
         super('No object generated');
 
         this.finishReason = finishReason;
+        this.response = response;
+        this.usage = usage;
       }
 
       public static isInstance(error: unknown) {
@@ -303,6 +315,8 @@ describe('AiService', () => {
         })
       ).rejects.toThrow(CallerFacingError);
 
+      expect(generateText).toHaveBeenCalledTimes(1);
+
       expect(Logger.prototype.warn).toHaveBeenCalled();
     });
 
@@ -329,11 +343,127 @@ describe('AiService', () => {
       expect(Logger.prototype.warn).toHaveBeenCalled();
     });
 
-    it('Passes on every other error of the vision model', async () => {
-      const error = new NoObjectGeneratedError({
+    it('Requests the rows one more time if the output does not agree with the schema', async () => {
+      jest.spyOn(Logger.prototype, 'warn').mockImplementation();
+
+      jest.mocked(generateText).mockRejectedValueOnce(
+        new NoObjectGeneratedError({
+          finishReason: 'stop',
+          response: { modelId: 'vision-model' },
+          usage: { totalTokens: 1000 }
+        } as ConstructorParameters<typeof NoObjectGeneratedError>[0])
+      );
+
+      await expect(
+        aiService.getActivityRowsFromFile({
+          abortSignal,
+          file: {
+            buffer: Buffer.from('%PDF-1.7'),
+            mediaType: 'application/pdf'
+          },
+          locale: 'de-CH'
+        })
+      ).resolves.toEqual({ broker: 'Swissquote', rows });
+
+      expect(generateText).toHaveBeenCalledTimes(2);
+
+      const [[firstOptions], [secondOptions]] =
+        jest.mocked(generateText).mock.calls;
+
+      expect(secondOptions).toEqual(firstOptions);
+      expect(secondOptions.abortSignal).toBe(abortSignal);
+
+      expect(Logger.prototype.warn).toHaveBeenCalledWith(
+        expect.stringContaining(
+          'the model vision-model one more time after 1000 tokens'
+        )
+      );
+    });
+
+    it('Requests the rows one more time if the output is empty', async () => {
+      jest.spyOn(Logger.prototype, 'warn').mockImplementation();
+
+      jest.mocked(generateText).mockResolvedValueOnce({
+        finalStep: { response: { modelId: 'vision-model' } },
+        finishReason: 'other',
+        text: '',
+        usage: { totalTokens: 10 },
+        get output() {
+          throw new Error('No output generated');
+        }
+      } as unknown as Awaited<ReturnType<typeof generateText>>);
+
+      await expect(
+        aiService.getActivityRowsFromFile({
+          file: {
+            buffer: Buffer.from('%PDF-1.7'),
+            mediaType: 'application/pdf'
+          },
+          locale: 'de-CH'
+        })
+      ).resolves.toEqual({ broker: 'Swissquote', rows });
+
+      expect(generateText).toHaveBeenCalledTimes(2);
+
+      expect(Logger.prototype.warn).toHaveBeenCalledWith(
+        expect.stringContaining(
+          'the model vision-model one more time after 10 tokens'
+        )
+      );
+    });
+
+    it('Passes on the error if the output does not agree with the schema a second time', async () => {
+      jest.spyOn(Logger.prototype, 'warn').mockImplementation();
+
+      const firstError = new NoObjectGeneratedError({
+        finishReason: 'stop'
+      } as ConstructorParameters<typeof NoObjectGeneratedError>[0]);
+
+      const secondError = new NoObjectGeneratedError({
         finishReason: 'stop'
       } as ConstructorParameters<typeof NoObjectGeneratedError>[0]);
 
+      jest
+        .mocked(generateText)
+        .mockRejectedValueOnce(firstError)
+        .mockRejectedValueOnce(secondError);
+
+      await expect(
+        aiService.getActivityRowsFromFile({
+          file: {
+            buffer: Buffer.from('%PDF-1.7'),
+            mediaType: 'application/pdf'
+          },
+          locale: 'de-CH'
+        })
+      ).rejects.toBe(secondError);
+
+      expect(generateText).toHaveBeenCalledTimes(2);
+    });
+
+    it('Passes on the error if the content filter stops the output', async () => {
+      const error = new NoObjectGeneratedError({
+        finishReason: 'content-filter'
+      } as ConstructorParameters<typeof NoObjectGeneratedError>[0]);
+
+      jest.mocked(generateText).mockRejectedValueOnce(error);
+
+      await expect(
+        aiService.getActivityRowsFromFile({
+          file: {
+            buffer: Buffer.from('%PDF-1.7'),
+            mediaType: 'application/pdf'
+          },
+          locale: 'de-CH'
+        })
+      ).rejects.toBe(error);
+
+      expect(generateText).toHaveBeenCalledTimes(1);
+    });
+
+    it('Passes on every other error of the vision model', async () => {
+      const error = new Error('Provider failed');
+
       jest.mocked(generateText).mockRejectedValueOnce(error);
 
       await expect(
@@ -345,6 +475,8 @@ describe('AiService', () => {
           locale: 'de-CH'
         })
       ).rejects.toBe(error);
+
+      expect(generateText).toHaveBeenCalledTimes(1);
     });
   });
 
@@ -1559,6 +1691,48 @@ describe('AiService', () => {
       );
     });
 
+    it('Requests the matches one more time if the output does not agree with the schema', async () => {
+      jest.spyOn(Logger.prototype, 'warn').mockImplementation();
+
+      jest.mocked(generateText).mockRejectedValueOnce(
+        new NoObjectGeneratedError({
+          finishReason: 'stop',
+          response: { modelId: 'agent-model' },
+          usage: { totalTokens: 1000 }
+        } as ConstructorParameters<typeof NoObjectGeneratedError>[0])
+      );
+
+      mockRowMatches({
+        assets: [
+          { currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' }
+        ]
+      });
+
+      const { activities } = await aiService.getActivitiesFromRows({
+        abortSignal,
+        accounts,
+        userCurrency,
+        userId,
+        rows: [createActivityRow({ accountName: null })]
+      });
+
+      expect(activities).toEqual([expect.objectContaining({ symbol: 'AAPL' })]);
+
+      expect(generateText).toHaveBeenCalledTimes(2);
+
+      const [[firstOptions], [secondOptions]] =
+        jest.mocked(generateText).mock.calls;
+
+      expect(secondOptions).toEqual(firstOptions);
+      expect(secondOptions.abortSignal).toBe(abortSignal);
+
+      expect(Logger.prototype.warn).toHaveBeenCalledWith(
+        expect.stringContaining(
+          'the model agent-model one more time after 1000 tokens'
+        )
+      );
+    });
+
     it('Falls back to the general model if no agent model is set', async () => {
       delete properties[PROPERTY_OPENROUTER_MODEL_AGENT];
 

apps/api/src/app/endpoints/ai/ai.service.ts

@@ -289,7 +289,8 @@ export class AiService {
    * name of the file is passed as well, as it can name the broker. The
    * reasoning is disabled, as it can add thousands of output tokens even for a
    * single row. The temperature is 0, so that the same file gives the same
-   * rows. A row above the maximum number of activities is left out. An
+   * rows. An output which does not agree with the schema is generated one more
+   * time. A row above the maximum number of activities is left out. An
    * output which reaches the maximum number of output tokens is cut, hence it
    * gives an error for the caller. A cut output without any text is not parsed
    * at all, hence the finish reason is checked as well.
@@ -314,7 +315,8 @@ export class AiService {
         : { data: file.buffer, mediaType: file.mediaType, type: 'file' }
     ];
 
-    const result = await generateText({
+    const result = await this.generateWithRetry(async () => {
+      return generateText({
         abortSignal,
         instructions: [
           'You read the activities of a portfolio from a file, for example a CSV export or a statement of a broker, a confirmation of a trade or a screenshot.',
@@ -346,6 +348,7 @@ export class AiService {
         }),
         output: Output.object({ schema: ACTIVITY_ROWS_OUTPUT }),
         temperature: 0
+      });
     }).catch((error: unknown) => {
       if (
         NoObjectGeneratedError.isInstance(error) &&
@@ -411,6 +414,63 @@ export class AiService {
     ].join('\n');
   }
 
+  /**
+   * Gives the result of a model, or an error if its output is empty, as an
+   * empty output is not parsed if the model does not stop normally
+   */
+  private checkOutput<
+    T extends Pick<
+      Awaited<ReturnType<typeof generateText>>,
+      'finalStep' | 'finishReason' | 'text' | 'usage'
+    >
+  >(result: T) {
+    if (result.finishReason === 'other' && result.text === '') {
+      throw new NoObjectGeneratedError({
+        finishReason: result.finishReason,
+        message: 'No object generated: the response is empty.',
+        response: result.finalStep.response,
+        text: result.text,
+        usage: result.usage
+      });
+    }
+
+    return result;
+  }
+
+  /**
+   * Generates the output of a model and generates it one more time if it does
+   * not agree with the schema, as a provider can give an output which does not
+   * agree with the schema although OpenRouter lists the structured outputs for
+   * it, for example with a code block around the JSON. An empty output is
+   * generated one more time as well. Only an output which stops normally or for
+   * another reason is generated again, as the second output would be the same,
+   * for example if it reaches the maximum number of output tokens or if the
+   * content filter stops it.
+   */
+  private async generateWithRetry<
+    T extends Pick<
+      Awaited<ReturnType<typeof generateText>>,
+      'finalStep' | 'finishReason' | 'text' | 'usage'
+    >
+  >(generate: () => Promise<T>) {
+    try {
+      return this.checkOutput(await generate());
+    } catch (error) {
+      if (
+        !NoObjectGeneratedError.isInstance(error) ||
+        (error.finishReason !== 'other' && error.finishReason !== 'stop')
+      ) {
+        throw error;
+      }
+
+      this.logger.warn(
+        `Generating the output of the model ${error.response?.modelId} one more time after ${error.usage?.totalTokens} tokens, as it does not agree with the schema: ${error.message}`
+      );
+
+      return this.checkOutput(await generate());
+    }
+  }
+
   /**
    * Gives each fee, interest and liability a custom asset profile with its
    * description as name, as the import after the dry run requires a valid
@@ -554,7 +614,8 @@ export class AiService {
    * If the holdings or the data sources cannot be got, the model gets no
    * holdings. The reasoning is disabled, as it can add thousands of output
    * tokens. The temperature is 0, so that the same assets give the same
-   * matches. An asset match is left out if neither the search of its own asset
+   * matches. An output which does not agree with the schema is generated one
+   * more time. An asset match is left out if neither the search of its own asset
    * nor the holdings gave its asset profile, or if its holding has another ISIN
    * than its own asset, as the import refuses a symbol which the data source
    * cannot resolve and as the model can mix up the ids of the assets. The ISINs
@@ -608,7 +669,9 @@ export class AiService {
       return assetProfiles;
     });
 
-    const { output, steps, totalUsage } = await generateText({
+    const { output, steps, totalUsage } = await this.generateWithRetry(
+      async () => {
+        return generateText({
           abortSignal,
           instructions: [
             'You match the assets and the account hints of a file to the asset profiles of Ghostfolio and to the accounts of the user.',
@@ -638,6 +701,8 @@ export class AiService {
           }),
           temperature: 0
         });
+      }
+    );
 
     const assetMatches = output.assets.filter(({ dataSource, id, symbol }) => {
       const assetProfileIdentifier = getAssetProfileIdentifier({