mirror of https://github.com/ghostfolio/ghostfolio
committed by
GitHub
28 changed files with 2796 additions and 35 deletions
@ -0,0 +1,279 @@ |
|||||
|
import { HAS_PERMISSION_KEY } from '@ghostfolio/api/decorators/has-permission.decorator'; |
||||
|
import { CallerFacingError } from '@ghostfolio/api/errors/caller-facing.error'; |
||||
|
import { CustomThrottlerGuard } from '@ghostfolio/api/guards/custom-throttler.guard'; |
||||
|
import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'; |
||||
|
import { ApiService } from '@ghostfolio/api/services/api/api.service'; |
||||
|
import { |
||||
|
AI_ACTIVITIES_TIMEOUT, |
||||
|
AI_MAX_ACTIVITIES, |
||||
|
THROTTLE_AI_ACTIVITIES_LIMIT, |
||||
|
THROTTLE_AI_ACTIVITIES_TTL |
||||
|
} from '@ghostfolio/common/config'; |
||||
|
import { permissions } from '@ghostfolio/common/permissions'; |
||||
|
import type { RequestWithUser } from '@ghostfolio/common/types'; |
||||
|
|
||||
|
import { HttpException, Logger } from '@nestjs/common'; |
||||
|
import { GUARDS_METADATA } from '@nestjs/common/constants'; |
||||
|
import { |
||||
|
THROTTLER_LIMIT, |
||||
|
THROTTLER_TRACKER, |
||||
|
THROTTLER_TTL |
||||
|
} from '@nestjs/throttler/dist/throttler.constants'; |
||||
|
import type { Response } from 'express'; |
||||
|
import { StatusCodes } from 'http-status-codes'; |
||||
|
import { EventEmitter } from 'node:events'; |
||||
|
|
||||
|
import { AiController } from './ai.controller'; |
||||
|
import { AiService } from './ai.service'; |
||||
|
|
||||
|
// The packages are ECMAScript modules, which Jest cannot load
|
||||
|
jest.mock('@openrouter/ai-sdk-provider', () => { |
||||
|
return {}; |
||||
|
}); |
||||
|
jest.mock('ai', () => { |
||||
|
return {}; |
||||
|
}); |
||||
|
|
||||
|
describe('AiController', () => { |
||||
|
const pdfFile = { |
||||
|
buffer: Buffer.from('%PDF-1.7'), |
||||
|
originalname: 'statement.pdf' |
||||
|
}; |
||||
|
|
||||
|
const user = { id: 'user-id' }; |
||||
|
|
||||
|
let aiController: AiController; |
||||
|
let aiService: AiService; |
||||
|
|
||||
|
function createCsvFile(dataRowCount: number) { |
||||
|
return { |
||||
|
buffer: Buffer.from( |
||||
|
[ |
||||
|
'Date,Symbol', |
||||
|
...Array.from({ length: dataRowCount }, () => { |
||||
|
return '2026-01-15,AAPL'; |
||||
|
}) |
||||
|
].join('\n') |
||||
|
), |
||||
|
originalname: 'statement.csv' |
||||
|
}; |
||||
|
} |
||||
|
|
||||
|
function createResponse() { |
||||
|
return Object.assign(new EventEmitter(), { |
||||
|
writableFinished: false |
||||
|
}) as unknown as Response; |
||||
|
} |
||||
|
|
||||
|
function getMetadataOfMethod<T>(metadataKey: string) { |
||||
|
return Reflect.getMetadata( |
||||
|
metadataKey, |
||||
|
AiController.prototype.getActivitiesFromFile |
||||
|
) as T; |
||||
|
} |
||||
|
|
||||
|
beforeEach(() => { |
||||
|
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
||||
|
|
||||
|
aiService = { |
||||
|
getActivitiesFromFile: jest.fn().mockResolvedValue({ |
||||
|
activities: [], |
||||
|
assetProfiles: [], |
||||
|
rowsWithoutMatch: [] |
||||
|
}) |
||||
|
} as unknown as AiService; |
||||
|
|
||||
|
aiController = new AiController( |
||||
|
aiService, |
||||
|
{} as ApiService, |
||||
|
{ |
||||
|
user |
||||
|
} as unknown as RequestWithUser |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
describe('getActivitiesFromFile', () => { |
||||
|
it('Requires the permission and limits the requests of each user per week', () => { |
||||
|
expect(getMetadataOfMethod<string>(HAS_PERMISSION_KEY)).toBe( |
||||
|
permissions.importActivitiesWithAi |
||||
|
); |
||||
|
|
||||
|
const guards = getMetadataOfMethod<unknown[]>(GUARDS_METADATA); |
||||
|
|
||||
|
expect(guards.slice(1)).toEqual([ |
||||
|
HasPermissionGuard, |
||||
|
CustomThrottlerGuard |
||||
|
]); |
||||
|
|
||||
|
expect(getMetadataOfMethod<number>(`${THROTTLER_LIMIT}default`)).toBe( |
||||
|
THROTTLE_AI_ACTIVITIES_LIMIT |
||||
|
); |
||||
|
|
||||
|
expect(getMetadataOfMethod<number>(`${THROTTLER_TTL}default`)).toBe( |
||||
|
THROTTLE_AI_ACTIVITIES_TTL |
||||
|
); |
||||
|
|
||||
|
expect( |
||||
|
getMetadataOfMethod<(request: unknown) => string>( |
||||
|
`${THROTTLER_TRACKER}default` |
||||
|
)({ ip: '127.0.0.1', user }) |
||||
|
).toBe(user.id); |
||||
|
}); |
||||
|
|
||||
|
it('Refuses a request without a file or with a file of another type', async () => { |
||||
|
for (const file of [ |
||||
|
undefined, |
||||
|
{ buffer: Buffer.from('Date,Symbol'), originalname: 'statement.txt' } |
||||
|
]) { |
||||
|
await expect( |
||||
|
aiController.getActivitiesFromFile(createResponse(), file) |
||||
|
).rejects.toThrow(HttpException); |
||||
|
} |
||||
|
|
||||
|
expect(aiService.getActivitiesFromFile).not.toHaveBeenCalled(); |
||||
|
}); |
||||
|
|
||||
|
it(`Refuses a CSV file with more than ${AI_MAX_ACTIVITIES} rows before a model call`, async () => { |
||||
|
await expect( |
||||
|
aiController.getActivitiesFromFile( |
||||
|
createResponse(), |
||||
|
createCsvFile(AI_MAX_ACTIVITIES + 1) |
||||
|
) |
||||
|
).rejects.toThrow(HttpException); |
||||
|
|
||||
|
expect(aiService.getActivitiesFromFile).not.toHaveBeenCalled(); |
||||
|
|
||||
|
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
||||
|
`Rejected a CSV file with ${AI_MAX_ACTIVITIES + 1} rows` |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it(`Accepts a CSV file with ${AI_MAX_ACTIVITIES} rows`, async () => { |
||||
|
const file = createCsvFile(AI_MAX_ACTIVITIES); |
||||
|
|
||||
|
await aiController.getActivitiesFromFile(createResponse(), file); |
||||
|
|
||||
|
expect(aiService.getActivitiesFromFile).toHaveBeenCalledWith( |
||||
|
expect.objectContaining({ |
||||
|
file: { |
||||
|
buffer: file.buffer, |
||||
|
mediaType: 'text/csv', |
||||
|
name: 'statement.csv' |
||||
|
} |
||||
|
}) |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('Passes the file with its media type and its name and the user of the request', async () => { |
||||
|
await expect( |
||||
|
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
||||
|
).resolves.toEqual({ |
||||
|
activities: [], |
||||
|
assetProfiles: [], |
||||
|
rowsWithoutMatch: [] |
||||
|
}); |
||||
|
|
||||
|
expect(aiService.getActivitiesFromFile).toHaveBeenCalledWith( |
||||
|
expect.objectContaining({ |
||||
|
user, |
||||
|
file: { |
||||
|
buffer: pdfFile.buffer, |
||||
|
mediaType: 'application/pdf', |
||||
|
name: 'statement.pdf' |
||||
|
} |
||||
|
}) |
||||
|
); |
||||
|
|
||||
|
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
||||
|
.mock.calls; |
||||
|
|
||||
|
expect(abortSignal).toBeInstanceOf(AbortSignal); |
||||
|
}); |
||||
|
|
||||
|
it('Stops the models if the client closes the request before the response', async () => { |
||||
|
const response = createResponse(); |
||||
|
|
||||
|
await aiController.getActivitiesFromFile(response, pdfFile); |
||||
|
|
||||
|
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
||||
|
.mock.calls; |
||||
|
|
||||
|
expect(abortSignal.aborted).toBe(false); |
||||
|
|
||||
|
response.emit('close'); |
||||
|
|
||||
|
expect(abortSignal.aborted).toBe(true); |
||||
|
}); |
||||
|
|
||||
|
it('Does not stop the models when the response is finished', async () => { |
||||
|
const response = createResponse(); |
||||
|
|
||||
|
await aiController.getActivitiesFromFile(response, pdfFile); |
||||
|
|
||||
|
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
||||
|
.mock.calls; |
||||
|
|
||||
|
Object.assign(response, { writableFinished: true }); |
||||
|
response.emit('close'); |
||||
|
|
||||
|
expect(abortSignal.aborted).toBe(false); |
||||
|
}); |
||||
|
|
||||
|
it('Gives the status 400 with the message of an error for the caller', async () => { |
||||
|
jest |
||||
|
.mocked(aiService.getActivitiesFromFile) |
||||
|
.mockRejectedValueOnce( |
||||
|
new CallerFacingError('The file has too many activities') |
||||
|
); |
||||
|
|
||||
|
await expect( |
||||
|
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
||||
|
).rejects.toMatchObject({ |
||||
|
response: { message: ['The file has too many activities'] }, |
||||
|
status: StatusCodes.BAD_REQUEST |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('Gives the status 400 with a message for the caller at the timeout', async () => { |
||||
|
const timeoutSignal = AbortSignal.abort( |
||||
|
new DOMException( |
||||
|
'The operation was aborted due to timeout', |
||||
|
'TimeoutError' |
||||
|
) |
||||
|
); |
||||
|
|
||||
|
jest.spyOn(AbortSignal, 'timeout').mockReturnValueOnce(timeoutSignal); |
||||
|
|
||||
|
jest |
||||
|
.mocked(aiService.getActivitiesFromFile) |
||||
|
.mockRejectedValueOnce(timeoutSignal.reason); |
||||
|
|
||||
|
await expect( |
||||
|
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
||||
|
).rejects.toMatchObject({ |
||||
|
response: { |
||||
|
message: [ |
||||
|
'Ghostfolio AI needs too much time for the file. Split it into smaller files.' |
||||
|
] |
||||
|
}, |
||||
|
status: StatusCodes.BAD_REQUEST |
||||
|
}); |
||||
|
|
||||
|
expect(AbortSignal.timeout).toHaveBeenCalledWith(AI_ACTIVITIES_TIMEOUT); |
||||
|
|
||||
|
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
||||
|
`Stopped reading a file of the type application/pdf at the timeout of ${AI_ACTIVITIES_TIMEOUT} ms` |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('Passes on every other error', async () => { |
||||
|
const error = new Error('Model error'); |
||||
|
|
||||
|
jest.mocked(aiService.getActivitiesFromFile).mockRejectedValueOnce(error); |
||||
|
|
||||
|
await expect( |
||||
|
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
||||
|
).rejects.toBe(error); |
||||
|
}); |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,99 @@ |
|||||
|
import { |
||||
|
getDataRowCountOfCsvFile, |
||||
|
getMediaTypeOfActivityFile |
||||
|
} from './ai.helper'; |
||||
|
|
||||
|
describe('getDataRowCountOfCsvFile', () => { |
||||
|
it('Gives the number of the lines which are not empty, without the header', () => { |
||||
|
expect( |
||||
|
getDataRowCountOfCsvFile( |
||||
|
Buffer.from('Date,Symbol\n2026-01-15,AAPL\n\n2026-01-16,MSFT\n') |
||||
|
) |
||||
|
).toBe(2); |
||||
|
}); |
||||
|
|
||||
|
it('Counts each line of a value with a line break in quotes', () => { |
||||
|
expect( |
||||
|
getDataRowCountOfCsvFile( |
||||
|
Buffer.from('Date,Comment\r\n2026-01-15,"First\r\nsecond"\r\n') |
||||
|
) |
||||
|
).toBe(2); |
||||
|
}); |
||||
|
|
||||
|
it('Counts each line after a quote which is not closed', () => { |
||||
|
expect( |
||||
|
getDataRowCountOfCsvFile( |
||||
|
Buffer.from( |
||||
|
[ |
||||
|
'Date,Symbol,Quantity', |
||||
|
'2026-01-15,"AAPL,1', |
||||
|
...Array.from({ length: 200 }, () => { |
||||
|
return '2026-01-16,MSFT,1'; |
||||
|
}) |
||||
|
].join('\n') |
||||
|
) |
||||
|
) |
||||
|
).toBe(201); |
||||
|
}); |
||||
|
|
||||
|
it('Gives 0 for a file with a header only or without content', () => { |
||||
|
expect(getDataRowCountOfCsvFile(Buffer.from('Date,Symbol\n'))).toBe(0); |
||||
|
expect(getDataRowCountOfCsvFile(Buffer.from(''))).toBe(0); |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
describe('getMediaTypeOfActivityFile', () => { |
||||
|
it('Identifies a JPEG, a PDF and a PNG file by the first bytes', () => { |
||||
|
expect( |
||||
|
getMediaTypeOfActivityFile({ |
||||
|
buffer: Buffer.from([0xff, 0xd8, 0xff, 0xe0]), |
||||
|
fileName: 'statement' |
||||
|
}) |
||||
|
).toBe('image/jpeg'); |
||||
|
|
||||
|
expect( |
||||
|
getMediaTypeOfActivityFile({ |
||||
|
buffer: Buffer.from('%PDF-1.7'), |
||||
|
fileName: 'statement.csv' |
||||
|
}) |
||||
|
).toBe('application/pdf'); |
||||
|
|
||||
|
expect( |
||||
|
getMediaTypeOfActivityFile({ |
||||
|
buffer: Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), |
||||
|
fileName: 'statement.jpg' |
||||
|
}) |
||||
|
).toBe('image/png'); |
||||
|
}); |
||||
|
|
||||
|
it('Identifies a CSV file by the extension and the content in UTF-8', () => { |
||||
|
for (const fileName of ['statement.csv', 'STATEMENT.CSV']) { |
||||
|
expect( |
||||
|
getMediaTypeOfActivityFile({ |
||||
|
fileName, |
||||
|
buffer: Buffer.from('Datum;Stück\n15.01.2026;10\n') |
||||
|
}) |
||||
|
).toBe('text/csv'); |
||||
|
} |
||||
|
}); |
||||
|
|
||||
|
it('Refuses a CSV file which is not in UTF-8', () => { |
||||
|
expect( |
||||
|
getMediaTypeOfActivityFile({ |
||||
|
buffer: Buffer.from([0x53, 0x74, 0xfc, 0x63, 0x6b]), |
||||
|
fileName: 'statement.csv' |
||||
|
}) |
||||
|
).toBeUndefined(); |
||||
|
}); |
||||
|
|
||||
|
it('Refuses any other file', () => { |
||||
|
for (const { buffer, fileName } of [ |
||||
|
{ buffer: Buffer.from('Date,Symbol'), fileName: 'statement.txt' }, |
||||
|
{ buffer: Buffer.from('<html></html>'), fileName: 'statement.pdf' }, |
||||
|
{ buffer: Buffer.from([0xff, 0xd8]), fileName: 'statement.jpg' }, |
||||
|
{ buffer: Buffer.from(''), fileName: 'statement.png' } |
||||
|
]) { |
||||
|
expect(getMediaTypeOfActivityFile({ buffer, fileName })).toBeUndefined(); |
||||
|
} |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,57 @@ |
|||||
|
import { isUtf8 } from 'node:buffer'; |
||||
|
|
||||
|
const JPEG_SIGNATURE = [0xff, 0xd8, 0xff]; |
||||
|
const PDF_SIGNATURE = [0x25, 0x50, 0x44, 0x46, 0x2d]; |
||||
|
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; |
||||
|
|
||||
|
/** |
||||
|
* Gives the number of the lines of a CSV file which are not empty, without |
||||
|
* the first line. A line before the header and each line of a value with a |
||||
|
* line break are counted as data rows, hence the number is never too low, |
||||
|
* also for a file with a quote which is not closed. |
||||
|
*/ |
||||
|
export function getDataRowCountOfCsvFile(buffer: Buffer) { |
||||
|
const lineCount = buffer |
||||
|
.toString('utf8') |
||||
|
.split(/\r\n|\r|\n/) |
||||
|
.filter((line) => { |
||||
|
return line.trim().length > 0; |
||||
|
}).length; |
||||
|
|
||||
|
return Math.max(lineCount - 1, 0); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Gives the media type of a file which Ghostfolio AI reads. A JPEG, a PDF or |
||||
|
* a PNG file is identified by its first bytes, a CSV file by its extension |
||||
|
* and by its content in UTF-8, as the media type which a client sends for a |
||||
|
* CSV file varies. |
||||
|
*/ |
||||
|
export function getMediaTypeOfActivityFile({ |
||||
|
buffer, |
||||
|
fileName |
||||
|
}: { |
||||
|
buffer: Buffer; |
||||
|
fileName: string; |
||||
|
}) { |
||||
|
if (startsWith(buffer, JPEG_SIGNATURE)) { |
||||
|
return 'image/jpeg'; |
||||
|
} else if (startsWith(buffer, PDF_SIGNATURE)) { |
||||
|
return 'application/pdf'; |
||||
|
} else if (startsWith(buffer, PNG_SIGNATURE)) { |
||||
|
return 'image/png'; |
||||
|
} else if (fileName.toLowerCase().endsWith('.csv') && isUtf8(buffer)) { |
||||
|
return 'text/csv'; |
||||
|
} |
||||
|
|
||||
|
return undefined; |
||||
|
} |
||||
|
|
||||
|
function startsWith(buffer: Buffer, signature: number[]) { |
||||
|
return ( |
||||
|
buffer.length >= signature.length && |
||||
|
signature.every((byte, index) => { |
||||
|
return buffer[index] === byte; |
||||
|
}) |
||||
|
); |
||||
|
} |
||||
@ -0,0 +1,101 @@ |
|||||
|
import { |
||||
|
AI_MAX_ACTIVITIES, |
||||
|
SEARCH_QUERY_MAXIMUM_LENGTH |
||||
|
} from '@ghostfolio/common/config'; |
||||
|
|
||||
|
import { isObject } from 'lodash-es'; |
||||
|
import { z } from 'zod'; |
||||
|
|
||||
|
import { ACTIVITY_ROWS_OUTPUT, ROW_MATCHES_OUTPUT } from './ai.schemas'; |
||||
|
import { createActivityRow } from './ai.test-utils'; |
||||
|
import { ActivityRow } from './types/activity-row.type'; |
||||
|
|
||||
|
describe('ACTIVITY_ROWS_OUTPUT', () => { |
||||
|
it('Accepts a row with null for the values which the file does not show', () => { |
||||
|
expect( |
||||
|
ACTIVITY_ROWS_OUTPUT.safeParse({ |
||||
|
broker: null, |
||||
|
rows: [ |
||||
|
createActivityRow({ |
||||
|
accountName: null, |
||||
|
currency: null, |
||||
|
isin: null, |
||||
|
symbol: null |
||||
|
}) |
||||
|
] |
||||
|
}).success |
||||
|
).toBe(true); |
||||
|
}); |
||||
|
|
||||
|
it('Refuses a row with a value which is left out instead of null', () => { |
||||
|
const row: Partial<ActivityRow> = createActivityRow(); |
||||
|
|
||||
|
delete row.isin; |
||||
|
|
||||
|
expect( |
||||
|
ACTIVITY_ROWS_OUTPUT.safeParse({ broker: null, rows: [row] }).success |
||||
|
).toBe(false); |
||||
|
}); |
||||
|
|
||||
|
it('Accepts a currency, a date, a number, a printed text and a number of rows which the join checks, so that one value does not refuse all rows', () => { |
||||
|
expect( |
||||
|
ACTIVITY_ROWS_OUTPUT.safeParse({ |
||||
|
broker: null, |
||||
|
rows: [ |
||||
|
createActivityRow({ |
||||
|
currency: 'GBX', |
||||
|
date: '15.01.2026', |
||||
|
name: 'A'.repeat(SEARCH_QUERY_MAXIMUM_LENGTH + 1) |
||||
|
}), |
||||
|
createActivityRow({ fee: -1, quantity: -1, unitPrice: -1 }), |
||||
|
...Array.from({ length: AI_MAX_ACTIVITIES }, () => { |
||||
|
return createActivityRow(); |
||||
|
}) |
||||
|
] |
||||
|
}).success |
||||
|
).toBe(true); |
||||
|
}); |
||||
|
|
||||
|
it('Trims a printed text', () => { |
||||
|
const { data } = ACTIVITY_ROWS_OUTPUT.safeParse({ |
||||
|
broker: ' Swissquote ', |
||||
|
rows: [createActivityRow({ name: ' Apple Inc. ' })] |
||||
|
}); |
||||
|
|
||||
|
expect(data.broker).toBe('Swissquote'); |
||||
|
expect(data.rows[0].name).toBe('Apple Inc.'); |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
describe.each([ |
||||
|
['ACTIVITY_ROWS_OUTPUT', ACTIVITY_ROWS_OUTPUT], |
||||
|
['ROW_MATCHES_OUTPUT', ROW_MATCHES_OUTPUT] |
||||
|
])('%s', (_name, schema) => { |
||||
|
it('Gives a JSON schema which the strict mode of the structured outputs supports', () => { |
||||
|
// The options of the conversion in the AI SDK
|
||||
|
const jsonSchema = z.toJSONSchema(schema, { |
||||
|
io: 'input', |
||||
|
target: 'draft-7' |
||||
|
}); |
||||
|
|
||||
|
const objectJsonSchemas: { properties: object; required: string[] }[] = []; |
||||
|
|
||||
|
JSON.stringify(jsonSchema, (_key, value: unknown) => { |
||||
|
if (isObject(value) && 'properties' in value) { |
||||
|
objectJsonSchemas.push( |
||||
|
value as { properties: object; required: string[] } |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
return value; |
||||
|
}); |
||||
|
|
||||
|
expect(objectJsonSchemas.length).toBeGreaterThan(1); |
||||
|
|
||||
|
for (const { properties, required } of objectJsonSchemas) { |
||||
|
expect([...required].sort()).toEqual(Object.keys(properties).sort()); |
||||
|
} |
||||
|
|
||||
|
expect(JSON.stringify(jsonSchema)).not.toMatch(/"(max|min)(imum|Length)"/); |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,87 @@ |
|||||
|
import { AI_MAX_ACTIVITIES } from '@ghostfolio/common/config'; |
||||
|
|
||||
|
import { Type as ActivityType } from '@prisma/client'; |
||||
|
import { z } from 'zod'; |
||||
|
|
||||
|
// The strict mode of the structured outputs requires every property, hence a
|
||||
|
// value which the file does not show is null instead of being left out. The
|
||||
|
// values are checked when the rows and the matches are joined to the
|
||||
|
// activities to import, so that a wrong value or a wrong match skips its rows
|
||||
|
// only
|
||||
|
const PRINTED_TEXT = z.string().trim().nullable(); |
||||
|
|
||||
|
export const ACTIVITY_ROWS_OUTPUT = z.object({ |
||||
|
broker: PRINTED_TEXT.describe( |
||||
|
'The name of the broker or of the bank which issued the file' |
||||
|
), |
||||
|
rows: z |
||||
|
.array( |
||||
|
z.object({ |
||||
|
accountName: PRINTED_TEXT.describe( |
||||
|
'The name or the number of the account, as printed' |
||||
|
), |
||||
|
currency: z |
||||
|
.string() |
||||
|
.nullable() |
||||
|
.describe( |
||||
|
'The currency of the fee and of the unit price, as an ISO 4217 code in upper case' |
||||
|
), |
||||
|
date: z |
||||
|
.string() |
||||
|
.describe('The date of the activity, as an ISO 8601 date'), |
||||
|
fee: z.number().describe('The fee of the activity'), |
||||
|
isin: PRINTED_TEXT.describe('The ISIN of the asset, as printed'), |
||||
|
name: PRINTED_TEXT.describe( |
||||
|
'The name of the asset, or the description of a fee, an interest or a liability, as printed' |
||||
|
), |
||||
|
quantity: z.number().describe('The quantity of the activity'), |
||||
|
symbol: PRINTED_TEXT.describe( |
||||
|
'The ticker symbol of the asset, as printed, not a national number such as a valor, a WKN or a CUSIP' |
||||
|
), |
||||
|
type: z.enum(ActivityType).describe('The type of the activity'), |
||||
|
unitPrice: z.number().describe('The unit price of the activity') |
||||
|
}) |
||||
|
) |
||||
|
.describe(`The activities of the file, at most ${AI_MAX_ACTIVITIES}`) |
||||
|
}); |
||||
|
|
||||
|
export const ROW_MATCHES_OUTPUT = z.object({ |
||||
|
accounts: z |
||||
|
.array( |
||||
|
z.object({ |
||||
|
accountId: z |
||||
|
.string() |
||||
|
.nullable() |
||||
|
.describe( |
||||
|
'The id of the account of the user which matches, as given by get-accounts, or null if no account matches' |
||||
|
), |
||||
|
id: z.number().describe('The id of the account hint, as given') |
||||
|
}) |
||||
|
) |
||||
|
.describe('One entry for each account hint of the file'), |
||||
|
assets: z |
||||
|
.array( |
||||
|
z.object({ |
||||
|
currency: z |
||||
|
.string() |
||||
|
.nullable() |
||||
|
.describe( |
||||
|
'The currency of the asset profile, as given by search-asset-profiles' |
||||
|
), |
||||
|
dataSource: z |
||||
|
.string() |
||||
|
.nullable() |
||||
|
.describe( |
||||
|
'The data source of the asset profile, as given by search-asset-profiles' |
||||
|
), |
||||
|
id: z.number().describe('The id of the asset, as given'), |
||||
|
symbol: z |
||||
|
.string() |
||||
|
.nullable() |
||||
|
.describe( |
||||
|
'The symbol of the asset profile, as given by search-asset-profiles, or null if no asset profile matches' |
||||
|
) |
||||
|
}) |
||||
|
) |
||||
|
.describe('One entry for each asset of the file') |
||||
|
}); |
||||
@ -0,0 +1,955 @@ |
|||||
|
import { McpService } from '@ghostfolio/api/app/endpoints/mcp/mcp.service'; |
||||
|
import { CallerFacingError } from '@ghostfolio/api/errors/caller-facing.error'; |
||||
|
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; |
||||
|
import { PortfolioTableService } from '@ghostfolio/api/services/portfolio-table/portfolio-table.service'; |
||||
|
import { PropertyService } from '@ghostfolio/api/services/property/property.service'; |
||||
|
import { |
||||
|
DEFAULT_LOCALE, |
||||
|
PROPERTY_API_KEY_OPENROUTER, |
||||
|
PROPERTY_OPENROUTER_MODEL, |
||||
|
PROPERTY_OPENROUTER_MODEL_AGENT, |
||||
|
PROPERTY_OPENROUTER_MODEL_VISION |
||||
|
} from '@ghostfolio/common/config'; |
||||
|
import { isValidCustomAssetProfileSymbol } from '@ghostfolio/common/helper'; |
||||
|
import type { UserWithSettings } from '@ghostfolio/common/types'; |
||||
|
|
||||
|
import { Logger } from '@nestjs/common'; |
||||
|
import { createOpenRouter } from '@openrouter/ai-sdk-provider'; |
||||
|
import { DataSource, Type as ActivityType } from '@prisma/client'; |
||||
|
import { generateText, isStepCount, NoObjectGeneratedError, ToolSet } from 'ai'; |
||||
|
import { z } from 'zod'; |
||||
|
|
||||
|
import { ACTIVITY_ROWS_OUTPUT, ROW_MATCHES_OUTPUT } from './ai.schemas'; |
||||
|
import { AiService } from './ai.service'; |
||||
|
import { createActivityRow } from './ai.test-utils'; |
||||
|
import { AccountHint } from './types/account-hint.type'; |
||||
|
import { AccountToMatch } from './types/account-to-match.type'; |
||||
|
import { ActivityRow } from './types/activity-row.type'; |
||||
|
|
||||
|
// The packages are ECMAScript modules, which Jest cannot load
|
||||
|
jest.mock('@openrouter/ai-sdk-provider', () => { |
||||
|
return { createOpenRouter: jest.fn() }; |
||||
|
}); |
||||
|
jest.mock('ai', () => { |
||||
|
return { |
||||
|
generateText: jest.fn(), |
||||
|
isStepCount: jest.fn((stepCount: number) => { |
||||
|
return { stepCount }; |
||||
|
}), |
||||
|
NoObjectGeneratedError: class MockNoObjectGeneratedError extends Error { |
||||
|
public readonly finishReason: string; |
||||
|
|
||||
|
public constructor({ finishReason }: { finishReason: string }) { |
||||
|
super('No object generated'); |
||||
|
|
||||
|
this.finishReason = finishReason; |
||||
|
} |
||||
|
|
||||
|
public static isInstance(error: unknown) { |
||||
|
return error instanceof MockNoObjectGeneratedError; |
||||
|
} |
||||
|
}, |
||||
|
Output: { |
||||
|
object: jest.fn((options: unknown) => { |
||||
|
return options; |
||||
|
}) |
||||
|
}, |
||||
|
tool: jest.fn((options: unknown) => { |
||||
|
return options; |
||||
|
}) |
||||
|
}; |
||||
|
}); |
||||
|
|
||||
|
describe('AiService', () => { |
||||
|
const abortSignal = new AbortController().signal; |
||||
|
const model = { modelId: 'model' }; |
||||
|
|
||||
|
const rows: ActivityRow[] = [ |
||||
|
createActivityRow({ accountName: null, symbol: null }) |
||||
|
]; |
||||
|
|
||||
|
let aiService: AiService; |
||||
|
let chat: jest.Mock; |
||||
|
let mcpService: McpService; |
||||
|
let properties: Record<string, string>; |
||||
|
|
||||
|
beforeEach(() => { |
||||
|
jest.clearAllMocks(); |
||||
|
|
||||
|
jest.spyOn(Logger.prototype, 'log').mockImplementation(); |
||||
|
|
||||
|
chat = jest.fn().mockReturnValue(model); |
||||
|
|
||||
|
jest |
||||
|
.mocked(createOpenRouter) |
||||
|
.mockReturnValue({ chat } as unknown as ReturnType< |
||||
|
typeof createOpenRouter |
||||
|
>); |
||||
|
|
||||
|
jest.mocked(generateText).mockResolvedValue({ |
||||
|
output: { broker: 'Swissquote', rows }, |
||||
|
steps: [{}], |
||||
|
totalUsage: { totalTokens: 1000 } |
||||
|
} as unknown as Awaited<ReturnType<typeof generateText>>); |
||||
|
|
||||
|
properties = { |
||||
|
[PROPERTY_API_KEY_OPENROUTER]: 'api-key', |
||||
|
[PROPERTY_OPENROUTER_MODEL]: 'general-model', |
||||
|
[PROPERTY_OPENROUTER_MODEL_AGENT]: 'agent-model', |
||||
|
[PROPERTY_OPENROUTER_MODEL_VISION]: 'vision-model' |
||||
|
}; |
||||
|
|
||||
|
mcpService = { |
||||
|
getAccounts: jest.fn().mockResolvedValue({ |
||||
|
content: [{ text: '## Accounts', type: 'text' }] |
||||
|
}), |
||||
|
searchAssetProfiles: jest.fn().mockResolvedValue({ |
||||
|
content: [{ text: '{ "assetProfiles": [] }', type: 'text' }] |
||||
|
}) |
||||
|
} as unknown as McpService; |
||||
|
|
||||
|
const propertyService = { |
||||
|
getByKey: jest.fn().mockImplementation((key: string) => { |
||||
|
return Promise.resolve(properties[key]); |
||||
|
}) |
||||
|
} as unknown as PropertyService; |
||||
|
|
||||
|
aiService = new AiService( |
||||
|
{} as ConfigurationService, |
||||
|
mcpService, |
||||
|
{} as PortfolioTableService, |
||||
|
propertyService |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
describe('getActivityRowsFromFile', () => { |
||||
|
it('Passes a PDF file as a file part to the vision model and gives the rows', async () => { |
||||
|
const buffer = Buffer.from('%PDF-1.7'); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivityRowsFromFile({ |
||||
|
abortSignal, |
||||
|
file: { buffer, mediaType: 'application/pdf' }, |
||||
|
locale: 'de-CH' |
||||
|
}) |
||||
|
).resolves.toEqual({ broker: 'Swissquote', rows }); |
||||
|
|
||||
|
expect(createOpenRouter).toHaveBeenCalledWith({ apiKey: 'api-key' }); |
||||
|
|
||||
|
expect(generateText).toHaveBeenCalledWith( |
||||
|
expect.objectContaining({ |
||||
|
abortSignal, |
||||
|
messages: [ |
||||
|
{ |
||||
|
content: [ |
||||
|
{ data: buffer, mediaType: 'application/pdf', type: 'file' } |
||||
|
], |
||||
|
role: 'user' |
||||
|
} |
||||
|
], |
||||
|
model, |
||||
|
output: { schema: ACTIVITY_ROWS_OUTPUT } |
||||
|
}) |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('Passes the name of the file as text before the file and gives the broker', async () => { |
||||
|
const buffer = Buffer.from('%PDF-1.7'); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivityRowsFromFile({ |
||||
|
file: { |
||||
|
buffer, |
||||
|
mediaType: 'application/pdf', |
||||
|
name: 'swissquote-2022.pdf' |
||||
|
}, |
||||
|
locale: 'de-CH' |
||||
|
}) |
||||
|
).resolves.toEqual({ broker: 'Swissquote', rows }); |
||||
|
|
||||
|
expect(generateText).toHaveBeenCalledWith( |
||||
|
expect.objectContaining({ |
||||
|
messages: [ |
||||
|
{ |
||||
|
content: [ |
||||
|
{ text: 'File name: swissquote-2022.pdf', type: 'text' }, |
||||
|
{ data: buffer, mediaType: 'application/pdf', type: 'file' } |
||||
|
], |
||||
|
role: 'user' |
||||
|
} |
||||
|
] |
||||
|
}) |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('Passes a CSV file as text', async () => { |
||||
|
const csv = 'Date;Type;ISIN\n15.01.2026;Kauf;US0378331005'; |
||||
|
|
||||
|
await aiService.getActivityRowsFromFile({ |
||||
|
file: { buffer: Buffer.from(csv), mediaType: 'text/csv' }, |
||||
|
locale: 'de-CH' |
||||
|
}); |
||||
|
|
||||
|
expect(generateText).toHaveBeenCalledWith( |
||||
|
expect.objectContaining({ |
||||
|
messages: [{ content: [{ text: csv, type: 'text' }], role: 'user' }] |
||||
|
}) |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('Restricts the routing to endpoints with zero data retention and passes a PDF file to the model itself', async () => { |
||||
|
await aiService.getActivityRowsFromFile({ |
||||
|
file: { buffer: Buffer.from('%PDF-1.7'), mediaType: 'application/pdf' }, |
||||
|
locale: 'de-CH' |
||||
|
}); |
||||
|
|
||||
|
expect(chat).toHaveBeenCalledWith('vision-model', { |
||||
|
plugins: [{ id: 'file-parser', pdf: { engine: 'native' } }], |
||||
|
provider: { require_parameters: true, zdr: true }, |
||||
|
usage: { include: true } |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('Falls back to the general model if no vision model is set', async () => { |
||||
|
delete properties[PROPERTY_OPENROUTER_MODEL_VISION]; |
||||
|
|
||||
|
await aiService.getActivityRowsFromFile({ |
||||
|
file: { buffer: Buffer.from('%PDF-1.7'), mediaType: 'application/pdf' }, |
||||
|
locale: 'de-CH' |
||||
|
}); |
||||
|
|
||||
|
expect(chat).toHaveBeenCalledWith('general-model', expect.any(Object)); |
||||
|
}); |
||||
|
|
||||
|
it('Gives the locale and the date of today in the instructions', async () => { |
||||
|
jest.useFakeTimers({ now: new Date('2026-10-11T12:00:00Z') }); |
||||
|
|
||||
|
try { |
||||
|
await aiService.getActivityRowsFromFile({ |
||||
|
file: { |
||||
|
buffer: Buffer.from('%PDF-1.7'), |
||||
|
mediaType: 'application/pdf' |
||||
|
}, |
||||
|
locale: 'de-CH' |
||||
|
}); |
||||
|
} finally { |
||||
|
jest.useRealTimers(); |
||||
|
} |
||||
|
|
||||
|
const { instructions } = jest.mocked(generateText).mock.calls[0][0]; |
||||
|
|
||||
|
expect(instructions).toContain('the locale de-CH'); |
||||
|
expect(instructions).toContain('Today is 2026-10-11'); |
||||
|
}); |
||||
|
|
||||
|
it('Tells the vision model that a national number of a security is not a ticker symbol', async () => { |
||||
|
await aiService.getActivityRowsFromFile({ |
||||
|
file: { |
||||
|
buffer: Buffer.from('%PDF-1.7'), |
||||
|
mediaType: 'application/pdf' |
||||
|
}, |
||||
|
locale: 'de-CH' |
||||
|
}); |
||||
|
|
||||
|
const { instructions } = jest.mocked(generateText).mock.calls[0][0]; |
||||
|
|
||||
|
expect(instructions).toContain( |
||||
|
'A national number of a security, for example a valor (NKN), a WKN or a CUSIP, is not a ticker symbol.' |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('Tells the vision model to add the fees of a buy or a sell to its row', async () => { |
||||
|
await aiService.getActivityRowsFromFile({ |
||||
|
file: { |
||||
|
buffer: Buffer.from('%PDF-1.7'), |
||||
|
mediaType: 'application/pdf' |
||||
|
}, |
||||
|
locale: 'de-CH' |
||||
|
}); |
||||
|
|
||||
|
const { instructions } = jest.mocked(generateText).mock.calls[0][0]; |
||||
|
|
||||
|
expect(instructions).toContain( |
||||
|
'Add every fee, commission and tax of a buy or a sell, for example a commission, a stamp duty or an exchange fee, to the fee of its row.' |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('Gives an error for the caller if a cut output reaches the maximum number of output tokens', async () => { |
||||
|
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
||||
|
|
||||
|
jest.mocked(generateText).mockRejectedValueOnce( |
||||
|
new NoObjectGeneratedError({ |
||||
|
finishReason: 'length' |
||||
|
} as ConstructorParameters<typeof NoObjectGeneratedError>[0]) |
||||
|
); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivityRowsFromFile({ |
||||
|
file: { |
||||
|
buffer: Buffer.from('%PDF-1.7'), |
||||
|
mediaType: 'application/pdf' |
||||
|
}, |
||||
|
locale: 'de-CH' |
||||
|
}) |
||||
|
).rejects.toThrow(CallerFacingError); |
||||
|
|
||||
|
expect(Logger.prototype.warn).toHaveBeenCalled(); |
||||
|
}); |
||||
|
|
||||
|
it('Gives an error for the caller if an output without any text reaches the maximum number of output tokens', async () => { |
||||
|
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
||||
|
|
||||
|
jest.mocked(generateText).mockResolvedValueOnce({ |
||||
|
finishReason: 'length', |
||||
|
get output() { |
||||
|
throw new Error('No output generated'); |
||||
|
} |
||||
|
} as unknown as Awaited<ReturnType<typeof generateText>>); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivityRowsFromFile({ |
||||
|
file: { |
||||
|
buffer: Buffer.from('%PDF-1.7'), |
||||
|
mediaType: 'application/pdf' |
||||
|
}, |
||||
|
locale: 'de-CH' |
||||
|
}) |
||||
|
).rejects.toThrow(CallerFacingError); |
||||
|
|
||||
|
expect(Logger.prototype.warn).toHaveBeenCalled(); |
||||
|
}); |
||||
|
|
||||
|
it('Passes on every other error of the vision model', async () => { |
||||
|
const error = new NoObjectGeneratedError({ |
||||
|
finishReason: 'stop' |
||||
|
} as ConstructorParameters<typeof NoObjectGeneratedError>[0]); |
||||
|
|
||||
|
jest.mocked(generateText).mockRejectedValueOnce(error); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivityRowsFromFile({ |
||||
|
file: { |
||||
|
buffer: Buffer.from('%PDF-1.7'), |
||||
|
mediaType: 'application/pdf' |
||||
|
}, |
||||
|
locale: 'de-CH' |
||||
|
}) |
||||
|
).rejects.toBe(error); |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
describe('getActivitiesFromFile', () => { |
||||
|
const file = { |
||||
|
buffer: Buffer.from('%PDF-1.7'), |
||||
|
mediaType: 'application/pdf' |
||||
|
}; |
||||
|
|
||||
|
const user = { |
||||
|
accounts: [{ currency: 'CHF', id: 'account-id', name: 'Brokerage' }], |
||||
|
id: 'user-id', |
||||
|
settings: { settings: { baseCurrency: 'EUR', locale: 'de-CH' } } |
||||
|
} as unknown as UserWithSettings; |
||||
|
|
||||
|
it('Reads the rows of the file and matches them with the accounts, the base currency and the id of the user', async () => { |
||||
|
const activities = [ |
||||
|
{ |
||||
|
currency: 'USD', |
||||
|
dataSource: 'YAHOO', |
||||
|
date: '2026-01-15', |
||||
|
fee: 1, |
||||
|
quantity: 10, |
||||
|
symbol: 'AAPL', |
||||
|
type: ActivityType.BUY, |
||||
|
unitPrice: 230 |
||||
|
} |
||||
|
]; |
||||
|
|
||||
|
jest |
||||
|
.spyOn(aiService, 'getActivityRowsFromFile') |
||||
|
.mockResolvedValue({ broker: 'Swissquote', rows }); |
||||
|
|
||||
|
jest |
||||
|
.spyOn(aiService, 'getActivitiesFromRows') |
||||
|
.mockResolvedValue({ activities, rowsWithoutMatch: [] }); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivitiesFromFile({ abortSignal, file, user }) |
||||
|
).resolves.toEqual({ |
||||
|
activities, |
||||
|
assetProfiles: [], |
||||
|
rowsWithoutMatch: [] |
||||
|
}); |
||||
|
|
||||
|
expect(aiService.getActivityRowsFromFile).toHaveBeenCalledWith({ |
||||
|
abortSignal, |
||||
|
file, |
||||
|
locale: 'de-CH' |
||||
|
}); |
||||
|
|
||||
|
expect(aiService.getActivitiesFromRows).toHaveBeenCalledWith({ |
||||
|
abortSignal, |
||||
|
rows, |
||||
|
accounts: user.accounts, |
||||
|
broker: 'Swissquote', |
||||
|
userCurrency: 'EUR', |
||||
|
userId: 'user-id' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('Gives a fee, an interest or a liability a custom asset profile with its description as name', async () => { |
||||
|
const fee = { |
||||
|
currency: 'CHF', |
||||
|
date: '2026-01-15', |
||||
|
fee: 5, |
||||
|
quantity: 0, |
||||
|
symbol: 'Custody fee', |
||||
|
type: ActivityType.FEE, |
||||
|
unitPrice: 0 |
||||
|
}; |
||||
|
|
||||
|
jest |
||||
|
.spyOn(aiService, 'getActivityRowsFromFile') |
||||
|
.mockResolvedValue({ broker: 'Swissquote', rows }); |
||||
|
|
||||
|
jest.spyOn(aiService, 'getActivitiesFromRows').mockResolvedValue({ |
||||
|
activities: [fee, { ...fee, date: '2026-02-15' }], |
||||
|
rowsWithoutMatch: [] |
||||
|
}); |
||||
|
|
||||
|
const { activities, assetProfiles } = |
||||
|
await aiService.getActivitiesFromFile({ file, user }); |
||||
|
|
||||
|
const [{ symbol }] = assetProfiles; |
||||
|
|
||||
|
expect(isValidCustomAssetProfileSymbol(symbol)).toBe(true); |
||||
|
|
||||
|
expect(assetProfiles).toEqual([ |
||||
|
{ |
||||
|
symbol, |
||||
|
currency: 'CHF', |
||||
|
dataSource: DataSource.MANUAL, |
||||
|
name: 'Custody fee' |
||||
|
} |
||||
|
]); |
||||
|
|
||||
|
expect(activities).toEqual([ |
||||
|
{ ...fee, symbol, dataSource: DataSource.MANUAL }, |
||||
|
{ ...fee, symbol, dataSource: DataSource.MANUAL, date: '2026-02-15' } |
||||
|
]); |
||||
|
}); |
||||
|
|
||||
|
it('Reads the file in the default locale if the user has no locale', async () => { |
||||
|
jest |
||||
|
.spyOn(aiService, 'getActivityRowsFromFile') |
||||
|
.mockResolvedValue({ broker: null, rows: [] }); |
||||
|
|
||||
|
jest |
||||
|
.spyOn(aiService, 'getActivitiesFromRows') |
||||
|
.mockResolvedValue({ activities: [], rowsWithoutMatch: [] }); |
||||
|
|
||||
|
await aiService.getActivitiesFromFile({ |
||||
|
file, |
||||
|
user: { |
||||
|
...user, |
||||
|
settings: { settings: { baseCurrency: 'EUR' } } |
||||
|
} as unknown as UserWithSettings |
||||
|
}); |
||||
|
|
||||
|
expect(aiService.getActivityRowsFromFile).toHaveBeenCalledWith( |
||||
|
expect.objectContaining({ locale: DEFAULT_LOCALE }) |
||||
|
); |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
describe('getActivitiesFromRows', () => { |
||||
|
const accounts: AccountToMatch[] = [ |
||||
|
{ currency: 'CHF', id: 'account-id', name: 'Brokerage' } |
||||
|
]; |
||||
|
|
||||
|
const userCurrency = 'EUR'; |
||||
|
const userId = 'user-id'; |
||||
|
|
||||
|
function mockRowMatches({ |
||||
|
accounts: accountMatches = [], |
||||
|
assetProfilesOfSearch, |
||||
|
assets: assetMatches = [] |
||||
|
}: Partial<z.infer<typeof ROW_MATCHES_OUTPUT>> & { |
||||
|
assetProfilesOfSearch?: { dataSource: string; symbol: string }[]; |
||||
|
}) { |
||||
|
jest.mocked(mcpService.searchAssetProfiles).mockResolvedValueOnce({ |
||||
|
content: [ |
||||
|
{ |
||||
|
text: JSON.stringify({ |
||||
|
assetProfiles: assetProfilesOfSearch ?? assetMatches |
||||
|
}), |
||||
|
type: 'text' |
||||
|
} |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
jest.mocked(generateText).mockImplementationOnce(async ({ tools }) => { |
||||
|
await tools['search-asset-profiles'].execute( |
||||
|
{ query: 'US0378331005' }, |
||||
|
undefined |
||||
|
); |
||||
|
|
||||
|
return { |
||||
|
output: { accounts: accountMatches, assets: assetMatches }, |
||||
|
steps: [{}, {}], |
||||
|
totalUsage: { totalTokens: 2000 } |
||||
|
} as unknown as Awaited<ReturnType<typeof generateText>>; |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
it('Gives an activity for each row whose asset matches an asset profile', async () => { |
||||
|
const row = createActivityRow({ currency: null }); |
||||
|
|
||||
|
mockRowMatches({ |
||||
|
accounts: [{ accountId: 'account-id', id: 0 }], |
||||
|
assets: [ |
||||
|
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [row] |
||||
|
}) |
||||
|
).resolves.toEqual({ |
||||
|
activities: [ |
||||
|
{ |
||||
|
accountId: 'account-id', |
||||
|
currency: 'USD', |
||||
|
dataSource: 'YAHOO', |
||||
|
date: row.date, |
||||
|
fee: row.fee, |
||||
|
quantity: row.quantity, |
||||
|
symbol: 'AAPL', |
||||
|
type: ActivityType.BUY, |
||||
|
unitPrice: row.unitPrice |
||||
|
} |
||||
|
], |
||||
|
rowsWithoutMatch: [] |
||||
|
}); |
||||
|
|
||||
|
expect(Logger.prototype.log).toHaveBeenCalledWith( |
||||
|
expect.stringContaining( |
||||
|
'Matched 1 of 1 assets and 1 of 1 account hints with 1 results of 1 searches' |
||||
|
) |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('Gives the currency of the row before the currency of the asset profile', async () => { |
||||
|
mockRowMatches({ |
||||
|
assets: [ |
||||
|
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
const { activities } = await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [createActivityRow({ accountName: null, currency: 'CHF' })] |
||||
|
}); |
||||
|
|
||||
|
expect(activities[0].currency).toBe('CHF'); |
||||
|
}); |
||||
|
|
||||
|
it('Gives the absolute values of a negative fee, quantity and unit price', async () => { |
||||
|
mockRowMatches({ |
||||
|
assets: [ |
||||
|
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
const { activities } = await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [ |
||||
|
createActivityRow({ |
||||
|
accountName: null, |
||||
|
fee: -1, |
||||
|
quantity: -10, |
||||
|
type: ActivityType.SELL, |
||||
|
unitPrice: -230 |
||||
|
}) |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
expect(activities[0]).toMatchObject({ |
||||
|
fee: 1, |
||||
|
quantity: 10, |
||||
|
unitPrice: 230 |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('Gives a row whose asset has no match as a row without a match', async () => { |
||||
|
const row = createActivityRow({ accountName: null }); |
||||
|
|
||||
|
mockRowMatches({ |
||||
|
assets: [{ currency: null, dataSource: null, id: 0, symbol: null }] |
||||
|
}); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [row] |
||||
|
}) |
||||
|
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
||||
|
}); |
||||
|
|
||||
|
it('Gives a row with a data source which is not valid as a row without a match', async () => { |
||||
|
const row = createActivityRow({ accountName: null }); |
||||
|
|
||||
|
mockRowMatches({ |
||||
|
assets: [ |
||||
|
{ currency: 'USD', dataSource: 'yahoo', id: 0, symbol: 'AAPL' } |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [row] |
||||
|
}) |
||||
|
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
||||
|
}); |
||||
|
|
||||
|
it('Gives a row whose asset profile no search gave as a row without a match and logs the left out match', async () => { |
||||
|
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
||||
|
|
||||
|
const row = createActivityRow({ accountName: null }); |
||||
|
|
||||
|
mockRowMatches({ |
||||
|
assetProfilesOfSearch: [{ dataSource: 'YAHOO', symbol: 'AAPL' }], |
||||
|
assets: [ |
||||
|
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL.XX' } |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [row] |
||||
|
}) |
||||
|
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
||||
|
|
||||
|
expect(Logger.prototype.log).toHaveBeenCalledWith( |
||||
|
expect.stringContaining( |
||||
|
'Matched 0 of 1 assets and 0 of 0 account hints with 1 results of 1 searches' |
||||
|
) |
||||
|
); |
||||
|
|
||||
|
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
||||
|
'Left out 1 asset matches which no search gave' |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('Matches a row without an account name by the broker of the file', async () => { |
||||
|
mockRowMatches({ |
||||
|
accounts: [{ accountId: 'account-id', id: 0 }], |
||||
|
assets: [ |
||||
|
{ currency: 'CHF', dataSource: 'YAHOO', id: 0, symbol: 'ROG.SW' } |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
const { activities } = await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
broker: 'Swissquote', |
||||
|
rows: [ |
||||
|
createActivityRow({ accountName: null }), |
||||
|
createActivityRow({ accountName: null, type: ActivityType.SELL }) |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
expect(activities.map(({ accountId }) => accountId)).toEqual([ |
||||
|
'account-id', |
||||
|
'account-id' |
||||
|
]); |
||||
|
|
||||
|
const { prompt } = jest.mocked(generateText).mock.calls[0][0]; |
||||
|
|
||||
|
const { accountHints } = JSON.parse(prompt as string) as { |
||||
|
accountHints: AccountHint[]; |
||||
|
}; |
||||
|
|
||||
|
expect(accountHints).toEqual([ |
||||
|
{ accountName: null, broker: 'Swissquote', id: 0 } |
||||
|
]); |
||||
|
}); |
||||
|
|
||||
|
it('Ignores an account id which is not the id of an account of the user', async () => { |
||||
|
mockRowMatches({ |
||||
|
accounts: [{ accountId: 'unknown-account-id', id: 0 }], |
||||
|
assets: [ |
||||
|
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
const { activities } = await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [createActivityRow()] |
||||
|
}); |
||||
|
|
||||
|
expect(activities[0].accountId).toBeUndefined(); |
||||
|
}); |
||||
|
|
||||
|
it('Gives a fee, an interest or a liability its description as symbol without a model call', async () => { |
||||
|
const rowsOfNonInvestments = [ |
||||
|
ActivityType.FEE, |
||||
|
ActivityType.INTEREST, |
||||
|
ActivityType.LIABILITY |
||||
|
].map((type) => { |
||||
|
return createActivityRow({ |
||||
|
type, |
||||
|
accountName: null, |
||||
|
currency: null, |
||||
|
isin: null, |
||||
|
name: `Description of ${type}`, |
||||
|
symbol: null |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
const { activities, rowsWithoutMatch } = |
||||
|
await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: rowsOfNonInvestments |
||||
|
}); |
||||
|
|
||||
|
expect(generateText).not.toHaveBeenCalled(); |
||||
|
|
||||
|
expect( |
||||
|
activities.map(({ currency, dataSource, symbol, type }) => { |
||||
|
return { currency, dataSource, symbol, type }; |
||||
|
}) |
||||
|
).toEqual( |
||||
|
rowsOfNonInvestments.map(({ name, type }) => { |
||||
|
return { |
||||
|
type, |
||||
|
currency: userCurrency, |
||||
|
dataSource: undefined, |
||||
|
symbol: name |
||||
|
}; |
||||
|
}) |
||||
|
); |
||||
|
|
||||
|
expect(rowsWithoutMatch).toEqual([]); |
||||
|
}); |
||||
|
|
||||
|
it('Gives a fee without a currency the currency of its account', async () => { |
||||
|
mockRowMatches({ |
||||
|
accounts: [{ accountId: 'account-id', id: 0 }] |
||||
|
}); |
||||
|
|
||||
|
const { activities } = await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [ |
||||
|
createActivityRow({ |
||||
|
currency: null, |
||||
|
isin: null, |
||||
|
name: 'Custody fee', |
||||
|
symbol: null, |
||||
|
type: ActivityType.FEE |
||||
|
}) |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
expect(activities[0]).toMatchObject({ |
||||
|
accountId: 'account-id', |
||||
|
currency: 'CHF' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('Gives a fee without a description as a row without a match', async () => { |
||||
|
const row = createActivityRow({ |
||||
|
accountName: null, |
||||
|
isin: null, |
||||
|
name: null, |
||||
|
symbol: null, |
||||
|
type: ActivityType.FEE |
||||
|
}); |
||||
|
|
||||
|
await expect( |
||||
|
aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [row] |
||||
|
}) |
||||
|
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
||||
|
}); |
||||
|
|
||||
|
it('Passes each asset and each account hint one time and no account to the model', async () => { |
||||
|
mockRowMatches({ |
||||
|
assets: [ |
||||
|
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
const { activities } = await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
broker: 'Swissquote', |
||||
|
rows: [ |
||||
|
createActivityRow(), |
||||
|
createActivityRow({ type: ActivityType.SELL }), |
||||
|
createActivityRow({ |
||||
|
accountName: 'Depot 1', |
||||
|
type: ActivityType.DIVIDEND |
||||
|
}) |
||||
|
] |
||||
|
}); |
||||
|
|
||||
|
expect(activities).toHaveLength(3); |
||||
|
|
||||
|
const { prompt } = jest.mocked(generateText).mock.calls[0][0]; |
||||
|
|
||||
|
expect(JSON.parse(prompt as string)).toEqual({ |
||||
|
accountHints: [ |
||||
|
{ accountName: 'Brokerage', broker: 'Swissquote', id: 0 }, |
||||
|
{ accountName: 'Depot 1', broker: 'Swissquote', id: 1 } |
||||
|
], |
||||
|
assets: [ |
||||
|
{ |
||||
|
currency: 'USD', |
||||
|
id: 0, |
||||
|
isin: 'US0378331005', |
||||
|
name: 'Apple Inc.', |
||||
|
symbol: 'AAPL' |
||||
|
} |
||||
|
] |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('Searches the asset profiles with the tool of the server of the Model Context Protocol', async () => { |
||||
|
mockRowMatches({}); |
||||
|
|
||||
|
await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [createActivityRow()] |
||||
|
}); |
||||
|
|
||||
|
const { tools } = jest.mocked(generateText).mock.calls[0][0] as { |
||||
|
tools: ToolSet; |
||||
|
}; |
||||
|
|
||||
|
await expect( |
||||
|
tools['search-asset-profiles'].execute( |
||||
|
{ query: 'US0378331005' }, |
||||
|
undefined |
||||
|
) |
||||
|
).resolves.toBe('{ "assetProfiles": [] }'); |
||||
|
|
||||
|
expect(mcpService.searchAssetProfiles).toHaveBeenCalledWith({ |
||||
|
userId, |
||||
|
query: 'US0378331005' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('Gets the accounts with the tool of the server of the Model Context Protocol without values', async () => { |
||||
|
mockRowMatches({}); |
||||
|
|
||||
|
await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [createActivityRow()] |
||||
|
}); |
||||
|
|
||||
|
const { tools } = jest.mocked(generateText).mock.calls[0][0] as { |
||||
|
tools: ToolSet; |
||||
|
}; |
||||
|
|
||||
|
await expect(tools['get-accounts'].execute({}, undefined)).resolves.toBe( |
||||
|
'## Accounts' |
||||
|
); |
||||
|
|
||||
|
expect(mcpService.getAccounts).toHaveBeenCalledWith({ |
||||
|
userId, |
||||
|
withValues: false |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('Restricts the routing to endpoints with zero data retention, limits the number of steps, requires a tool call in the first step and allows no tool call in the last step', async () => { |
||||
|
mockRowMatches({}); |
||||
|
|
||||
|
await aiService.getActivitiesFromRows({ |
||||
|
abortSignal, |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [createActivityRow()] |
||||
|
}); |
||||
|
|
||||
|
expect(chat).toHaveBeenCalledWith('agent-model', { |
||||
|
provider: { require_parameters: true, zdr: true }, |
||||
|
usage: { include: true } |
||||
|
}); |
||||
|
|
||||
|
expect(isStepCount).toHaveBeenCalled(); |
||||
|
|
||||
|
expect(generateText).toHaveBeenCalledWith( |
||||
|
expect.objectContaining({ |
||||
|
abortSignal, |
||||
|
model, |
||||
|
output: { schema: ROW_MATCHES_OUTPUT } |
||||
|
}) |
||||
|
); |
||||
|
|
||||
|
const { prepareStep, stopWhen } = jest.mocked(generateText).mock |
||||
|
.calls[0][0] as { |
||||
|
prepareStep: (options: { stepNumber: number }) => unknown; |
||||
|
stopWhen: unknown; |
||||
|
}; |
||||
|
|
||||
|
expect(stopWhen).toBe(jest.mocked(isStepCount).mock.results[0].value); |
||||
|
|
||||
|
const [stepCount] = jest.mocked(isStepCount).mock.calls[0]; |
||||
|
|
||||
|
expect(prepareStep({ stepNumber: 0 })).toEqual({ |
||||
|
toolChoice: 'required' |
||||
|
}); |
||||
|
expect(prepareStep({ stepNumber: stepCount - 2 })).toBeUndefined(); |
||||
|
expect(prepareStep({ stepNumber: stepCount - 1 })).toEqual({ |
||||
|
toolChoice: 'none' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('Falls back to the general model if no agent model is set', async () => { |
||||
|
delete properties[PROPERTY_OPENROUTER_MODEL_AGENT]; |
||||
|
|
||||
|
mockRowMatches({}); |
||||
|
|
||||
|
await aiService.getActivitiesFromRows({ |
||||
|
accounts, |
||||
|
userCurrency, |
||||
|
userId, |
||||
|
rows: [createActivityRow()] |
||||
|
}); |
||||
|
|
||||
|
expect(chat).toHaveBeenCalledWith('general-model', expect.any(Object)); |
||||
|
}); |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,21 @@ |
|||||
|
import { Type as ActivityType } from '@prisma/client'; |
||||
|
|
||||
|
import { ActivityRow } from './types/activity-row.type'; |
||||
|
|
||||
|
export function createActivityRow( |
||||
|
overrides: Partial<ActivityRow> = {} |
||||
|
): ActivityRow { |
||||
|
return { |
||||
|
accountName: 'Brokerage', |
||||
|
currency: 'USD', |
||||
|
date: '2026-01-15', |
||||
|
fee: 1, |
||||
|
isin: 'US0378331005', |
||||
|
name: 'Apple Inc.', |
||||
|
quantity: 10, |
||||
|
symbol: 'AAPL', |
||||
|
type: ActivityType.BUY, |
||||
|
unitPrice: 230, |
||||
|
...overrides |
||||
|
}; |
||||
|
} |
||||
@ -0,0 +1,6 @@ |
|||||
|
import { ActivityRow } from './activity-row.type'; |
||||
|
|
||||
|
export type AccountHint = Pick<ActivityRow, 'accountName'> & { |
||||
|
broker: string | null; |
||||
|
id: number; |
||||
|
}; |
||||
@ -0,0 +1,3 @@ |
|||||
|
import { Account } from '@prisma/client'; |
||||
|
|
||||
|
export type AccountToMatch = Pick<Account, 'currency' | 'id' | 'name'>; |
||||
@ -0,0 +1,5 @@ |
|||||
|
import { z } from 'zod'; |
||||
|
|
||||
|
import { ACTIVITY_ROWS_OUTPUT } from '../ai.schemas'; |
||||
|
|
||||
|
export type ActivityRow = z.infer<typeof ACTIVITY_ROWS_OUTPUT>['rows'][number]; |
||||
@ -0,0 +1,6 @@ |
|||||
|
import { ActivityRow } from './activity-row.type'; |
||||
|
|
||||
|
export type AssetToMatch = Pick< |
||||
|
ActivityRow, |
||||
|
'currency' | 'isin' | 'name' | 'symbol' |
||||
|
> & { id: number }; |
||||
@ -0,0 +1,18 @@ |
|||||
|
import { Type as ActivityType } from '@ghostfolio/prisma/enums'; |
||||
|
|
||||
|
/** |
||||
|
* An activity as Ghostfolio AI reads it from a file, with the asset and the |
||||
|
* account as printed |
||||
|
*/ |
||||
|
export interface AiActivityRow { |
||||
|
accountName?: string | null; |
||||
|
currency?: string | null; |
||||
|
date: string; |
||||
|
fee: number; |
||||
|
isin?: string | null; |
||||
|
name?: string | null; |
||||
|
quantity: number; |
||||
|
symbol?: string | null; |
||||
|
type: ActivityType; |
||||
|
unitPrice: number; |
||||
|
} |
||||
@ -0,0 +1,12 @@ |
|||||
|
import type { |
||||
|
CreateAssetProfileWithMarketDataDto, |
||||
|
CreateOrderDto |
||||
|
} from '@ghostfolio/common/dtos'; |
||||
|
|
||||
|
import { AiActivityRow } from '../ai-activity-row.interface'; |
||||
|
|
||||
|
export interface AiActivitiesResponse { |
||||
|
activities: CreateOrderDto[]; |
||||
|
assetProfiles: CreateAssetProfileWithMarketDataDto[]; |
||||
|
rowsWithoutMatch: AiActivityRow[]; |
||||
|
} |
||||
Loading…
Reference in new issue