mirror of https://github.com/ghostfolio/ghostfolio
committed by
GitHub
28 changed files with 2796 additions and 35 deletions
@ -0,0 +1,279 @@ |
|||
import { HAS_PERMISSION_KEY } from '@ghostfolio/api/decorators/has-permission.decorator'; |
|||
import { CallerFacingError } from '@ghostfolio/api/errors/caller-facing.error'; |
|||
import { CustomThrottlerGuard } from '@ghostfolio/api/guards/custom-throttler.guard'; |
|||
import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'; |
|||
import { ApiService } from '@ghostfolio/api/services/api/api.service'; |
|||
import { |
|||
AI_ACTIVITIES_TIMEOUT, |
|||
AI_MAX_ACTIVITIES, |
|||
THROTTLE_AI_ACTIVITIES_LIMIT, |
|||
THROTTLE_AI_ACTIVITIES_TTL |
|||
} from '@ghostfolio/common/config'; |
|||
import { permissions } from '@ghostfolio/common/permissions'; |
|||
import type { RequestWithUser } from '@ghostfolio/common/types'; |
|||
|
|||
import { HttpException, Logger } from '@nestjs/common'; |
|||
import { GUARDS_METADATA } from '@nestjs/common/constants'; |
|||
import { |
|||
THROTTLER_LIMIT, |
|||
THROTTLER_TRACKER, |
|||
THROTTLER_TTL |
|||
} from '@nestjs/throttler/dist/throttler.constants'; |
|||
import type { Response } from 'express'; |
|||
import { StatusCodes } from 'http-status-codes'; |
|||
import { EventEmitter } from 'node:events'; |
|||
|
|||
import { AiController } from './ai.controller'; |
|||
import { AiService } from './ai.service'; |
|||
|
|||
// The packages are ECMAScript modules, which Jest cannot load
|
|||
jest.mock('@openrouter/ai-sdk-provider', () => { |
|||
return {}; |
|||
}); |
|||
jest.mock('ai', () => { |
|||
return {}; |
|||
}); |
|||
|
|||
describe('AiController', () => { |
|||
const pdfFile = { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
originalname: 'statement.pdf' |
|||
}; |
|||
|
|||
const user = { id: 'user-id' }; |
|||
|
|||
let aiController: AiController; |
|||
let aiService: AiService; |
|||
|
|||
function createCsvFile(dataRowCount: number) { |
|||
return { |
|||
buffer: Buffer.from( |
|||
[ |
|||
'Date,Symbol', |
|||
...Array.from({ length: dataRowCount }, () => { |
|||
return '2026-01-15,AAPL'; |
|||
}) |
|||
].join('\n') |
|||
), |
|||
originalname: 'statement.csv' |
|||
}; |
|||
} |
|||
|
|||
function createResponse() { |
|||
return Object.assign(new EventEmitter(), { |
|||
writableFinished: false |
|||
}) as unknown as Response; |
|||
} |
|||
|
|||
function getMetadataOfMethod<T>(metadataKey: string) { |
|||
return Reflect.getMetadata( |
|||
metadataKey, |
|||
AiController.prototype.getActivitiesFromFile |
|||
) as T; |
|||
} |
|||
|
|||
beforeEach(() => { |
|||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
|||
|
|||
aiService = { |
|||
getActivitiesFromFile: jest.fn().mockResolvedValue({ |
|||
activities: [], |
|||
assetProfiles: [], |
|||
rowsWithoutMatch: [] |
|||
}) |
|||
} as unknown as AiService; |
|||
|
|||
aiController = new AiController( |
|||
aiService, |
|||
{} as ApiService, |
|||
{ |
|||
user |
|||
} as unknown as RequestWithUser |
|||
); |
|||
}); |
|||
|
|||
describe('getActivitiesFromFile', () => { |
|||
it('Requires the permission and limits the requests of each user per week', () => { |
|||
expect(getMetadataOfMethod<string>(HAS_PERMISSION_KEY)).toBe( |
|||
permissions.importActivitiesWithAi |
|||
); |
|||
|
|||
const guards = getMetadataOfMethod<unknown[]>(GUARDS_METADATA); |
|||
|
|||
expect(guards.slice(1)).toEqual([ |
|||
HasPermissionGuard, |
|||
CustomThrottlerGuard |
|||
]); |
|||
|
|||
expect(getMetadataOfMethod<number>(`${THROTTLER_LIMIT}default`)).toBe( |
|||
THROTTLE_AI_ACTIVITIES_LIMIT |
|||
); |
|||
|
|||
expect(getMetadataOfMethod<number>(`${THROTTLER_TTL}default`)).toBe( |
|||
THROTTLE_AI_ACTIVITIES_TTL |
|||
); |
|||
|
|||
expect( |
|||
getMetadataOfMethod<(request: unknown) => string>( |
|||
`${THROTTLER_TRACKER}default` |
|||
)({ ip: '127.0.0.1', user }) |
|||
).toBe(user.id); |
|||
}); |
|||
|
|||
it('Refuses a request without a file or with a file of another type', async () => { |
|||
for (const file of [ |
|||
undefined, |
|||
{ buffer: Buffer.from('Date,Symbol'), originalname: 'statement.txt' } |
|||
]) { |
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), file) |
|||
).rejects.toThrow(HttpException); |
|||
} |
|||
|
|||
expect(aiService.getActivitiesFromFile).not.toHaveBeenCalled(); |
|||
}); |
|||
|
|||
it(`Refuses a CSV file with more than ${AI_MAX_ACTIVITIES} rows before a model call`, async () => { |
|||
await expect( |
|||
aiController.getActivitiesFromFile( |
|||
createResponse(), |
|||
createCsvFile(AI_MAX_ACTIVITIES + 1) |
|||
) |
|||
).rejects.toThrow(HttpException); |
|||
|
|||
expect(aiService.getActivitiesFromFile).not.toHaveBeenCalled(); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
|||
`Rejected a CSV file with ${AI_MAX_ACTIVITIES + 1} rows` |
|||
); |
|||
}); |
|||
|
|||
it(`Accepts a CSV file with ${AI_MAX_ACTIVITIES} rows`, async () => { |
|||
const file = createCsvFile(AI_MAX_ACTIVITIES); |
|||
|
|||
await aiController.getActivitiesFromFile(createResponse(), file); |
|||
|
|||
expect(aiService.getActivitiesFromFile).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
file: { |
|||
buffer: file.buffer, |
|||
mediaType: 'text/csv', |
|||
name: 'statement.csv' |
|||
} |
|||
}) |
|||
); |
|||
}); |
|||
|
|||
it('Passes the file with its media type and its name and the user of the request', async () => { |
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
|||
).resolves.toEqual({ |
|||
activities: [], |
|||
assetProfiles: [], |
|||
rowsWithoutMatch: [] |
|||
}); |
|||
|
|||
expect(aiService.getActivitiesFromFile).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
user, |
|||
file: { |
|||
buffer: pdfFile.buffer, |
|||
mediaType: 'application/pdf', |
|||
name: 'statement.pdf' |
|||
} |
|||
}) |
|||
); |
|||
|
|||
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
|||
.mock.calls; |
|||
|
|||
expect(abortSignal).toBeInstanceOf(AbortSignal); |
|||
}); |
|||
|
|||
it('Stops the models if the client closes the request before the response', async () => { |
|||
const response = createResponse(); |
|||
|
|||
await aiController.getActivitiesFromFile(response, pdfFile); |
|||
|
|||
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
|||
.mock.calls; |
|||
|
|||
expect(abortSignal.aborted).toBe(false); |
|||
|
|||
response.emit('close'); |
|||
|
|||
expect(abortSignal.aborted).toBe(true); |
|||
}); |
|||
|
|||
it('Does not stop the models when the response is finished', async () => { |
|||
const response = createResponse(); |
|||
|
|||
await aiController.getActivitiesFromFile(response, pdfFile); |
|||
|
|||
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
|||
.mock.calls; |
|||
|
|||
Object.assign(response, { writableFinished: true }); |
|||
response.emit('close'); |
|||
|
|||
expect(abortSignal.aborted).toBe(false); |
|||
}); |
|||
|
|||
it('Gives the status 400 with the message of an error for the caller', async () => { |
|||
jest |
|||
.mocked(aiService.getActivitiesFromFile) |
|||
.mockRejectedValueOnce( |
|||
new CallerFacingError('The file has too many activities') |
|||
); |
|||
|
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
|||
).rejects.toMatchObject({ |
|||
response: { message: ['The file has too many activities'] }, |
|||
status: StatusCodes.BAD_REQUEST |
|||
}); |
|||
}); |
|||
|
|||
it('Gives the status 400 with a message for the caller at the timeout', async () => { |
|||
const timeoutSignal = AbortSignal.abort( |
|||
new DOMException( |
|||
'The operation was aborted due to timeout', |
|||
'TimeoutError' |
|||
) |
|||
); |
|||
|
|||
jest.spyOn(AbortSignal, 'timeout').mockReturnValueOnce(timeoutSignal); |
|||
|
|||
jest |
|||
.mocked(aiService.getActivitiesFromFile) |
|||
.mockRejectedValueOnce(timeoutSignal.reason); |
|||
|
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
|||
).rejects.toMatchObject({ |
|||
response: { |
|||
message: [ |
|||
'Ghostfolio AI needs too much time for the file. Split it into smaller files.' |
|||
] |
|||
}, |
|||
status: StatusCodes.BAD_REQUEST |
|||
}); |
|||
|
|||
expect(AbortSignal.timeout).toHaveBeenCalledWith(AI_ACTIVITIES_TIMEOUT); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
|||
`Stopped reading a file of the type application/pdf at the timeout of ${AI_ACTIVITIES_TIMEOUT} ms` |
|||
); |
|||
}); |
|||
|
|||
it('Passes on every other error', async () => { |
|||
const error = new Error('Model error'); |
|||
|
|||
jest.mocked(aiService.getActivitiesFromFile).mockRejectedValueOnce(error); |
|||
|
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
|||
).rejects.toBe(error); |
|||
}); |
|||
}); |
|||
}); |
|||
@ -0,0 +1,99 @@ |
|||
import { |
|||
getDataRowCountOfCsvFile, |
|||
getMediaTypeOfActivityFile |
|||
} from './ai.helper'; |
|||
|
|||
describe('getDataRowCountOfCsvFile', () => { |
|||
it('Gives the number of the lines which are not empty, without the header', () => { |
|||
expect( |
|||
getDataRowCountOfCsvFile( |
|||
Buffer.from('Date,Symbol\n2026-01-15,AAPL\n\n2026-01-16,MSFT\n') |
|||
) |
|||
).toBe(2); |
|||
}); |
|||
|
|||
it('Counts each line of a value with a line break in quotes', () => { |
|||
expect( |
|||
getDataRowCountOfCsvFile( |
|||
Buffer.from('Date,Comment\r\n2026-01-15,"First\r\nsecond"\r\n') |
|||
) |
|||
).toBe(2); |
|||
}); |
|||
|
|||
it('Counts each line after a quote which is not closed', () => { |
|||
expect( |
|||
getDataRowCountOfCsvFile( |
|||
Buffer.from( |
|||
[ |
|||
'Date,Symbol,Quantity', |
|||
'2026-01-15,"AAPL,1', |
|||
...Array.from({ length: 200 }, () => { |
|||
return '2026-01-16,MSFT,1'; |
|||
}) |
|||
].join('\n') |
|||
) |
|||
) |
|||
).toBe(201); |
|||
}); |
|||
|
|||
it('Gives 0 for a file with a header only or without content', () => { |
|||
expect(getDataRowCountOfCsvFile(Buffer.from('Date,Symbol\n'))).toBe(0); |
|||
expect(getDataRowCountOfCsvFile(Buffer.from(''))).toBe(0); |
|||
}); |
|||
}); |
|||
|
|||
describe('getMediaTypeOfActivityFile', () => { |
|||
it('Identifies a JPEG, a PDF and a PNG file by the first bytes', () => { |
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from([0xff, 0xd8, 0xff, 0xe0]), |
|||
fileName: 'statement' |
|||
}) |
|||
).toBe('image/jpeg'); |
|||
|
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
fileName: 'statement.csv' |
|||
}) |
|||
).toBe('application/pdf'); |
|||
|
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), |
|||
fileName: 'statement.jpg' |
|||
}) |
|||
).toBe('image/png'); |
|||
}); |
|||
|
|||
it('Identifies a CSV file by the extension and the content in UTF-8', () => { |
|||
for (const fileName of ['statement.csv', 'STATEMENT.CSV']) { |
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
fileName, |
|||
buffer: Buffer.from('Datum;Stück\n15.01.2026;10\n') |
|||
}) |
|||
).toBe('text/csv'); |
|||
} |
|||
}); |
|||
|
|||
it('Refuses a CSV file which is not in UTF-8', () => { |
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from([0x53, 0x74, 0xfc, 0x63, 0x6b]), |
|||
fileName: 'statement.csv' |
|||
}) |
|||
).toBeUndefined(); |
|||
}); |
|||
|
|||
it('Refuses any other file', () => { |
|||
for (const { buffer, fileName } of [ |
|||
{ buffer: Buffer.from('Date,Symbol'), fileName: 'statement.txt' }, |
|||
{ buffer: Buffer.from('<html></html>'), fileName: 'statement.pdf' }, |
|||
{ buffer: Buffer.from([0xff, 0xd8]), fileName: 'statement.jpg' }, |
|||
{ buffer: Buffer.from(''), fileName: 'statement.png' } |
|||
]) { |
|||
expect(getMediaTypeOfActivityFile({ buffer, fileName })).toBeUndefined(); |
|||
} |
|||
}); |
|||
}); |
|||
@ -0,0 +1,57 @@ |
|||
import { isUtf8 } from 'node:buffer'; |
|||
|
|||
const JPEG_SIGNATURE = [0xff, 0xd8, 0xff]; |
|||
const PDF_SIGNATURE = [0x25, 0x50, 0x44, 0x46, 0x2d]; |
|||
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; |
|||
|
|||
/** |
|||
* Gives the number of the lines of a CSV file which are not empty, without |
|||
* the first line. A line before the header and each line of a value with a |
|||
* line break are counted as data rows, hence the number is never too low, |
|||
* also for a file with a quote which is not closed. |
|||
*/ |
|||
export function getDataRowCountOfCsvFile(buffer: Buffer) { |
|||
const lineCount = buffer |
|||
.toString('utf8') |
|||
.split(/\r\n|\r|\n/) |
|||
.filter((line) => { |
|||
return line.trim().length > 0; |
|||
}).length; |
|||
|
|||
return Math.max(lineCount - 1, 0); |
|||
} |
|||
|
|||
/** |
|||
* Gives the media type of a file which Ghostfolio AI reads. A JPEG, a PDF or |
|||
* a PNG file is identified by its first bytes, a CSV file by its extension |
|||
* and by its content in UTF-8, as the media type which a client sends for a |
|||
* CSV file varies. |
|||
*/ |
|||
export function getMediaTypeOfActivityFile({ |
|||
buffer, |
|||
fileName |
|||
}: { |
|||
buffer: Buffer; |
|||
fileName: string; |
|||
}) { |
|||
if (startsWith(buffer, JPEG_SIGNATURE)) { |
|||
return 'image/jpeg'; |
|||
} else if (startsWith(buffer, PDF_SIGNATURE)) { |
|||
return 'application/pdf'; |
|||
} else if (startsWith(buffer, PNG_SIGNATURE)) { |
|||
return 'image/png'; |
|||
} else if (fileName.toLowerCase().endsWith('.csv') && isUtf8(buffer)) { |
|||
return 'text/csv'; |
|||
} |
|||
|
|||
return undefined; |
|||
} |
|||
|
|||
function startsWith(buffer: Buffer, signature: number[]) { |
|||
return ( |
|||
buffer.length >= signature.length && |
|||
signature.every((byte, index) => { |
|||
return buffer[index] === byte; |
|||
}) |
|||
); |
|||
} |
|||
@ -0,0 +1,101 @@ |
|||
import { |
|||
AI_MAX_ACTIVITIES, |
|||
SEARCH_QUERY_MAXIMUM_LENGTH |
|||
} from '@ghostfolio/common/config'; |
|||
|
|||
import { isObject } from 'lodash-es'; |
|||
import { z } from 'zod'; |
|||
|
|||
import { ACTIVITY_ROWS_OUTPUT, ROW_MATCHES_OUTPUT } from './ai.schemas'; |
|||
import { createActivityRow } from './ai.test-utils'; |
|||
import { ActivityRow } from './types/activity-row.type'; |
|||
|
|||
describe('ACTIVITY_ROWS_OUTPUT', () => { |
|||
it('Accepts a row with null for the values which the file does not show', () => { |
|||
expect( |
|||
ACTIVITY_ROWS_OUTPUT.safeParse({ |
|||
broker: null, |
|||
rows: [ |
|||
createActivityRow({ |
|||
accountName: null, |
|||
currency: null, |
|||
isin: null, |
|||
symbol: null |
|||
}) |
|||
] |
|||
}).success |
|||
).toBe(true); |
|||
}); |
|||
|
|||
it('Refuses a row with a value which is left out instead of null', () => { |
|||
const row: Partial<ActivityRow> = createActivityRow(); |
|||
|
|||
delete row.isin; |
|||
|
|||
expect( |
|||
ACTIVITY_ROWS_OUTPUT.safeParse({ broker: null, rows: [row] }).success |
|||
).toBe(false); |
|||
}); |
|||
|
|||
it('Accepts a currency, a date, a number, a printed text and a number of rows which the join checks, so that one value does not refuse all rows', () => { |
|||
expect( |
|||
ACTIVITY_ROWS_OUTPUT.safeParse({ |
|||
broker: null, |
|||
rows: [ |
|||
createActivityRow({ |
|||
currency: 'GBX', |
|||
date: '15.01.2026', |
|||
name: 'A'.repeat(SEARCH_QUERY_MAXIMUM_LENGTH + 1) |
|||
}), |
|||
createActivityRow({ fee: -1, quantity: -1, unitPrice: -1 }), |
|||
...Array.from({ length: AI_MAX_ACTIVITIES }, () => { |
|||
return createActivityRow(); |
|||
}) |
|||
] |
|||
}).success |
|||
).toBe(true); |
|||
}); |
|||
|
|||
it('Trims a printed text', () => { |
|||
const { data } = ACTIVITY_ROWS_OUTPUT.safeParse({ |
|||
broker: ' Swissquote ', |
|||
rows: [createActivityRow({ name: ' Apple Inc. ' })] |
|||
}); |
|||
|
|||
expect(data.broker).toBe('Swissquote'); |
|||
expect(data.rows[0].name).toBe('Apple Inc.'); |
|||
}); |
|||
}); |
|||
|
|||
describe.each([ |
|||
['ACTIVITY_ROWS_OUTPUT', ACTIVITY_ROWS_OUTPUT], |
|||
['ROW_MATCHES_OUTPUT', ROW_MATCHES_OUTPUT] |
|||
])('%s', (_name, schema) => { |
|||
it('Gives a JSON schema which the strict mode of the structured outputs supports', () => { |
|||
// The options of the conversion in the AI SDK
|
|||
const jsonSchema = z.toJSONSchema(schema, { |
|||
io: 'input', |
|||
target: 'draft-7' |
|||
}); |
|||
|
|||
const objectJsonSchemas: { properties: object; required: string[] }[] = []; |
|||
|
|||
JSON.stringify(jsonSchema, (_key, value: unknown) => { |
|||
if (isObject(value) && 'properties' in value) { |
|||
objectJsonSchemas.push( |
|||
value as { properties: object; required: string[] } |
|||
); |
|||
} |
|||
|
|||
return value; |
|||
}); |
|||
|
|||
expect(objectJsonSchemas.length).toBeGreaterThan(1); |
|||
|
|||
for (const { properties, required } of objectJsonSchemas) { |
|||
expect([...required].sort()).toEqual(Object.keys(properties).sort()); |
|||
} |
|||
|
|||
expect(JSON.stringify(jsonSchema)).not.toMatch(/"(max|min)(imum|Length)"/); |
|||
}); |
|||
}); |
|||
@ -0,0 +1,87 @@ |
|||
import { AI_MAX_ACTIVITIES } from '@ghostfolio/common/config'; |
|||
|
|||
import { Type as ActivityType } from '@prisma/client'; |
|||
import { z } from 'zod'; |
|||
|
|||
// The strict mode of the structured outputs requires every property, hence a
|
|||
// value which the file does not show is null instead of being left out. The
|
|||
// values are checked when the rows and the matches are joined to the
|
|||
// activities to import, so that a wrong value or a wrong match skips its rows
|
|||
// only
|
|||
const PRINTED_TEXT = z.string().trim().nullable(); |
|||
|
|||
export const ACTIVITY_ROWS_OUTPUT = z.object({ |
|||
broker: PRINTED_TEXT.describe( |
|||
'The name of the broker or of the bank which issued the file' |
|||
), |
|||
rows: z |
|||
.array( |
|||
z.object({ |
|||
accountName: PRINTED_TEXT.describe( |
|||
'The name or the number of the account, as printed' |
|||
), |
|||
currency: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The currency of the fee and of the unit price, as an ISO 4217 code in upper case' |
|||
), |
|||
date: z |
|||
.string() |
|||
.describe('The date of the activity, as an ISO 8601 date'), |
|||
fee: z.number().describe('The fee of the activity'), |
|||
isin: PRINTED_TEXT.describe('The ISIN of the asset, as printed'), |
|||
name: PRINTED_TEXT.describe( |
|||
'The name of the asset, or the description of a fee, an interest or a liability, as printed' |
|||
), |
|||
quantity: z.number().describe('The quantity of the activity'), |
|||
symbol: PRINTED_TEXT.describe( |
|||
'The ticker symbol of the asset, as printed, not a national number such as a valor, a WKN or a CUSIP' |
|||
), |
|||
type: z.enum(ActivityType).describe('The type of the activity'), |
|||
unitPrice: z.number().describe('The unit price of the activity') |
|||
}) |
|||
) |
|||
.describe(`The activities of the file, at most ${AI_MAX_ACTIVITIES}`) |
|||
}); |
|||
|
|||
export const ROW_MATCHES_OUTPUT = z.object({ |
|||
accounts: z |
|||
.array( |
|||
z.object({ |
|||
accountId: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The id of the account of the user which matches, as given by get-accounts, or null if no account matches' |
|||
), |
|||
id: z.number().describe('The id of the account hint, as given') |
|||
}) |
|||
) |
|||
.describe('One entry for each account hint of the file'), |
|||
assets: z |
|||
.array( |
|||
z.object({ |
|||
currency: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The currency of the asset profile, as given by search-asset-profiles' |
|||
), |
|||
dataSource: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The data source of the asset profile, as given by search-asset-profiles' |
|||
), |
|||
id: z.number().describe('The id of the asset, as given'), |
|||
symbol: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The symbol of the asset profile, as given by search-asset-profiles, or null if no asset profile matches' |
|||
) |
|||
}) |
|||
) |
|||
.describe('One entry for each asset of the file') |
|||
}); |
|||
@ -0,0 +1,955 @@ |
|||
import { McpService } from '@ghostfolio/api/app/endpoints/mcp/mcp.service'; |
|||
import { CallerFacingError } from '@ghostfolio/api/errors/caller-facing.error'; |
|||
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; |
|||
import { PortfolioTableService } from '@ghostfolio/api/services/portfolio-table/portfolio-table.service'; |
|||
import { PropertyService } from '@ghostfolio/api/services/property/property.service'; |
|||
import { |
|||
DEFAULT_LOCALE, |
|||
PROPERTY_API_KEY_OPENROUTER, |
|||
PROPERTY_OPENROUTER_MODEL, |
|||
PROPERTY_OPENROUTER_MODEL_AGENT, |
|||
PROPERTY_OPENROUTER_MODEL_VISION |
|||
} from '@ghostfolio/common/config'; |
|||
import { isValidCustomAssetProfileSymbol } from '@ghostfolio/common/helper'; |
|||
import type { UserWithSettings } from '@ghostfolio/common/types'; |
|||
|
|||
import { Logger } from '@nestjs/common'; |
|||
import { createOpenRouter } from '@openrouter/ai-sdk-provider'; |
|||
import { DataSource, Type as ActivityType } from '@prisma/client'; |
|||
import { generateText, isStepCount, NoObjectGeneratedError, ToolSet } from 'ai'; |
|||
import { z } from 'zod'; |
|||
|
|||
import { ACTIVITY_ROWS_OUTPUT, ROW_MATCHES_OUTPUT } from './ai.schemas'; |
|||
import { AiService } from './ai.service'; |
|||
import { createActivityRow } from './ai.test-utils'; |
|||
import { AccountHint } from './types/account-hint.type'; |
|||
import { AccountToMatch } from './types/account-to-match.type'; |
|||
import { ActivityRow } from './types/activity-row.type'; |
|||
|
|||
// The packages are ECMAScript modules, which Jest cannot load
|
|||
jest.mock('@openrouter/ai-sdk-provider', () => { |
|||
return { createOpenRouter: jest.fn() }; |
|||
}); |
|||
jest.mock('ai', () => { |
|||
return { |
|||
generateText: jest.fn(), |
|||
isStepCount: jest.fn((stepCount: number) => { |
|||
return { stepCount }; |
|||
}), |
|||
NoObjectGeneratedError: class MockNoObjectGeneratedError extends Error { |
|||
public readonly finishReason: string; |
|||
|
|||
public constructor({ finishReason }: { finishReason: string }) { |
|||
super('No object generated'); |
|||
|
|||
this.finishReason = finishReason; |
|||
} |
|||
|
|||
public static isInstance(error: unknown) { |
|||
return error instanceof MockNoObjectGeneratedError; |
|||
} |
|||
}, |
|||
Output: { |
|||
object: jest.fn((options: unknown) => { |
|||
return options; |
|||
}) |
|||
}, |
|||
tool: jest.fn((options: unknown) => { |
|||
return options; |
|||
}) |
|||
}; |
|||
}); |
|||
|
|||
describe('AiService', () => { |
|||
const abortSignal = new AbortController().signal; |
|||
const model = { modelId: 'model' }; |
|||
|
|||
const rows: ActivityRow[] = [ |
|||
createActivityRow({ accountName: null, symbol: null }) |
|||
]; |
|||
|
|||
let aiService: AiService; |
|||
let chat: jest.Mock; |
|||
let mcpService: McpService; |
|||
let properties: Record<string, string>; |
|||
|
|||
beforeEach(() => { |
|||
jest.clearAllMocks(); |
|||
|
|||
jest.spyOn(Logger.prototype, 'log').mockImplementation(); |
|||
|
|||
chat = jest.fn().mockReturnValue(model); |
|||
|
|||
jest |
|||
.mocked(createOpenRouter) |
|||
.mockReturnValue({ chat } as unknown as ReturnType< |
|||
typeof createOpenRouter |
|||
>); |
|||
|
|||
jest.mocked(generateText).mockResolvedValue({ |
|||
output: { broker: 'Swissquote', rows }, |
|||
steps: [{}], |
|||
totalUsage: { totalTokens: 1000 } |
|||
} as unknown as Awaited<ReturnType<typeof generateText>>); |
|||
|
|||
properties = { |
|||
[PROPERTY_API_KEY_OPENROUTER]: 'api-key', |
|||
[PROPERTY_OPENROUTER_MODEL]: 'general-model', |
|||
[PROPERTY_OPENROUTER_MODEL_AGENT]: 'agent-model', |
|||
[PROPERTY_OPENROUTER_MODEL_VISION]: 'vision-model' |
|||
}; |
|||
|
|||
mcpService = { |
|||
getAccounts: jest.fn().mockResolvedValue({ |
|||
content: [{ text: '## Accounts', type: 'text' }] |
|||
}), |
|||
searchAssetProfiles: jest.fn().mockResolvedValue({ |
|||
content: [{ text: '{ "assetProfiles": [] }', type: 'text' }] |
|||
}) |
|||
} as unknown as McpService; |
|||
|
|||
const propertyService = { |
|||
getByKey: jest.fn().mockImplementation((key: string) => { |
|||
return Promise.resolve(properties[key]); |
|||
}) |
|||
} as unknown as PropertyService; |
|||
|
|||
aiService = new AiService( |
|||
{} as ConfigurationService, |
|||
mcpService, |
|||
{} as PortfolioTableService, |
|||
propertyService |
|||
); |
|||
}); |
|||
|
|||
describe('getActivityRowsFromFile', () => { |
|||
it('Passes a PDF file as a file part to the vision model and gives the rows', async () => { |
|||
const buffer = Buffer.from('%PDF-1.7'); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
abortSignal, |
|||
file: { buffer, mediaType: 'application/pdf' }, |
|||
locale: 'de-CH' |
|||
}) |
|||
).resolves.toEqual({ broker: 'Swissquote', rows }); |
|||
|
|||
expect(createOpenRouter).toHaveBeenCalledWith({ apiKey: 'api-key' }); |
|||
|
|||
expect(generateText).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
abortSignal, |
|||
messages: [ |
|||
{ |
|||
content: [ |
|||
{ data: buffer, mediaType: 'application/pdf', type: 'file' } |
|||
], |
|||
role: 'user' |
|||
} |
|||
], |
|||
model, |
|||
output: { schema: ACTIVITY_ROWS_OUTPUT } |
|||
}) |
|||
); |
|||
}); |
|||
|
|||
it('Passes the name of the file as text before the file and gives the broker', async () => { |
|||
const buffer = Buffer.from('%PDF-1.7'); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer, |
|||
mediaType: 'application/pdf', |
|||
name: 'swissquote-2022.pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}) |
|||
).resolves.toEqual({ broker: 'Swissquote', rows }); |
|||
|
|||
expect(generateText).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
messages: [ |
|||
{ |
|||
content: [ |
|||
{ text: 'File name: swissquote-2022.pdf', type: 'text' }, |
|||
{ data: buffer, mediaType: 'application/pdf', type: 'file' } |
|||
], |
|||
role: 'user' |
|||
} |
|||
] |
|||
}) |
|||
); |
|||
}); |
|||
|
|||
it('Passes a CSV file as text', async () => { |
|||
const csv = 'Date;Type;ISIN\n15.01.2026;Kauf;US0378331005'; |
|||
|
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { buffer: Buffer.from(csv), mediaType: 'text/csv' }, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
expect(generateText).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
messages: [{ content: [{ text: csv, type: 'text' }], role: 'user' }] |
|||
}) |
|||
); |
|||
}); |
|||
|
|||
it('Restricts the routing to endpoints with zero data retention and passes a PDF file to the model itself', async () => { |
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { buffer: Buffer.from('%PDF-1.7'), mediaType: 'application/pdf' }, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
expect(chat).toHaveBeenCalledWith('vision-model', { |
|||
plugins: [{ id: 'file-parser', pdf: { engine: 'native' } }], |
|||
provider: { require_parameters: true, zdr: true }, |
|||
usage: { include: true } |
|||
}); |
|||
}); |
|||
|
|||
it('Falls back to the general model if no vision model is set', async () => { |
|||
delete properties[PROPERTY_OPENROUTER_MODEL_VISION]; |
|||
|
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { buffer: Buffer.from('%PDF-1.7'), mediaType: 'application/pdf' }, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
expect(chat).toHaveBeenCalledWith('general-model', expect.any(Object)); |
|||
}); |
|||
|
|||
it('Gives the locale and the date of today in the instructions', async () => { |
|||
jest.useFakeTimers({ now: new Date('2026-10-11T12:00:00Z') }); |
|||
|
|||
try { |
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}); |
|||
} finally { |
|||
jest.useRealTimers(); |
|||
} |
|||
|
|||
const { instructions } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
expect(instructions).toContain('the locale de-CH'); |
|||
expect(instructions).toContain('Today is 2026-10-11'); |
|||
}); |
|||
|
|||
it('Tells the vision model that a national number of a security is not a ticker symbol', async () => { |
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
const { instructions } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
expect(instructions).toContain( |
|||
'A national number of a security, for example a valor (NKN), a WKN or a CUSIP, is not a ticker symbol.' |
|||
); |
|||
}); |
|||
|
|||
it('Tells the vision model to add the fees of a buy or a sell to its row', async () => { |
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
const { instructions } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
expect(instructions).toContain( |
|||
'Add every fee, commission and tax of a buy or a sell, for example a commission, a stamp duty or an exchange fee, to the fee of its row.' |
|||
); |
|||
}); |
|||
|
|||
it('Gives an error for the caller if a cut output reaches the maximum number of output tokens', async () => { |
|||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
|||
|
|||
jest.mocked(generateText).mockRejectedValueOnce( |
|||
new NoObjectGeneratedError({ |
|||
finishReason: 'length' |
|||
} as ConstructorParameters<typeof NoObjectGeneratedError>[0]) |
|||
); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}) |
|||
).rejects.toThrow(CallerFacingError); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalled(); |
|||
}); |
|||
|
|||
it('Gives an error for the caller if an output without any text reaches the maximum number of output tokens', async () => { |
|||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
|||
|
|||
jest.mocked(generateText).mockResolvedValueOnce({ |
|||
finishReason: 'length', |
|||
get output() { |
|||
throw new Error('No output generated'); |
|||
} |
|||
} as unknown as Awaited<ReturnType<typeof generateText>>); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}) |
|||
).rejects.toThrow(CallerFacingError); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalled(); |
|||
}); |
|||
|
|||
it('Passes on every other error of the vision model', async () => { |
|||
const error = new NoObjectGeneratedError({ |
|||
finishReason: 'stop' |
|||
} as ConstructorParameters<typeof NoObjectGeneratedError>[0]); |
|||
|
|||
jest.mocked(generateText).mockRejectedValueOnce(error); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}) |
|||
).rejects.toBe(error); |
|||
}); |
|||
}); |
|||
|
|||
describe('getActivitiesFromFile', () => { |
|||
const file = { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}; |
|||
|
|||
const user = { |
|||
accounts: [{ currency: 'CHF', id: 'account-id', name: 'Brokerage' }], |
|||
id: 'user-id', |
|||
settings: { settings: { baseCurrency: 'EUR', locale: 'de-CH' } } |
|||
} as unknown as UserWithSettings; |
|||
|
|||
it('Reads the rows of the file and matches them with the accounts, the base currency and the id of the user', async () => { |
|||
const activities = [ |
|||
{ |
|||
currency: 'USD', |
|||
dataSource: 'YAHOO', |
|||
date: '2026-01-15', |
|||
fee: 1, |
|||
quantity: 10, |
|||
symbol: 'AAPL', |
|||
type: ActivityType.BUY, |
|||
unitPrice: 230 |
|||
} |
|||
]; |
|||
|
|||
jest |
|||
.spyOn(aiService, 'getActivityRowsFromFile') |
|||
.mockResolvedValue({ broker: 'Swissquote', rows }); |
|||
|
|||
jest |
|||
.spyOn(aiService, 'getActivitiesFromRows') |
|||
.mockResolvedValue({ activities, rowsWithoutMatch: [] }); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromFile({ abortSignal, file, user }) |
|||
).resolves.toEqual({ |
|||
activities, |
|||
assetProfiles: [], |
|||
rowsWithoutMatch: [] |
|||
}); |
|||
|
|||
expect(aiService.getActivityRowsFromFile).toHaveBeenCalledWith({ |
|||
abortSignal, |
|||
file, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
expect(aiService.getActivitiesFromRows).toHaveBeenCalledWith({ |
|||
abortSignal, |
|||
rows, |
|||
accounts: user.accounts, |
|||
broker: 'Swissquote', |
|||
userCurrency: 'EUR', |
|||
userId: 'user-id' |
|||
}); |
|||
}); |
|||
|
|||
it('Gives a fee, an interest or a liability a custom asset profile with its description as name', async () => { |
|||
const fee = { |
|||
currency: 'CHF', |
|||
date: '2026-01-15', |
|||
fee: 5, |
|||
quantity: 0, |
|||
symbol: 'Custody fee', |
|||
type: ActivityType.FEE, |
|||
unitPrice: 0 |
|||
}; |
|||
|
|||
jest |
|||
.spyOn(aiService, 'getActivityRowsFromFile') |
|||
.mockResolvedValue({ broker: 'Swissquote', rows }); |
|||
|
|||
jest.spyOn(aiService, 'getActivitiesFromRows').mockResolvedValue({ |
|||
activities: [fee, { ...fee, date: '2026-02-15' }], |
|||
rowsWithoutMatch: [] |
|||
}); |
|||
|
|||
const { activities, assetProfiles } = |
|||
await aiService.getActivitiesFromFile({ file, user }); |
|||
|
|||
const [{ symbol }] = assetProfiles; |
|||
|
|||
expect(isValidCustomAssetProfileSymbol(symbol)).toBe(true); |
|||
|
|||
expect(assetProfiles).toEqual([ |
|||
{ |
|||
symbol, |
|||
currency: 'CHF', |
|||
dataSource: DataSource.MANUAL, |
|||
name: 'Custody fee' |
|||
} |
|||
]); |
|||
|
|||
expect(activities).toEqual([ |
|||
{ ...fee, symbol, dataSource: DataSource.MANUAL }, |
|||
{ ...fee, symbol, dataSource: DataSource.MANUAL, date: '2026-02-15' } |
|||
]); |
|||
}); |
|||
|
|||
it('Reads the file in the default locale if the user has no locale', async () => { |
|||
jest |
|||
.spyOn(aiService, 'getActivityRowsFromFile') |
|||
.mockResolvedValue({ broker: null, rows: [] }); |
|||
|
|||
jest |
|||
.spyOn(aiService, 'getActivitiesFromRows') |
|||
.mockResolvedValue({ activities: [], rowsWithoutMatch: [] }); |
|||
|
|||
await aiService.getActivitiesFromFile({ |
|||
file, |
|||
user: { |
|||
...user, |
|||
settings: { settings: { baseCurrency: 'EUR' } } |
|||
} as unknown as UserWithSettings |
|||
}); |
|||
|
|||
expect(aiService.getActivityRowsFromFile).toHaveBeenCalledWith( |
|||
expect.objectContaining({ locale: DEFAULT_LOCALE }) |
|||
); |
|||
}); |
|||
}); |
|||
|
|||
describe('getActivitiesFromRows', () => { |
|||
const accounts: AccountToMatch[] = [ |
|||
{ currency: 'CHF', id: 'account-id', name: 'Brokerage' } |
|||
]; |
|||
|
|||
const userCurrency = 'EUR'; |
|||
const userId = 'user-id'; |
|||
|
|||
function mockRowMatches({ |
|||
accounts: accountMatches = [], |
|||
assetProfilesOfSearch, |
|||
assets: assetMatches = [] |
|||
}: Partial<z.infer<typeof ROW_MATCHES_OUTPUT>> & { |
|||
assetProfilesOfSearch?: { dataSource: string; symbol: string }[]; |
|||
}) { |
|||
jest.mocked(mcpService.searchAssetProfiles).mockResolvedValueOnce({ |
|||
content: [ |
|||
{ |
|||
text: JSON.stringify({ |
|||
assetProfiles: assetProfilesOfSearch ?? assetMatches |
|||
}), |
|||
type: 'text' |
|||
} |
|||
] |
|||
}); |
|||
|
|||
jest.mocked(generateText).mockImplementationOnce(async ({ tools }) => { |
|||
await tools['search-asset-profiles'].execute( |
|||
{ query: 'US0378331005' }, |
|||
undefined |
|||
); |
|||
|
|||
return { |
|||
output: { accounts: accountMatches, assets: assetMatches }, |
|||
steps: [{}, {}], |
|||
totalUsage: { totalTokens: 2000 } |
|||
} as unknown as Awaited<ReturnType<typeof generateText>>; |
|||
}); |
|||
} |
|||
|
|||
it('Gives an activity for each row whose asset matches an asset profile', async () => { |
|||
const row = createActivityRow({ currency: null }); |
|||
|
|||
mockRowMatches({ |
|||
accounts: [{ accountId: 'account-id', id: 0 }], |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ |
|||
activities: [ |
|||
{ |
|||
accountId: 'account-id', |
|||
currency: 'USD', |
|||
dataSource: 'YAHOO', |
|||
date: row.date, |
|||
fee: row.fee, |
|||
quantity: row.quantity, |
|||
symbol: 'AAPL', |
|||
type: ActivityType.BUY, |
|||
unitPrice: row.unitPrice |
|||
} |
|||
], |
|||
rowsWithoutMatch: [] |
|||
}); |
|||
|
|||
expect(Logger.prototype.log).toHaveBeenCalledWith( |
|||
expect.stringContaining( |
|||
'Matched 1 of 1 assets and 1 of 1 account hints with 1 results of 1 searches' |
|||
) |
|||
); |
|||
}); |
|||
|
|||
it('Gives the currency of the row before the currency of the asset profile', async () => { |
|||
mockRowMatches({ |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow({ accountName: null, currency: 'CHF' })] |
|||
}); |
|||
|
|||
expect(activities[0].currency).toBe('CHF'); |
|||
}); |
|||
|
|||
it('Gives the absolute values of a negative fee, quantity and unit price', async () => { |
|||
mockRowMatches({ |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [ |
|||
createActivityRow({ |
|||
accountName: null, |
|||
fee: -1, |
|||
quantity: -10, |
|||
type: ActivityType.SELL, |
|||
unitPrice: -230 |
|||
}) |
|||
] |
|||
}); |
|||
|
|||
expect(activities[0]).toMatchObject({ |
|||
fee: 1, |
|||
quantity: 10, |
|||
unitPrice: 230 |
|||
}); |
|||
}); |
|||
|
|||
it('Gives a row whose asset has no match as a row without a match', async () => { |
|||
const row = createActivityRow({ accountName: null }); |
|||
|
|||
mockRowMatches({ |
|||
assets: [{ currency: null, dataSource: null, id: 0, symbol: null }] |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
|||
}); |
|||
|
|||
it('Gives a row with a data source which is not valid as a row without a match', async () => { |
|||
const row = createActivityRow({ accountName: null }); |
|||
|
|||
mockRowMatches({ |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'yahoo', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
|||
}); |
|||
|
|||
it('Gives a row whose asset profile no search gave as a row without a match and logs the left out match', async () => { |
|||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
|||
|
|||
const row = createActivityRow({ accountName: null }); |
|||
|
|||
mockRowMatches({ |
|||
assetProfilesOfSearch: [{ dataSource: 'YAHOO', symbol: 'AAPL' }], |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL.XX' } |
|||
] |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
|||
|
|||
expect(Logger.prototype.log).toHaveBeenCalledWith( |
|||
expect.stringContaining( |
|||
'Matched 0 of 1 assets and 0 of 0 account hints with 1 results of 1 searches' |
|||
) |
|||
); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
|||
'Left out 1 asset matches which no search gave' |
|||
); |
|||
}); |
|||
|
|||
it('Matches a row without an account name by the broker of the file', async () => { |
|||
mockRowMatches({ |
|||
accounts: [{ accountId: 'account-id', id: 0 }], |
|||
assets: [ |
|||
{ currency: 'CHF', dataSource: 'YAHOO', id: 0, symbol: 'ROG.SW' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
broker: 'Swissquote', |
|||
rows: [ |
|||
createActivityRow({ accountName: null }), |
|||
createActivityRow({ accountName: null, type: ActivityType.SELL }) |
|||
] |
|||
}); |
|||
|
|||
expect(activities.map(({ accountId }) => accountId)).toEqual([ |
|||
'account-id', |
|||
'account-id' |
|||
]); |
|||
|
|||
const { prompt } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
const { accountHints } = JSON.parse(prompt as string) as { |
|||
accountHints: AccountHint[]; |
|||
}; |
|||
|
|||
expect(accountHints).toEqual([ |
|||
{ accountName: null, broker: 'Swissquote', id: 0 } |
|||
]); |
|||
}); |
|||
|
|||
it('Ignores an account id which is not the id of an account of the user', async () => { |
|||
mockRowMatches({ |
|||
accounts: [{ accountId: 'unknown-account-id', id: 0 }], |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
expect(activities[0].accountId).toBeUndefined(); |
|||
}); |
|||
|
|||
it('Gives a fee, an interest or a liability its description as symbol without a model call', async () => { |
|||
const rowsOfNonInvestments = [ |
|||
ActivityType.FEE, |
|||
ActivityType.INTEREST, |
|||
ActivityType.LIABILITY |
|||
].map((type) => { |
|||
return createActivityRow({ |
|||
type, |
|||
accountName: null, |
|||
currency: null, |
|||
isin: null, |
|||
name: `Description of ${type}`, |
|||
symbol: null |
|||
}); |
|||
}); |
|||
|
|||
const { activities, rowsWithoutMatch } = |
|||
await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: rowsOfNonInvestments |
|||
}); |
|||
|
|||
expect(generateText).not.toHaveBeenCalled(); |
|||
|
|||
expect( |
|||
activities.map(({ currency, dataSource, symbol, type }) => { |
|||
return { currency, dataSource, symbol, type }; |
|||
}) |
|||
).toEqual( |
|||
rowsOfNonInvestments.map(({ name, type }) => { |
|||
return { |
|||
type, |
|||
currency: userCurrency, |
|||
dataSource: undefined, |
|||
symbol: name |
|||
}; |
|||
}) |
|||
); |
|||
|
|||
expect(rowsWithoutMatch).toEqual([]); |
|||
}); |
|||
|
|||
it('Gives a fee without a currency the currency of its account', async () => { |
|||
mockRowMatches({ |
|||
accounts: [{ accountId: 'account-id', id: 0 }] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [ |
|||
createActivityRow({ |
|||
currency: null, |
|||
isin: null, |
|||
name: 'Custody fee', |
|||
symbol: null, |
|||
type: ActivityType.FEE |
|||
}) |
|||
] |
|||
}); |
|||
|
|||
expect(activities[0]).toMatchObject({ |
|||
accountId: 'account-id', |
|||
currency: 'CHF' |
|||
}); |
|||
}); |
|||
|
|||
it('Gives a fee without a description as a row without a match', async () => { |
|||
const row = createActivityRow({ |
|||
accountName: null, |
|||
isin: null, |
|||
name: null, |
|||
symbol: null, |
|||
type: ActivityType.FEE |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
|||
}); |
|||
|
|||
it('Passes each asset and each account hint one time and no account to the model', async () => { |
|||
mockRowMatches({ |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
broker: 'Swissquote', |
|||
rows: [ |
|||
createActivityRow(), |
|||
createActivityRow({ type: ActivityType.SELL }), |
|||
createActivityRow({ |
|||
accountName: 'Depot 1', |
|||
type: ActivityType.DIVIDEND |
|||
}) |
|||
] |
|||
}); |
|||
|
|||
expect(activities).toHaveLength(3); |
|||
|
|||
const { prompt } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
expect(JSON.parse(prompt as string)).toEqual({ |
|||
accountHints: [ |
|||
{ accountName: 'Brokerage', broker: 'Swissquote', id: 0 }, |
|||
{ accountName: 'Depot 1', broker: 'Swissquote', id: 1 } |
|||
], |
|||
assets: [ |
|||
{ |
|||
currency: 'USD', |
|||
id: 0, |
|||
isin: 'US0378331005', |
|||
name: 'Apple Inc.', |
|||
symbol: 'AAPL' |
|||
} |
|||
] |
|||
}); |
|||
}); |
|||
|
|||
it('Searches the asset profiles with the tool of the server of the Model Context Protocol', async () => { |
|||
mockRowMatches({}); |
|||
|
|||
await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
const { tools } = jest.mocked(generateText).mock.calls[0][0] as { |
|||
tools: ToolSet; |
|||
}; |
|||
|
|||
await expect( |
|||
tools['search-asset-profiles'].execute( |
|||
{ query: 'US0378331005' }, |
|||
undefined |
|||
) |
|||
).resolves.toBe('{ "assetProfiles": [] }'); |
|||
|
|||
expect(mcpService.searchAssetProfiles).toHaveBeenCalledWith({ |
|||
userId, |
|||
query: 'US0378331005' |
|||
}); |
|||
}); |
|||
|
|||
it('Gets the accounts with the tool of the server of the Model Context Protocol without values', async () => { |
|||
mockRowMatches({}); |
|||
|
|||
await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
const { tools } = jest.mocked(generateText).mock.calls[0][0] as { |
|||
tools: ToolSet; |
|||
}; |
|||
|
|||
await expect(tools['get-accounts'].execute({}, undefined)).resolves.toBe( |
|||
'## Accounts' |
|||
); |
|||
|
|||
expect(mcpService.getAccounts).toHaveBeenCalledWith({ |
|||
userId, |
|||
withValues: false |
|||
}); |
|||
}); |
|||
|
|||
it('Restricts the routing to endpoints with zero data retention, limits the number of steps, requires a tool call in the first step and allows no tool call in the last step', async () => { |
|||
mockRowMatches({}); |
|||
|
|||
await aiService.getActivitiesFromRows({ |
|||
abortSignal, |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
expect(chat).toHaveBeenCalledWith('agent-model', { |
|||
provider: { require_parameters: true, zdr: true }, |
|||
usage: { include: true } |
|||
}); |
|||
|
|||
expect(isStepCount).toHaveBeenCalled(); |
|||
|
|||
expect(generateText).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
abortSignal, |
|||
model, |
|||
output: { schema: ROW_MATCHES_OUTPUT } |
|||
}) |
|||
); |
|||
|
|||
const { prepareStep, stopWhen } = jest.mocked(generateText).mock |
|||
.calls[0][0] as { |
|||
prepareStep: (options: { stepNumber: number }) => unknown; |
|||
stopWhen: unknown; |
|||
}; |
|||
|
|||
expect(stopWhen).toBe(jest.mocked(isStepCount).mock.results[0].value); |
|||
|
|||
const [stepCount] = jest.mocked(isStepCount).mock.calls[0]; |
|||
|
|||
expect(prepareStep({ stepNumber: 0 })).toEqual({ |
|||
toolChoice: 'required' |
|||
}); |
|||
expect(prepareStep({ stepNumber: stepCount - 2 })).toBeUndefined(); |
|||
expect(prepareStep({ stepNumber: stepCount - 1 })).toEqual({ |
|||
toolChoice: 'none' |
|||
}); |
|||
}); |
|||
|
|||
it('Falls back to the general model if no agent model is set', async () => { |
|||
delete properties[PROPERTY_OPENROUTER_MODEL_AGENT]; |
|||
|
|||
mockRowMatches({}); |
|||
|
|||
await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
expect(chat).toHaveBeenCalledWith('general-model', expect.any(Object)); |
|||
}); |
|||
}); |
|||
}); |
|||
@ -0,0 +1,21 @@ |
|||
import { Type as ActivityType } from '@prisma/client'; |
|||
|
|||
import { ActivityRow } from './types/activity-row.type'; |
|||
|
|||
export function createActivityRow( |
|||
overrides: Partial<ActivityRow> = {} |
|||
): ActivityRow { |
|||
return { |
|||
accountName: 'Brokerage', |
|||
currency: 'USD', |
|||
date: '2026-01-15', |
|||
fee: 1, |
|||
isin: 'US0378331005', |
|||
name: 'Apple Inc.', |
|||
quantity: 10, |
|||
symbol: 'AAPL', |
|||
type: ActivityType.BUY, |
|||
unitPrice: 230, |
|||
...overrides |
|||
}; |
|||
} |
|||
@ -0,0 +1,6 @@ |
|||
import { ActivityRow } from './activity-row.type'; |
|||
|
|||
export type AccountHint = Pick<ActivityRow, 'accountName'> & { |
|||
broker: string | null; |
|||
id: number; |
|||
}; |
|||
@ -0,0 +1,3 @@ |
|||
import { Account } from '@prisma/client'; |
|||
|
|||
export type AccountToMatch = Pick<Account, 'currency' | 'id' | 'name'>; |
|||
@ -0,0 +1,5 @@ |
|||
import { z } from 'zod'; |
|||
|
|||
import { ACTIVITY_ROWS_OUTPUT } from '../ai.schemas'; |
|||
|
|||
export type ActivityRow = z.infer<typeof ACTIVITY_ROWS_OUTPUT>['rows'][number]; |
|||
@ -0,0 +1,6 @@ |
|||
import { ActivityRow } from './activity-row.type'; |
|||
|
|||
export type AssetToMatch = Pick< |
|||
ActivityRow, |
|||
'currency' | 'isin' | 'name' | 'symbol' |
|||
> & { id: number }; |
|||
@ -0,0 +1,18 @@ |
|||
import { Type as ActivityType } from '@ghostfolio/prisma/enums'; |
|||
|
|||
/** |
|||
* An activity as Ghostfolio AI reads it from a file, with the asset and the |
|||
* account as printed |
|||
*/ |
|||
export interface AiActivityRow { |
|||
accountName?: string | null; |
|||
currency?: string | null; |
|||
date: string; |
|||
fee: number; |
|||
isin?: string | null; |
|||
name?: string | null; |
|||
quantity: number; |
|||
symbol?: string | null; |
|||
type: ActivityType; |
|||
unitPrice: number; |
|||
} |
|||
@ -0,0 +1,12 @@ |
|||
import type { |
|||
CreateAssetProfileWithMarketDataDto, |
|||
CreateOrderDto |
|||
} from '@ghostfolio/common/dtos'; |
|||
|
|||
import { AiActivityRow } from '../ai-activity-row.interface'; |
|||
|
|||
export interface AiActivitiesResponse { |
|||
activities: CreateOrderDto[]; |
|||
assetProfiles: CreateAssetProfileWithMarketDataDto[]; |
|||
rowsWithoutMatch: AiActivityRow[]; |
|||
} |
|||
Loading…
Reference in new issue