mirror of https://github.com/ghostfolio/ghostfolio
KenTandrian
3 days ago
57 changed files with 7108 additions and 2731 deletions
@ -0,0 +1,279 @@ |
|||
import { HAS_PERMISSION_KEY } from '@ghostfolio/api/decorators/has-permission.decorator'; |
|||
import { CallerFacingError } from '@ghostfolio/api/errors/caller-facing.error'; |
|||
import { CustomThrottlerGuard } from '@ghostfolio/api/guards/custom-throttler.guard'; |
|||
import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'; |
|||
import { ApiService } from '@ghostfolio/api/services/api/api.service'; |
|||
import { |
|||
AI_ACTIVITIES_TIMEOUT, |
|||
AI_MAX_ACTIVITIES, |
|||
THROTTLE_AI_ACTIVITIES_LIMIT, |
|||
THROTTLE_AI_ACTIVITIES_TTL |
|||
} from '@ghostfolio/common/config'; |
|||
import { permissions } from '@ghostfolio/common/permissions'; |
|||
import type { RequestWithUser } from '@ghostfolio/common/types'; |
|||
|
|||
import { HttpException, Logger } from '@nestjs/common'; |
|||
import { GUARDS_METADATA } from '@nestjs/common/constants'; |
|||
import { |
|||
THROTTLER_LIMIT, |
|||
THROTTLER_TRACKER, |
|||
THROTTLER_TTL |
|||
} from '@nestjs/throttler/dist/throttler.constants'; |
|||
import type { Response } from 'express'; |
|||
import { StatusCodes } from 'http-status-codes'; |
|||
import { EventEmitter } from 'node:events'; |
|||
|
|||
import { AiController } from './ai.controller'; |
|||
import { AiService } from './ai.service'; |
|||
|
|||
// The packages are ECMAScript modules, which Jest cannot load
|
|||
jest.mock('@openrouter/ai-sdk-provider', () => { |
|||
return {}; |
|||
}); |
|||
jest.mock('ai', () => { |
|||
return {}; |
|||
}); |
|||
|
|||
describe('AiController', () => { |
|||
const pdfFile = { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
originalname: 'statement.pdf' |
|||
}; |
|||
|
|||
const user = { id: 'user-id' }; |
|||
|
|||
let aiController: AiController; |
|||
let aiService: AiService; |
|||
|
|||
function createCsvFile(dataRowCount: number) { |
|||
return { |
|||
buffer: Buffer.from( |
|||
[ |
|||
'Date,Symbol', |
|||
...Array.from({ length: dataRowCount }, () => { |
|||
return '2026-01-15,AAPL'; |
|||
}) |
|||
].join('\n') |
|||
), |
|||
originalname: 'statement.csv' |
|||
}; |
|||
} |
|||
|
|||
function createResponse() { |
|||
return Object.assign(new EventEmitter(), { |
|||
writableFinished: false |
|||
}) as unknown as Response; |
|||
} |
|||
|
|||
function getMetadataOfMethod<T>(metadataKey: string) { |
|||
return Reflect.getMetadata( |
|||
metadataKey, |
|||
AiController.prototype.getActivitiesFromFile |
|||
) as T; |
|||
} |
|||
|
|||
beforeEach(() => { |
|||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
|||
|
|||
aiService = { |
|||
getActivitiesFromFile: jest.fn().mockResolvedValue({ |
|||
activities: [], |
|||
assetProfiles: [], |
|||
rowsWithoutMatch: [] |
|||
}) |
|||
} as unknown as AiService; |
|||
|
|||
aiController = new AiController( |
|||
aiService, |
|||
{} as ApiService, |
|||
{ |
|||
user |
|||
} as unknown as RequestWithUser |
|||
); |
|||
}); |
|||
|
|||
describe('getActivitiesFromFile', () => { |
|||
it('Requires the permission and limits the requests of each user per week', () => { |
|||
expect(getMetadataOfMethod<string>(HAS_PERMISSION_KEY)).toBe( |
|||
permissions.importActivitiesWithAi |
|||
); |
|||
|
|||
const guards = getMetadataOfMethod<unknown[]>(GUARDS_METADATA); |
|||
|
|||
expect(guards.slice(1)).toEqual([ |
|||
HasPermissionGuard, |
|||
CustomThrottlerGuard |
|||
]); |
|||
|
|||
expect(getMetadataOfMethod<number>(`${THROTTLER_LIMIT}default`)).toBe( |
|||
THROTTLE_AI_ACTIVITIES_LIMIT |
|||
); |
|||
|
|||
expect(getMetadataOfMethod<number>(`${THROTTLER_TTL}default`)).toBe( |
|||
THROTTLE_AI_ACTIVITIES_TTL |
|||
); |
|||
|
|||
expect( |
|||
getMetadataOfMethod<(request: unknown) => string>( |
|||
`${THROTTLER_TRACKER}default` |
|||
)({ ip: '127.0.0.1', user }) |
|||
).toBe(user.id); |
|||
}); |
|||
|
|||
it('Refuses a request without a file or with a file of another type', async () => { |
|||
for (const file of [ |
|||
undefined, |
|||
{ buffer: Buffer.from('Date,Symbol'), originalname: 'statement.txt' } |
|||
]) { |
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), file) |
|||
).rejects.toThrow(HttpException); |
|||
} |
|||
|
|||
expect(aiService.getActivitiesFromFile).not.toHaveBeenCalled(); |
|||
}); |
|||
|
|||
it(`Refuses a CSV file with more than ${AI_MAX_ACTIVITIES} rows before a model call`, async () => { |
|||
await expect( |
|||
aiController.getActivitiesFromFile( |
|||
createResponse(), |
|||
createCsvFile(AI_MAX_ACTIVITIES + 1) |
|||
) |
|||
).rejects.toThrow(HttpException); |
|||
|
|||
expect(aiService.getActivitiesFromFile).not.toHaveBeenCalled(); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
|||
`Rejected a CSV file with ${AI_MAX_ACTIVITIES + 1} rows` |
|||
); |
|||
}); |
|||
|
|||
it(`Accepts a CSV file with ${AI_MAX_ACTIVITIES} rows`, async () => { |
|||
const file = createCsvFile(AI_MAX_ACTIVITIES); |
|||
|
|||
await aiController.getActivitiesFromFile(createResponse(), file); |
|||
|
|||
expect(aiService.getActivitiesFromFile).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
file: { |
|||
buffer: file.buffer, |
|||
mediaType: 'text/csv', |
|||
name: 'statement.csv' |
|||
} |
|||
}) |
|||
); |
|||
}); |
|||
|
|||
it('Passes the file with its media type and its name and the user of the request', async () => { |
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
|||
).resolves.toEqual({ |
|||
activities: [], |
|||
assetProfiles: [], |
|||
rowsWithoutMatch: [] |
|||
}); |
|||
|
|||
expect(aiService.getActivitiesFromFile).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
user, |
|||
file: { |
|||
buffer: pdfFile.buffer, |
|||
mediaType: 'application/pdf', |
|||
name: 'statement.pdf' |
|||
} |
|||
}) |
|||
); |
|||
|
|||
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
|||
.mock.calls; |
|||
|
|||
expect(abortSignal).toBeInstanceOf(AbortSignal); |
|||
}); |
|||
|
|||
it('Stops the models if the client closes the request before the response', async () => { |
|||
const response = createResponse(); |
|||
|
|||
await aiController.getActivitiesFromFile(response, pdfFile); |
|||
|
|||
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
|||
.mock.calls; |
|||
|
|||
expect(abortSignal.aborted).toBe(false); |
|||
|
|||
response.emit('close'); |
|||
|
|||
expect(abortSignal.aborted).toBe(true); |
|||
}); |
|||
|
|||
it('Does not stop the models when the response is finished', async () => { |
|||
const response = createResponse(); |
|||
|
|||
await aiController.getActivitiesFromFile(response, pdfFile); |
|||
|
|||
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
|||
.mock.calls; |
|||
|
|||
Object.assign(response, { writableFinished: true }); |
|||
response.emit('close'); |
|||
|
|||
expect(abortSignal.aborted).toBe(false); |
|||
}); |
|||
|
|||
it('Gives the status 400 with the message of an error for the caller', async () => { |
|||
jest |
|||
.mocked(aiService.getActivitiesFromFile) |
|||
.mockRejectedValueOnce( |
|||
new CallerFacingError('The file has too many activities') |
|||
); |
|||
|
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
|||
).rejects.toMatchObject({ |
|||
response: { message: ['The file has too many activities'] }, |
|||
status: StatusCodes.BAD_REQUEST |
|||
}); |
|||
}); |
|||
|
|||
it('Gives the status 400 with a message for the caller at the timeout', async () => { |
|||
const timeoutSignal = AbortSignal.abort( |
|||
new DOMException( |
|||
'The operation was aborted due to timeout', |
|||
'TimeoutError' |
|||
) |
|||
); |
|||
|
|||
jest.spyOn(AbortSignal, 'timeout').mockReturnValueOnce(timeoutSignal); |
|||
|
|||
jest |
|||
.mocked(aiService.getActivitiesFromFile) |
|||
.mockRejectedValueOnce(timeoutSignal.reason); |
|||
|
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
|||
).rejects.toMatchObject({ |
|||
response: { |
|||
message: [ |
|||
'Ghostfolio AI needs too much time for the file. Split it into smaller files.' |
|||
] |
|||
}, |
|||
status: StatusCodes.BAD_REQUEST |
|||
}); |
|||
|
|||
expect(AbortSignal.timeout).toHaveBeenCalledWith(AI_ACTIVITIES_TIMEOUT); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
|||
`Stopped reading a file of the type application/pdf at the timeout of ${AI_ACTIVITIES_TIMEOUT} ms` |
|||
); |
|||
}); |
|||
|
|||
it('Passes on every other error', async () => { |
|||
const error = new Error('Model error'); |
|||
|
|||
jest.mocked(aiService.getActivitiesFromFile).mockRejectedValueOnce(error); |
|||
|
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
|||
).rejects.toBe(error); |
|||
}); |
|||
}); |
|||
}); |
|||
@ -0,0 +1,99 @@ |
|||
import { |
|||
getDataRowCountOfCsvFile, |
|||
getMediaTypeOfActivityFile |
|||
} from './ai.helper'; |
|||
|
|||
describe('getDataRowCountOfCsvFile', () => { |
|||
it('Gives the number of the lines which are not empty, without the header', () => { |
|||
expect( |
|||
getDataRowCountOfCsvFile( |
|||
Buffer.from('Date,Symbol\n2026-01-15,AAPL\n\n2026-01-16,MSFT\n') |
|||
) |
|||
).toBe(2); |
|||
}); |
|||
|
|||
it('Counts each line of a value with a line break in quotes', () => { |
|||
expect( |
|||
getDataRowCountOfCsvFile( |
|||
Buffer.from('Date,Comment\r\n2026-01-15,"First\r\nsecond"\r\n') |
|||
) |
|||
).toBe(2); |
|||
}); |
|||
|
|||
it('Counts each line after a quote which is not closed', () => { |
|||
expect( |
|||
getDataRowCountOfCsvFile( |
|||
Buffer.from( |
|||
[ |
|||
'Date,Symbol,Quantity', |
|||
'2026-01-15,"AAPL,1', |
|||
...Array.from({ length: 200 }, () => { |
|||
return '2026-01-16,MSFT,1'; |
|||
}) |
|||
].join('\n') |
|||
) |
|||
) |
|||
).toBe(201); |
|||
}); |
|||
|
|||
it('Gives 0 for a file with a header only or without content', () => { |
|||
expect(getDataRowCountOfCsvFile(Buffer.from('Date,Symbol\n'))).toBe(0); |
|||
expect(getDataRowCountOfCsvFile(Buffer.from(''))).toBe(0); |
|||
}); |
|||
}); |
|||
|
|||
describe('getMediaTypeOfActivityFile', () => { |
|||
it('Identifies a JPEG, a PDF and a PNG file by the first bytes', () => { |
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from([0xff, 0xd8, 0xff, 0xe0]), |
|||
fileName: 'statement' |
|||
}) |
|||
).toBe('image/jpeg'); |
|||
|
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
fileName: 'statement.csv' |
|||
}) |
|||
).toBe('application/pdf'); |
|||
|
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), |
|||
fileName: 'statement.jpg' |
|||
}) |
|||
).toBe('image/png'); |
|||
}); |
|||
|
|||
it('Identifies a CSV file by the extension and the content in UTF-8', () => { |
|||
for (const fileName of ['statement.csv', 'STATEMENT.CSV']) { |
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
fileName, |
|||
buffer: Buffer.from('Datum;Stück\n15.01.2026;10\n') |
|||
}) |
|||
).toBe('text/csv'); |
|||
} |
|||
}); |
|||
|
|||
it('Refuses a CSV file which is not in UTF-8', () => { |
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from([0x53, 0x74, 0xfc, 0x63, 0x6b]), |
|||
fileName: 'statement.csv' |
|||
}) |
|||
).toBeUndefined(); |
|||
}); |
|||
|
|||
it('Refuses any other file', () => { |
|||
for (const { buffer, fileName } of [ |
|||
{ buffer: Buffer.from('Date,Symbol'), fileName: 'statement.txt' }, |
|||
{ buffer: Buffer.from('<html></html>'), fileName: 'statement.pdf' }, |
|||
{ buffer: Buffer.from([0xff, 0xd8]), fileName: 'statement.jpg' }, |
|||
{ buffer: Buffer.from(''), fileName: 'statement.png' } |
|||
]) { |
|||
expect(getMediaTypeOfActivityFile({ buffer, fileName })).toBeUndefined(); |
|||
} |
|||
}); |
|||
}); |
|||
@ -0,0 +1,57 @@ |
|||
import { isUtf8 } from 'node:buffer'; |
|||
|
|||
const JPEG_SIGNATURE = [0xff, 0xd8, 0xff]; |
|||
const PDF_SIGNATURE = [0x25, 0x50, 0x44, 0x46, 0x2d]; |
|||
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; |
|||
|
|||
/** |
|||
* Gives the number of the lines of a CSV file which are not empty, without |
|||
* the first line. A line before the header and each line of a value with a |
|||
* line break are counted as data rows, hence the number is never too low, |
|||
* also for a file with a quote which is not closed. |
|||
*/ |
|||
export function getDataRowCountOfCsvFile(buffer: Buffer) { |
|||
const lineCount = buffer |
|||
.toString('utf8') |
|||
.split(/\r\n|\r|\n/) |
|||
.filter((line) => { |
|||
return line.trim().length > 0; |
|||
}).length; |
|||
|
|||
return Math.max(lineCount - 1, 0); |
|||
} |
|||
|
|||
/** |
|||
* Gives the media type of a file which Ghostfolio AI reads. A JPEG, a PDF or |
|||
* a PNG file is identified by its first bytes, a CSV file by its extension |
|||
* and by its content in UTF-8, as the media type which a client sends for a |
|||
* CSV file varies. |
|||
*/ |
|||
export function getMediaTypeOfActivityFile({ |
|||
buffer, |
|||
fileName |
|||
}: { |
|||
buffer: Buffer; |
|||
fileName: string; |
|||
}) { |
|||
if (startsWith(buffer, JPEG_SIGNATURE)) { |
|||
return 'image/jpeg'; |
|||
} else if (startsWith(buffer, PDF_SIGNATURE)) { |
|||
return 'application/pdf'; |
|||
} else if (startsWith(buffer, PNG_SIGNATURE)) { |
|||
return 'image/png'; |
|||
} else if (fileName.toLowerCase().endsWith('.csv') && isUtf8(buffer)) { |
|||
return 'text/csv'; |
|||
} |
|||
|
|||
return undefined; |
|||
} |
|||
|
|||
function startsWith(buffer: Buffer, signature: number[]) { |
|||
return ( |
|||
buffer.length >= signature.length && |
|||
signature.every((byte, index) => { |
|||
return buffer[index] === byte; |
|||
}) |
|||
); |
|||
} |
|||
@ -0,0 +1,101 @@ |
|||
import { |
|||
AI_MAX_ACTIVITIES, |
|||
SEARCH_QUERY_MAXIMUM_LENGTH |
|||
} from '@ghostfolio/common/config'; |
|||
|
|||
import { isObject } from 'lodash-es'; |
|||
import { z } from 'zod'; |
|||
|
|||
import { ACTIVITY_ROWS_OUTPUT, ROW_MATCHES_OUTPUT } from './ai.schemas'; |
|||
import { createActivityRow } from './ai.test-utils'; |
|||
import { ActivityRow } from './types/activity-row.type'; |
|||
|
|||
describe('ACTIVITY_ROWS_OUTPUT', () => { |
|||
it('Accepts a row with null for the values which the file does not show', () => { |
|||
expect( |
|||
ACTIVITY_ROWS_OUTPUT.safeParse({ |
|||
broker: null, |
|||
rows: [ |
|||
createActivityRow({ |
|||
accountName: null, |
|||
currency: null, |
|||
isin: null, |
|||
symbol: null |
|||
}) |
|||
] |
|||
}).success |
|||
).toBe(true); |
|||
}); |
|||
|
|||
it('Refuses a row with a value which is left out instead of null', () => { |
|||
const row: Partial<ActivityRow> = createActivityRow(); |
|||
|
|||
delete row.isin; |
|||
|
|||
expect( |
|||
ACTIVITY_ROWS_OUTPUT.safeParse({ broker: null, rows: [row] }).success |
|||
).toBe(false); |
|||
}); |
|||
|
|||
it('Accepts a currency, a date, a number, a printed text and a number of rows which the join checks, so that one value does not refuse all rows', () => { |
|||
expect( |
|||
ACTIVITY_ROWS_OUTPUT.safeParse({ |
|||
broker: null, |
|||
rows: [ |
|||
createActivityRow({ |
|||
currency: 'GBX', |
|||
date: '15.01.2026', |
|||
name: 'A'.repeat(SEARCH_QUERY_MAXIMUM_LENGTH + 1) |
|||
}), |
|||
createActivityRow({ fee: -1, quantity: -1, unitPrice: -1 }), |
|||
...Array.from({ length: AI_MAX_ACTIVITIES }, () => { |
|||
return createActivityRow(); |
|||
}) |
|||
] |
|||
}).success |
|||
).toBe(true); |
|||
}); |
|||
|
|||
it('Trims a printed text', () => { |
|||
const { data } = ACTIVITY_ROWS_OUTPUT.safeParse({ |
|||
broker: ' Swissquote ', |
|||
rows: [createActivityRow({ name: ' Apple Inc. ' })] |
|||
}); |
|||
|
|||
expect(data.broker).toBe('Swissquote'); |
|||
expect(data.rows[0].name).toBe('Apple Inc.'); |
|||
}); |
|||
}); |
|||
|
|||
describe.each([ |
|||
['ACTIVITY_ROWS_OUTPUT', ACTIVITY_ROWS_OUTPUT], |
|||
['ROW_MATCHES_OUTPUT', ROW_MATCHES_OUTPUT] |
|||
])('%s', (_name, schema) => { |
|||
it('Gives a JSON schema which the strict mode of the structured outputs supports', () => { |
|||
// The options of the conversion in the AI SDK
|
|||
const jsonSchema = z.toJSONSchema(schema, { |
|||
io: 'input', |
|||
target: 'draft-7' |
|||
}); |
|||
|
|||
const objectJsonSchemas: { properties: object; required: string[] }[] = []; |
|||
|
|||
JSON.stringify(jsonSchema, (_key, value: unknown) => { |
|||
if (isObject(value) && 'properties' in value) { |
|||
objectJsonSchemas.push( |
|||
value as { properties: object; required: string[] } |
|||
); |
|||
} |
|||
|
|||
return value; |
|||
}); |
|||
|
|||
expect(objectJsonSchemas.length).toBeGreaterThan(1); |
|||
|
|||
for (const { properties, required } of objectJsonSchemas) { |
|||
expect([...required].sort()).toEqual(Object.keys(properties).sort()); |
|||
} |
|||
|
|||
expect(JSON.stringify(jsonSchema)).not.toMatch(/"(max|min)(imum|Length)"/); |
|||
}); |
|||
}); |
|||
@ -0,0 +1,87 @@ |
|||
import { AI_MAX_ACTIVITIES } from '@ghostfolio/common/config'; |
|||
|
|||
import { Type as ActivityType } from '@prisma/client'; |
|||
import { z } from 'zod'; |
|||
|
|||
// The strict mode of the structured outputs requires every property, hence a
|
|||
// value which the file does not show is null instead of being left out. The
|
|||
// values are checked when the rows and the matches are joined to the
|
|||
// activities to import, so that a wrong value or a wrong match skips its rows
|
|||
// only
|
|||
const PRINTED_TEXT = z.string().trim().nullable(); |
|||
|
|||
export const ACTIVITY_ROWS_OUTPUT = z.object({ |
|||
broker: PRINTED_TEXT.describe( |
|||
'The name of the broker or of the bank which issued the file' |
|||
), |
|||
rows: z |
|||
.array( |
|||
z.object({ |
|||
accountName: PRINTED_TEXT.describe( |
|||
'The name or the number of the account, as printed' |
|||
), |
|||
currency: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The currency of the fee and of the unit price, as an ISO 4217 code in upper case' |
|||
), |
|||
date: z |
|||
.string() |
|||
.describe('The date of the activity, as an ISO 8601 date'), |
|||
fee: z.number().describe('The fee of the activity'), |
|||
isin: PRINTED_TEXT.describe('The ISIN of the asset, as printed'), |
|||
name: PRINTED_TEXT.describe( |
|||
'The name of the asset, or the description of a fee, an interest or a liability, as printed' |
|||
), |
|||
quantity: z.number().describe('The quantity of the activity'), |
|||
symbol: PRINTED_TEXT.describe( |
|||
'The ticker symbol of the asset, as printed, not a national number such as a valor, a WKN or a CUSIP' |
|||
), |
|||
type: z.enum(ActivityType).describe('The type of the activity'), |
|||
unitPrice: z.number().describe('The unit price of the activity') |
|||
}) |
|||
) |
|||
.describe(`The activities of the file, at most ${AI_MAX_ACTIVITIES}`) |
|||
}); |
|||
|
|||
export const ROW_MATCHES_OUTPUT = z.object({ |
|||
accounts: z |
|||
.array( |
|||
z.object({ |
|||
accountId: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The id of the account of the user which matches, as given by get-accounts, or null if no account matches' |
|||
), |
|||
id: z.number().describe('The id of the account hint, as given') |
|||
}) |
|||
) |
|||
.describe('One entry for each account hint of the file'), |
|||
assets: z |
|||
.array( |
|||
z.object({ |
|||
currency: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The currency of the asset profile, as given by search-asset-profiles' |
|||
), |
|||
dataSource: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The data source of the asset profile, as given by search-asset-profiles' |
|||
), |
|||
id: z.number().describe('The id of the asset, as given'), |
|||
symbol: z |
|||
.string() |
|||
.nullable() |
|||
.describe( |
|||
'The symbol of the asset profile, as given by search-asset-profiles, or null if no asset profile matches' |
|||
) |
|||
}) |
|||
) |
|||
.describe('One entry for each asset of the file') |
|||
}); |
|||
@ -0,0 +1,957 @@ |
|||
import { McpService } from '@ghostfolio/api/app/endpoints/mcp/mcp.service'; |
|||
import { CallerFacingError } from '@ghostfolio/api/errors/caller-facing.error'; |
|||
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; |
|||
import { PortfolioTableService } from '@ghostfolio/api/services/portfolio-table/portfolio-table.service'; |
|||
import { PropertyService } from '@ghostfolio/api/services/property/property.service'; |
|||
import { |
|||
DEFAULT_LOCALE, |
|||
PROPERTY_API_KEY_OPENROUTER, |
|||
PROPERTY_OPENROUTER_MODEL, |
|||
PROPERTY_OPENROUTER_MODEL_AGENT, |
|||
PROPERTY_OPENROUTER_MODEL_VISION |
|||
} from '@ghostfolio/common/config'; |
|||
import { isValidCustomAssetProfileSymbol } from '@ghostfolio/common/helper'; |
|||
import type { UserWithSettings } from '@ghostfolio/common/types'; |
|||
|
|||
import { Logger } from '@nestjs/common'; |
|||
import { createOpenRouter } from '@openrouter/ai-sdk-provider'; |
|||
import { DataSource, Type as ActivityType } from '@prisma/client'; |
|||
import { generateText, isStepCount, NoObjectGeneratedError, ToolSet } from 'ai'; |
|||
import { z } from 'zod'; |
|||
|
|||
import { ACTIVITY_ROWS_OUTPUT, ROW_MATCHES_OUTPUT } from './ai.schemas'; |
|||
import { AiService } from './ai.service'; |
|||
import { createActivityRow } from './ai.test-utils'; |
|||
import { AccountHint } from './types/account-hint.type'; |
|||
import { AccountToMatch } from './types/account-to-match.type'; |
|||
import { ActivityRow } from './types/activity-row.type'; |
|||
|
|||
// The packages are ECMAScript modules, which Jest cannot load
|
|||
jest.mock('@openrouter/ai-sdk-provider', () => { |
|||
return { createOpenRouter: jest.fn() }; |
|||
}); |
|||
jest.mock('ai', () => { |
|||
return { |
|||
generateText: jest.fn(), |
|||
isStepCount: jest.fn((stepCount: number) => { |
|||
return { stepCount }; |
|||
}), |
|||
NoObjectGeneratedError: class MockNoObjectGeneratedError extends Error { |
|||
public readonly finishReason: string; |
|||
|
|||
public constructor({ finishReason }: { finishReason: string }) { |
|||
super('No object generated'); |
|||
|
|||
this.finishReason = finishReason; |
|||
} |
|||
|
|||
public static isInstance(error: unknown) { |
|||
return error instanceof MockNoObjectGeneratedError; |
|||
} |
|||
}, |
|||
Output: { |
|||
object: jest.fn((options: unknown) => { |
|||
return options; |
|||
}) |
|||
}, |
|||
tool: jest.fn((options: unknown) => { |
|||
return options; |
|||
}) |
|||
}; |
|||
}); |
|||
|
|||
describe('AiService', () => { |
|||
const abortSignal = new AbortController().signal; |
|||
const model = { modelId: 'model' }; |
|||
|
|||
const rows: ActivityRow[] = [ |
|||
createActivityRow({ accountName: null, symbol: null }) |
|||
]; |
|||
|
|||
let aiService: AiService; |
|||
let chat: jest.Mock; |
|||
let mcpService: McpService; |
|||
let properties: Record<string, string>; |
|||
|
|||
beforeEach(() => { |
|||
jest.clearAllMocks(); |
|||
|
|||
jest.spyOn(Logger.prototype, 'log').mockImplementation(); |
|||
|
|||
chat = jest.fn().mockReturnValue(model); |
|||
|
|||
jest |
|||
.mocked(createOpenRouter) |
|||
.mockReturnValue({ chat } as unknown as ReturnType< |
|||
typeof createOpenRouter |
|||
>); |
|||
|
|||
jest.mocked(generateText).mockResolvedValue({ |
|||
output: { broker: 'Swissquote', rows }, |
|||
steps: [{}], |
|||
totalUsage: { totalTokens: 1000 } |
|||
} as unknown as Awaited<ReturnType<typeof generateText>>); |
|||
|
|||
properties = { |
|||
[PROPERTY_API_KEY_OPENROUTER]: 'api-key', |
|||
[PROPERTY_OPENROUTER_MODEL]: 'general-model', |
|||
[PROPERTY_OPENROUTER_MODEL_AGENT]: 'agent-model', |
|||
[PROPERTY_OPENROUTER_MODEL_VISION]: 'vision-model' |
|||
}; |
|||
|
|||
mcpService = { |
|||
getAccounts: jest.fn().mockResolvedValue({ |
|||
content: [{ text: '## Accounts', type: 'text' }] |
|||
}), |
|||
searchAssetProfiles: jest.fn().mockResolvedValue({ |
|||
content: [{ text: '{ "assetProfiles": [] }', type: 'text' }] |
|||
}) |
|||
} as unknown as McpService; |
|||
|
|||
const propertyService = { |
|||
getByKey: jest.fn().mockImplementation((key: string) => { |
|||
return Promise.resolve(properties[key]); |
|||
}) |
|||
} as unknown as PropertyService; |
|||
|
|||
aiService = new AiService( |
|||
{} as ConfigurationService, |
|||
mcpService, |
|||
{} as PortfolioTableService, |
|||
propertyService |
|||
); |
|||
}); |
|||
|
|||
describe('getActivityRowsFromFile', () => { |
|||
it('Passes a PDF file as a file part to the vision model and gives the rows', async () => { |
|||
const buffer = Buffer.from('%PDF-1.7'); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
abortSignal, |
|||
file: { buffer, mediaType: 'application/pdf' }, |
|||
locale: 'de-CH' |
|||
}) |
|||
).resolves.toEqual({ broker: 'Swissquote', rows }); |
|||
|
|||
expect(createOpenRouter).toHaveBeenCalledWith({ apiKey: 'api-key' }); |
|||
|
|||
expect(generateText).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
abortSignal, |
|||
messages: [ |
|||
{ |
|||
content: [ |
|||
{ data: buffer, mediaType: 'application/pdf', type: 'file' } |
|||
], |
|||
role: 'user' |
|||
} |
|||
], |
|||
model, |
|||
output: { schema: ACTIVITY_ROWS_OUTPUT } |
|||
}) |
|||
); |
|||
}); |
|||
|
|||
it('Passes the name of the file as text before the file and gives the broker', async () => { |
|||
const buffer = Buffer.from('%PDF-1.7'); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer, |
|||
mediaType: 'application/pdf', |
|||
name: 'swissquote-2022.pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}) |
|||
).resolves.toEqual({ broker: 'Swissquote', rows }); |
|||
|
|||
expect(generateText).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
messages: [ |
|||
{ |
|||
content: [ |
|||
{ text: 'File name: swissquote-2022.pdf', type: 'text' }, |
|||
{ data: buffer, mediaType: 'application/pdf', type: 'file' } |
|||
], |
|||
role: 'user' |
|||
} |
|||
] |
|||
}) |
|||
); |
|||
}); |
|||
|
|||
it('Passes a CSV file as text', async () => { |
|||
const csv = 'Date;Type;ISIN\n15.01.2026;Kauf;US0378331005'; |
|||
|
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { buffer: Buffer.from(csv), mediaType: 'text/csv' }, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
expect(generateText).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
messages: [{ content: [{ text: csv, type: 'text' }], role: 'user' }] |
|||
}) |
|||
); |
|||
}); |
|||
|
|||
it('Restricts the routing to endpoints with zero data retention, disables the reasoning and passes a PDF file to the model itself', async () => { |
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { buffer: Buffer.from('%PDF-1.7'), mediaType: 'application/pdf' }, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
expect(chat).toHaveBeenCalledWith('vision-model', { |
|||
plugins: [{ id: 'file-parser', pdf: { engine: 'native' } }], |
|||
provider: { require_parameters: true, zdr: true }, |
|||
reasoning: { effort: 'none' }, |
|||
usage: { include: true } |
|||
}); |
|||
}); |
|||
|
|||
it('Falls back to the general model if no vision model is set', async () => { |
|||
delete properties[PROPERTY_OPENROUTER_MODEL_VISION]; |
|||
|
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { buffer: Buffer.from('%PDF-1.7'), mediaType: 'application/pdf' }, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
expect(chat).toHaveBeenCalledWith('general-model', expect.any(Object)); |
|||
}); |
|||
|
|||
it('Gives the locale and the date of today in the instructions', async () => { |
|||
jest.useFakeTimers({ now: new Date('2026-10-11T12:00:00Z') }); |
|||
|
|||
try { |
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}); |
|||
} finally { |
|||
jest.useRealTimers(); |
|||
} |
|||
|
|||
const { instructions } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
expect(instructions).toContain('the locale de-CH'); |
|||
expect(instructions).toContain('Today is 2026-10-11'); |
|||
}); |
|||
|
|||
it('Tells the vision model that a national number of a security is not a ticker symbol', async () => { |
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
const { instructions } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
expect(instructions).toContain( |
|||
'A national number of a security, for example a valor (NKN), a WKN or a CUSIP, is not a ticker symbol.' |
|||
); |
|||
}); |
|||
|
|||
it('Tells the vision model to add the fees of a buy or a sell to its row', async () => { |
|||
await aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
const { instructions } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
expect(instructions).toContain( |
|||
'Add every fee, commission and tax of a buy or a sell, for example a commission, a stamp duty or an exchange fee, to the fee of its row.' |
|||
); |
|||
}); |
|||
|
|||
it('Gives an error for the caller if a cut output reaches the maximum number of output tokens', async () => { |
|||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
|||
|
|||
jest.mocked(generateText).mockRejectedValueOnce( |
|||
new NoObjectGeneratedError({ |
|||
finishReason: 'length' |
|||
} as ConstructorParameters<typeof NoObjectGeneratedError>[0]) |
|||
); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}) |
|||
).rejects.toThrow(CallerFacingError); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalled(); |
|||
}); |
|||
|
|||
it('Gives an error for the caller if an output without any text reaches the maximum number of output tokens', async () => { |
|||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
|||
|
|||
jest.mocked(generateText).mockResolvedValueOnce({ |
|||
finishReason: 'length', |
|||
get output() { |
|||
throw new Error('No output generated'); |
|||
} |
|||
} as unknown as Awaited<ReturnType<typeof generateText>>); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}) |
|||
).rejects.toThrow(CallerFacingError); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalled(); |
|||
}); |
|||
|
|||
it('Passes on every other error of the vision model', async () => { |
|||
const error = new NoObjectGeneratedError({ |
|||
finishReason: 'stop' |
|||
} as ConstructorParameters<typeof NoObjectGeneratedError>[0]); |
|||
|
|||
jest.mocked(generateText).mockRejectedValueOnce(error); |
|||
|
|||
await expect( |
|||
aiService.getActivityRowsFromFile({ |
|||
file: { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}, |
|||
locale: 'de-CH' |
|||
}) |
|||
).rejects.toBe(error); |
|||
}); |
|||
}); |
|||
|
|||
describe('getActivitiesFromFile', () => { |
|||
const file = { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
mediaType: 'application/pdf' |
|||
}; |
|||
|
|||
const user = { |
|||
accounts: [{ currency: 'CHF', id: 'account-id', name: 'Brokerage' }], |
|||
id: 'user-id', |
|||
settings: { settings: { baseCurrency: 'EUR', locale: 'de-CH' } } |
|||
} as unknown as UserWithSettings; |
|||
|
|||
it('Reads the rows of the file and matches them with the accounts, the base currency and the id of the user', async () => { |
|||
const activities = [ |
|||
{ |
|||
currency: 'USD', |
|||
dataSource: 'YAHOO', |
|||
date: '2026-01-15', |
|||
fee: 1, |
|||
quantity: 10, |
|||
symbol: 'AAPL', |
|||
type: ActivityType.BUY, |
|||
unitPrice: 230 |
|||
} |
|||
]; |
|||
|
|||
jest |
|||
.spyOn(aiService, 'getActivityRowsFromFile') |
|||
.mockResolvedValue({ broker: 'Swissquote', rows }); |
|||
|
|||
jest |
|||
.spyOn(aiService, 'getActivitiesFromRows') |
|||
.mockResolvedValue({ activities, rowsWithoutMatch: [] }); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromFile({ abortSignal, file, user }) |
|||
).resolves.toEqual({ |
|||
activities, |
|||
assetProfiles: [], |
|||
rowsWithoutMatch: [] |
|||
}); |
|||
|
|||
expect(aiService.getActivityRowsFromFile).toHaveBeenCalledWith({ |
|||
abortSignal, |
|||
file, |
|||
locale: 'de-CH' |
|||
}); |
|||
|
|||
expect(aiService.getActivitiesFromRows).toHaveBeenCalledWith({ |
|||
abortSignal, |
|||
rows, |
|||
accounts: user.accounts, |
|||
broker: 'Swissquote', |
|||
userCurrency: 'EUR', |
|||
userId: 'user-id' |
|||
}); |
|||
}); |
|||
|
|||
it('Gives a fee, an interest or a liability a custom asset profile with its description as name', async () => { |
|||
const fee = { |
|||
currency: 'CHF', |
|||
date: '2026-01-15', |
|||
fee: 5, |
|||
quantity: 0, |
|||
symbol: 'Custody fee', |
|||
type: ActivityType.FEE, |
|||
unitPrice: 0 |
|||
}; |
|||
|
|||
jest |
|||
.spyOn(aiService, 'getActivityRowsFromFile') |
|||
.mockResolvedValue({ broker: 'Swissquote', rows }); |
|||
|
|||
jest.spyOn(aiService, 'getActivitiesFromRows').mockResolvedValue({ |
|||
activities: [fee, { ...fee, date: '2026-02-15' }], |
|||
rowsWithoutMatch: [] |
|||
}); |
|||
|
|||
const { activities, assetProfiles } = |
|||
await aiService.getActivitiesFromFile({ file, user }); |
|||
|
|||
const [{ symbol }] = assetProfiles; |
|||
|
|||
expect(isValidCustomAssetProfileSymbol(symbol)).toBe(true); |
|||
|
|||
expect(assetProfiles).toEqual([ |
|||
{ |
|||
symbol, |
|||
currency: 'CHF', |
|||
dataSource: DataSource.MANUAL, |
|||
name: 'Custody fee' |
|||
} |
|||
]); |
|||
|
|||
expect(activities).toEqual([ |
|||
{ ...fee, symbol, dataSource: DataSource.MANUAL }, |
|||
{ ...fee, symbol, dataSource: DataSource.MANUAL, date: '2026-02-15' } |
|||
]); |
|||
}); |
|||
|
|||
it('Reads the file in the default locale if the user has no locale', async () => { |
|||
jest |
|||
.spyOn(aiService, 'getActivityRowsFromFile') |
|||
.mockResolvedValue({ broker: null, rows: [] }); |
|||
|
|||
jest |
|||
.spyOn(aiService, 'getActivitiesFromRows') |
|||
.mockResolvedValue({ activities: [], rowsWithoutMatch: [] }); |
|||
|
|||
await aiService.getActivitiesFromFile({ |
|||
file, |
|||
user: { |
|||
...user, |
|||
settings: { settings: { baseCurrency: 'EUR' } } |
|||
} as unknown as UserWithSettings |
|||
}); |
|||
|
|||
expect(aiService.getActivityRowsFromFile).toHaveBeenCalledWith( |
|||
expect.objectContaining({ locale: DEFAULT_LOCALE }) |
|||
); |
|||
}); |
|||
}); |
|||
|
|||
describe('getActivitiesFromRows', () => { |
|||
const accounts: AccountToMatch[] = [ |
|||
{ currency: 'CHF', id: 'account-id', name: 'Brokerage' } |
|||
]; |
|||
|
|||
const userCurrency = 'EUR'; |
|||
const userId = 'user-id'; |
|||
|
|||
function mockRowMatches({ |
|||
accounts: accountMatches = [], |
|||
assetProfilesOfSearch, |
|||
assets: assetMatches = [] |
|||
}: Partial<z.infer<typeof ROW_MATCHES_OUTPUT>> & { |
|||
assetProfilesOfSearch?: { dataSource: string; symbol: string }[]; |
|||
}) { |
|||
jest.mocked(mcpService.searchAssetProfiles).mockResolvedValueOnce({ |
|||
content: [ |
|||
{ |
|||
text: JSON.stringify({ |
|||
assetProfiles: assetProfilesOfSearch ?? assetMatches |
|||
}), |
|||
type: 'text' |
|||
} |
|||
] |
|||
}); |
|||
|
|||
jest.mocked(generateText).mockImplementationOnce(async ({ tools }) => { |
|||
await tools['search-asset-profiles'].execute( |
|||
{ query: 'US0378331005' }, |
|||
undefined |
|||
); |
|||
|
|||
return { |
|||
output: { accounts: accountMatches, assets: assetMatches }, |
|||
steps: [{}, {}], |
|||
totalUsage: { totalTokens: 2000 } |
|||
} as unknown as Awaited<ReturnType<typeof generateText>>; |
|||
}); |
|||
} |
|||
|
|||
it('Gives an activity for each row whose asset matches an asset profile', async () => { |
|||
const row = createActivityRow({ currency: null }); |
|||
|
|||
mockRowMatches({ |
|||
accounts: [{ accountId: 'account-id', id: 0 }], |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ |
|||
activities: [ |
|||
{ |
|||
accountId: 'account-id', |
|||
currency: 'USD', |
|||
dataSource: 'YAHOO', |
|||
date: row.date, |
|||
fee: row.fee, |
|||
quantity: row.quantity, |
|||
symbol: 'AAPL', |
|||
type: ActivityType.BUY, |
|||
unitPrice: row.unitPrice |
|||
} |
|||
], |
|||
rowsWithoutMatch: [] |
|||
}); |
|||
|
|||
expect(Logger.prototype.log).toHaveBeenCalledWith( |
|||
expect.stringContaining( |
|||
'Matched 1 of 1 assets and 1 of 1 account hints with 1 results of 1 searches' |
|||
) |
|||
); |
|||
}); |
|||
|
|||
it('Gives the currency of the row before the currency of the asset profile', async () => { |
|||
mockRowMatches({ |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow({ accountName: null, currency: 'CHF' })] |
|||
}); |
|||
|
|||
expect(activities[0].currency).toBe('CHF'); |
|||
}); |
|||
|
|||
it('Gives the absolute values of a negative fee, quantity and unit price', async () => { |
|||
mockRowMatches({ |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [ |
|||
createActivityRow({ |
|||
accountName: null, |
|||
fee: -1, |
|||
quantity: -10, |
|||
type: ActivityType.SELL, |
|||
unitPrice: -230 |
|||
}) |
|||
] |
|||
}); |
|||
|
|||
expect(activities[0]).toMatchObject({ |
|||
fee: 1, |
|||
quantity: 10, |
|||
unitPrice: 230 |
|||
}); |
|||
}); |
|||
|
|||
it('Gives a row whose asset has no match as a row without a match', async () => { |
|||
const row = createActivityRow({ accountName: null }); |
|||
|
|||
mockRowMatches({ |
|||
assets: [{ currency: null, dataSource: null, id: 0, symbol: null }] |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
|||
}); |
|||
|
|||
it('Gives a row with a data source which is not valid as a row without a match', async () => { |
|||
const row = createActivityRow({ accountName: null }); |
|||
|
|||
mockRowMatches({ |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'yahoo', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
|||
}); |
|||
|
|||
it('Gives a row whose asset profile no search gave as a row without a match and logs the left out match', async () => { |
|||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
|||
|
|||
const row = createActivityRow({ accountName: null }); |
|||
|
|||
mockRowMatches({ |
|||
assetProfilesOfSearch: [{ dataSource: 'YAHOO', symbol: 'AAPL' }], |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL.XX' } |
|||
] |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
|||
|
|||
expect(Logger.prototype.log).toHaveBeenCalledWith( |
|||
expect.stringContaining( |
|||
'Matched 0 of 1 assets and 0 of 0 account hints with 1 results of 1 searches' |
|||
) |
|||
); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
|||
'Left out 1 asset matches which no search gave' |
|||
); |
|||
}); |
|||
|
|||
it('Matches a row without an account name by the broker of the file', async () => { |
|||
mockRowMatches({ |
|||
accounts: [{ accountId: 'account-id', id: 0 }], |
|||
assets: [ |
|||
{ currency: 'CHF', dataSource: 'YAHOO', id: 0, symbol: 'ROG.SW' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
broker: 'Swissquote', |
|||
rows: [ |
|||
createActivityRow({ accountName: null }), |
|||
createActivityRow({ accountName: null, type: ActivityType.SELL }) |
|||
] |
|||
}); |
|||
|
|||
expect(activities.map(({ accountId }) => accountId)).toEqual([ |
|||
'account-id', |
|||
'account-id' |
|||
]); |
|||
|
|||
const { prompt } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
const { accountHints } = JSON.parse(prompt as string) as { |
|||
accountHints: AccountHint[]; |
|||
}; |
|||
|
|||
expect(accountHints).toEqual([ |
|||
{ accountName: null, broker: 'Swissquote', id: 0 } |
|||
]); |
|||
}); |
|||
|
|||
it('Ignores an account id which is not the id of an account of the user', async () => { |
|||
mockRowMatches({ |
|||
accounts: [{ accountId: 'unknown-account-id', id: 0 }], |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
expect(activities[0].accountId).toBeUndefined(); |
|||
}); |
|||
|
|||
it('Gives a fee, an interest or a liability its description as symbol without a model call', async () => { |
|||
const rowsOfNonInvestments = [ |
|||
ActivityType.FEE, |
|||
ActivityType.INTEREST, |
|||
ActivityType.LIABILITY |
|||
].map((type) => { |
|||
return createActivityRow({ |
|||
type, |
|||
accountName: null, |
|||
currency: null, |
|||
isin: null, |
|||
name: `Description of ${type}`, |
|||
symbol: null |
|||
}); |
|||
}); |
|||
|
|||
const { activities, rowsWithoutMatch } = |
|||
await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: rowsOfNonInvestments |
|||
}); |
|||
|
|||
expect(generateText).not.toHaveBeenCalled(); |
|||
|
|||
expect( |
|||
activities.map(({ currency, dataSource, symbol, type }) => { |
|||
return { currency, dataSource, symbol, type }; |
|||
}) |
|||
).toEqual( |
|||
rowsOfNonInvestments.map(({ name, type }) => { |
|||
return { |
|||
type, |
|||
currency: userCurrency, |
|||
dataSource: undefined, |
|||
symbol: name |
|||
}; |
|||
}) |
|||
); |
|||
|
|||
expect(rowsWithoutMatch).toEqual([]); |
|||
}); |
|||
|
|||
it('Gives a fee without a currency the currency of its account', async () => { |
|||
mockRowMatches({ |
|||
accounts: [{ accountId: 'account-id', id: 0 }] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [ |
|||
createActivityRow({ |
|||
currency: null, |
|||
isin: null, |
|||
name: 'Custody fee', |
|||
symbol: null, |
|||
type: ActivityType.FEE |
|||
}) |
|||
] |
|||
}); |
|||
|
|||
expect(activities[0]).toMatchObject({ |
|||
accountId: 'account-id', |
|||
currency: 'CHF' |
|||
}); |
|||
}); |
|||
|
|||
it('Gives a fee without a description as a row without a match', async () => { |
|||
const row = createActivityRow({ |
|||
accountName: null, |
|||
isin: null, |
|||
name: null, |
|||
symbol: null, |
|||
type: ActivityType.FEE |
|||
}); |
|||
|
|||
await expect( |
|||
aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [row] |
|||
}) |
|||
).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] }); |
|||
}); |
|||
|
|||
it('Passes each asset and each account hint one time and no account to the model', async () => { |
|||
mockRowMatches({ |
|||
assets: [ |
|||
{ currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'AAPL' } |
|||
] |
|||
}); |
|||
|
|||
const { activities } = await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
broker: 'Swissquote', |
|||
rows: [ |
|||
createActivityRow(), |
|||
createActivityRow({ type: ActivityType.SELL }), |
|||
createActivityRow({ |
|||
accountName: 'Depot 1', |
|||
type: ActivityType.DIVIDEND |
|||
}) |
|||
] |
|||
}); |
|||
|
|||
expect(activities).toHaveLength(3); |
|||
|
|||
const { prompt } = jest.mocked(generateText).mock.calls[0][0]; |
|||
|
|||
expect(JSON.parse(prompt as string)).toEqual({ |
|||
accountHints: [ |
|||
{ accountName: 'Brokerage', broker: 'Swissquote', id: 0 }, |
|||
{ accountName: 'Depot 1', broker: 'Swissquote', id: 1 } |
|||
], |
|||
assets: [ |
|||
{ |
|||
currency: 'USD', |
|||
id: 0, |
|||
isin: 'US0378331005', |
|||
name: 'Apple Inc.', |
|||
symbol: 'AAPL' |
|||
} |
|||
] |
|||
}); |
|||
}); |
|||
|
|||
it('Searches the asset profiles with the tool of the server of the Model Context Protocol', async () => { |
|||
mockRowMatches({}); |
|||
|
|||
await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
const { tools } = jest.mocked(generateText).mock.calls[0][0] as { |
|||
tools: ToolSet; |
|||
}; |
|||
|
|||
await expect( |
|||
tools['search-asset-profiles'].execute( |
|||
{ query: 'US0378331005' }, |
|||
undefined |
|||
) |
|||
).resolves.toBe('{ "assetProfiles": [] }'); |
|||
|
|||
expect(mcpService.searchAssetProfiles).toHaveBeenCalledWith({ |
|||
userId, |
|||
query: 'US0378331005' |
|||
}); |
|||
}); |
|||
|
|||
it('Gets the accounts with the tool of the server of the Model Context Protocol without values', async () => { |
|||
mockRowMatches({}); |
|||
|
|||
await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
const { tools } = jest.mocked(generateText).mock.calls[0][0] as { |
|||
tools: ToolSet; |
|||
}; |
|||
|
|||
await expect(tools['get-accounts'].execute({}, undefined)).resolves.toBe( |
|||
'## Accounts' |
|||
); |
|||
|
|||
expect(mcpService.getAccounts).toHaveBeenCalledWith({ |
|||
userId, |
|||
withValues: false |
|||
}); |
|||
}); |
|||
|
|||
it('Restricts the routing to endpoints with zero data retention, disables the reasoning, limits the number of steps, requires a tool call in the first step and allows no tool call in the last step', async () => { |
|||
mockRowMatches({}); |
|||
|
|||
await aiService.getActivitiesFromRows({ |
|||
abortSignal, |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
expect(chat).toHaveBeenCalledWith('agent-model', { |
|||
provider: { require_parameters: true, zdr: true }, |
|||
reasoning: { effort: 'none' }, |
|||
usage: { include: true } |
|||
}); |
|||
|
|||
expect(isStepCount).toHaveBeenCalled(); |
|||
|
|||
expect(generateText).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
abortSignal, |
|||
model, |
|||
output: { schema: ROW_MATCHES_OUTPUT } |
|||
}) |
|||
); |
|||
|
|||
const { prepareStep, stopWhen } = jest.mocked(generateText).mock |
|||
.calls[0][0] as { |
|||
prepareStep: (options: { stepNumber: number }) => unknown; |
|||
stopWhen: unknown; |
|||
}; |
|||
|
|||
expect(stopWhen).toBe(jest.mocked(isStepCount).mock.results[0].value); |
|||
|
|||
const [stepCount] = jest.mocked(isStepCount).mock.calls[0]; |
|||
|
|||
expect(prepareStep({ stepNumber: 0 })).toEqual({ |
|||
toolChoice: 'required' |
|||
}); |
|||
expect(prepareStep({ stepNumber: stepCount - 2 })).toBeUndefined(); |
|||
expect(prepareStep({ stepNumber: stepCount - 1 })).toEqual({ |
|||
toolChoice: 'none' |
|||
}); |
|||
}); |
|||
|
|||
it('Falls back to the general model if no agent model is set', async () => { |
|||
delete properties[PROPERTY_OPENROUTER_MODEL_AGENT]; |
|||
|
|||
mockRowMatches({}); |
|||
|
|||
await aiService.getActivitiesFromRows({ |
|||
accounts, |
|||
userCurrency, |
|||
userId, |
|||
rows: [createActivityRow()] |
|||
}); |
|||
|
|||
expect(chat).toHaveBeenCalledWith('general-model', expect.any(Object)); |
|||
}); |
|||
}); |
|||
}); |
|||
@ -0,0 +1,21 @@ |
|||
import { Type as ActivityType } from '@prisma/client'; |
|||
|
|||
import { ActivityRow } from './types/activity-row.type'; |
|||
|
|||
export function createActivityRow( |
|||
overrides: Partial<ActivityRow> = {} |
|||
): ActivityRow { |
|||
return { |
|||
accountName: 'Brokerage', |
|||
currency: 'USD', |
|||
date: '2026-01-15', |
|||
fee: 1, |
|||
isin: 'US0378331005', |
|||
name: 'Apple Inc.', |
|||
quantity: 10, |
|||
symbol: 'AAPL', |
|||
type: ActivityType.BUY, |
|||
unitPrice: 230, |
|||
...overrides |
|||
}; |
|||
} |
|||
@ -0,0 +1,6 @@ |
|||
import { ActivityRow } from './activity-row.type'; |
|||
|
|||
export type AccountHint = Pick<ActivityRow, 'accountName'> & { |
|||
broker: string | null; |
|||
id: number; |
|||
}; |
|||
@ -0,0 +1,3 @@ |
|||
import { Account } from '@prisma/client'; |
|||
|
|||
export type AccountToMatch = Pick<Account, 'currency' | 'id' | 'name'>; |
|||
@ -0,0 +1,5 @@ |
|||
import { z } from 'zod'; |
|||
|
|||
import { ACTIVITY_ROWS_OUTPUT } from '../ai.schemas'; |
|||
|
|||
export type ActivityRow = z.infer<typeof ACTIVITY_ROWS_OUTPUT>['rows'][number]; |
|||
@ -0,0 +1,6 @@ |
|||
import { ActivityRow } from './activity-row.type'; |
|||
|
|||
export type AssetToMatch = Pick< |
|||
ActivityRow, |
|||
'currency' | 'isin' | 'name' | 'symbol' |
|||
> & { id: number }; |
|||
@ -0,0 +1,45 @@ |
|||
import { isBrowserUserAgent } from './user-agent.helper'; |
|||
|
|||
describe('isBrowserUserAgent', () => { |
|||
it('should tell that the user agent of a browser is one', () => { |
|||
// Chrome on macOS
|
|||
expect( |
|||
isBrowserUserAgent( |
|||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36' |
|||
) |
|||
).toBe(true); |
|||
|
|||
// Chrome on Android
|
|||
expect( |
|||
isBrowserUserAgent( |
|||
'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Mobile Safari/537.36' |
|||
) |
|||
).toBe(true); |
|||
|
|||
// Firefox on Windows
|
|||
expect( |
|||
isBrowserUserAgent( |
|||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:143.0) Gecko/20100101 Firefox/143.0' |
|||
) |
|||
).toBe(true); |
|||
|
|||
// Safari on iPhone
|
|||
expect( |
|||
isBrowserUserAgent( |
|||
'Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Mobile/15E148 Safari/604.1' |
|||
) |
|||
).toBe(true); |
|||
}); |
|||
|
|||
it('should tell that the user agent of another client is not one', () => { |
|||
expect(isBrowserUserAgent('curl/8.7.1')).toBe(false); |
|||
expect(isBrowserUserAgent('node')).toBe(false); |
|||
expect(isBrowserUserAgent('python-requests/2.32.3')).toBe(false); |
|||
expect(isBrowserUserAgent('Dart/3.5 (dart:io)')).toBe(false); |
|||
}); |
|||
|
|||
it('should tell that a missing user agent is not one', () => { |
|||
expect(isBrowserUserAgent()).toBe(false); |
|||
expect(isBrowserUserAgent('')).toBe(false); |
|||
}); |
|||
}); |
|||
@ -0,0 +1,14 @@ |
|||
const PREFIX_OF_BROWSER_USER_AGENT = 'Mozilla/'; |
|||
|
|||
/** |
|||
* Tells if the user agent is the one of a browser. Every major browser starts |
|||
* its user agent with the same prefix for compatibility, while other clients, |
|||
* such as scripts and native apps, give the name of their library instead. |
|||
* The value is for analytics only, because a client can send any user agent. |
|||
*/ |
|||
export function isBrowserUserAgent(userAgent?: string) { |
|||
return ( |
|||
typeof userAgent === 'string' && |
|||
userAgent.startsWith(PREFIX_OF_BROWSER_USER_AGENT) |
|||
); |
|||
} |
|||
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
@ -0,0 +1,18 @@ |
|||
import { Type as ActivityType } from '@ghostfolio/prisma/enums'; |
|||
|
|||
/** |
|||
* An activity as Ghostfolio AI reads it from a file, with the asset and the |
|||
* account as printed |
|||
*/ |
|||
export interface AiActivityRow { |
|||
accountName?: string | null; |
|||
currency?: string | null; |
|||
date: string; |
|||
fee: number; |
|||
isin?: string | null; |
|||
name?: string | null; |
|||
quantity: number; |
|||
symbol?: string | null; |
|||
type: ActivityType; |
|||
unitPrice: number; |
|||
} |
|||
@ -0,0 +1,12 @@ |
|||
import type { |
|||
CreateAssetProfileWithMarketDataDto, |
|||
CreateOrderDto |
|||
} from '@ghostfolio/common/dtos'; |
|||
|
|||
import { AiActivityRow } from '../ai-activity-row.interface'; |
|||
|
|||
export interface AiActivitiesResponse { |
|||
activities: CreateOrderDto[]; |
|||
assetProfiles: CreateAssetProfileWithMarketDataDto[]; |
|||
rowsWithoutMatch: AiActivityRow[]; |
|||
} |
|||
@ -0,0 +1,5 @@ |
|||
-- CreateEnum |
|||
CREATE TYPE "RequestClient" AS ENUM ('API', 'MCP', 'SELF_HOSTING', 'WEB'); |
|||
|
|||
-- AlterTable |
|||
ALTER TABLE "Analytics" ADD COLUMN "lastRequestClient" "RequestClient"; |
|||
Loading…
Reference in new issue