mirror of https://github.com/ghostfolio/ghostfolio
Thomas Kaul
4 days ago
12 changed files with 644 additions and 9 deletions
@ -0,0 +1,205 @@ |
|||
import { HAS_PERMISSION_KEY } from '@ghostfolio/api/decorators/has-permission.decorator'; |
|||
import { CustomThrottlerGuard } from '@ghostfolio/api/guards/custom-throttler.guard'; |
|||
import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'; |
|||
import { ApiService } from '@ghostfolio/api/services/api/api.service'; |
|||
import { |
|||
MCP_MAX_ACTIVITIES, |
|||
THROTTLE_AI_ACTIVITIES_LIMIT, |
|||
THROTTLE_DAILY_TTL |
|||
} from '@ghostfolio/common/config'; |
|||
import { permissions } from '@ghostfolio/common/permissions'; |
|||
import type { RequestWithUser } from '@ghostfolio/common/types'; |
|||
|
|||
import { HttpException, Logger } from '@nestjs/common'; |
|||
import { GUARDS_METADATA } from '@nestjs/common/constants'; |
|||
import { |
|||
THROTTLER_LIMIT, |
|||
THROTTLER_TRACKER, |
|||
THROTTLER_TTL |
|||
} from '@nestjs/throttler/dist/throttler.constants'; |
|||
import type { Response } from 'express'; |
|||
import { EventEmitter } from 'node:events'; |
|||
|
|||
import { AiController } from './ai.controller'; |
|||
import { AiService } from './ai.service'; |
|||
|
|||
// The packages are ECMAScript modules, which Jest cannot load
|
|||
jest.mock('@openrouter/ai-sdk-provider', () => { |
|||
return {}; |
|||
}); |
|||
jest.mock('ai', () => { |
|||
return {}; |
|||
}); |
|||
|
|||
describe('AiController', () => { |
|||
const pdfFile = { |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
originalname: 'statement.pdf' |
|||
}; |
|||
|
|||
const user = { id: 'user-id' }; |
|||
|
|||
let aiController: AiController; |
|||
let aiService: AiService; |
|||
|
|||
function createCsvFile(dataRowCount: number) { |
|||
return { |
|||
buffer: Buffer.from( |
|||
[ |
|||
'Date,Symbol', |
|||
...Array.from({ length: dataRowCount }, () => { |
|||
return '2026-01-15,AAPL'; |
|||
}) |
|||
].join('\n') |
|||
), |
|||
originalname: 'statement.csv' |
|||
}; |
|||
} |
|||
|
|||
function createResponse() { |
|||
return Object.assign(new EventEmitter(), { |
|||
writableFinished: false |
|||
}) as unknown as Response; |
|||
} |
|||
|
|||
function getMetadataOfMethod<T>(metadataKey: string) { |
|||
return Reflect.getMetadata( |
|||
metadataKey, |
|||
AiController.prototype.getActivitiesFromFile |
|||
) as T; |
|||
} |
|||
|
|||
beforeEach(() => { |
|||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(); |
|||
|
|||
aiService = { |
|||
getActivitiesFromFile: jest |
|||
.fn() |
|||
.mockResolvedValue({ activities: [], rowsWithoutMatch: [] }) |
|||
} as unknown as AiService; |
|||
|
|||
aiController = new AiController( |
|||
aiService, |
|||
{} as ApiService, |
|||
{ |
|||
user |
|||
} as unknown as RequestWithUser |
|||
); |
|||
}); |
|||
|
|||
describe('getActivitiesFromFile', () => { |
|||
it('Requires the permission and limits the requests of each user per day', () => { |
|||
expect(getMetadataOfMethod<string>(HAS_PERMISSION_KEY)).toBe( |
|||
permissions.importActivitiesWithAi |
|||
); |
|||
|
|||
const guards = getMetadataOfMethod<unknown[]>(GUARDS_METADATA); |
|||
|
|||
expect(guards.slice(1)).toEqual([ |
|||
HasPermissionGuard, |
|||
CustomThrottlerGuard |
|||
]); |
|||
|
|||
expect(getMetadataOfMethod<number>(`${THROTTLER_LIMIT}default`)).toBe( |
|||
THROTTLE_AI_ACTIVITIES_LIMIT |
|||
); |
|||
|
|||
expect(getMetadataOfMethod<number>(`${THROTTLER_TTL}default`)).toBe( |
|||
THROTTLE_DAILY_TTL |
|||
); |
|||
|
|||
expect( |
|||
getMetadataOfMethod<(request: unknown) => string>( |
|||
`${THROTTLER_TRACKER}default` |
|||
)({ ip: '127.0.0.1', user }) |
|||
).toBe(user.id); |
|||
}); |
|||
|
|||
it('Refuses a request without a file or with a file of another type', async () => { |
|||
for (const file of [ |
|||
undefined, |
|||
{ buffer: Buffer.from('Date,Symbol'), originalname: 'statement.txt' } |
|||
]) { |
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), file) |
|||
).rejects.toThrow(HttpException); |
|||
} |
|||
|
|||
expect(aiService.getActivitiesFromFile).not.toHaveBeenCalled(); |
|||
}); |
|||
|
|||
it(`Refuses a CSV file with more than ${MCP_MAX_ACTIVITIES} rows before a model call`, async () => { |
|||
await expect( |
|||
aiController.getActivitiesFromFile( |
|||
createResponse(), |
|||
createCsvFile(MCP_MAX_ACTIVITIES + 1) |
|||
) |
|||
).rejects.toThrow(HttpException); |
|||
|
|||
expect(aiService.getActivitiesFromFile).not.toHaveBeenCalled(); |
|||
|
|||
expect(Logger.prototype.warn).toHaveBeenCalledWith( |
|||
`Rejected a CSV file with ${MCP_MAX_ACTIVITIES + 1} rows` |
|||
); |
|||
}); |
|||
|
|||
it(`Accepts a CSV file with ${MCP_MAX_ACTIVITIES} rows`, async () => { |
|||
const file = createCsvFile(MCP_MAX_ACTIVITIES); |
|||
|
|||
await aiController.getActivitiesFromFile(createResponse(), file); |
|||
|
|||
expect(aiService.getActivitiesFromFile).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
file: { buffer: file.buffer, mediaType: 'text/csv' } |
|||
}) |
|||
); |
|||
}); |
|||
|
|||
it('Passes the file with its media type and the user of the request', async () => { |
|||
await expect( |
|||
aiController.getActivitiesFromFile(createResponse(), pdfFile) |
|||
).resolves.toEqual({ activities: [], rowsWithoutMatch: [] }); |
|||
|
|||
expect(aiService.getActivitiesFromFile).toHaveBeenCalledWith( |
|||
expect.objectContaining({ |
|||
user, |
|||
file: { buffer: pdfFile.buffer, mediaType: 'application/pdf' } |
|||
}) |
|||
); |
|||
|
|||
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
|||
.mock.calls; |
|||
|
|||
expect(abortSignal).toBeInstanceOf(AbortSignal); |
|||
}); |
|||
|
|||
it('Stops the models if the client closes the request before the response', async () => { |
|||
const response = createResponse(); |
|||
|
|||
await aiController.getActivitiesFromFile(response, pdfFile); |
|||
|
|||
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
|||
.mock.calls; |
|||
|
|||
expect(abortSignal.aborted).toBe(false); |
|||
|
|||
response.emit('close'); |
|||
|
|||
expect(abortSignal.aborted).toBe(true); |
|||
}); |
|||
|
|||
it('Does not stop the models when the response is finished', async () => { |
|||
const response = createResponse(); |
|||
|
|||
await aiController.getActivitiesFromFile(response, pdfFile); |
|||
|
|||
const [[{ abortSignal }]] = jest.mocked(aiService.getActivitiesFromFile) |
|||
.mock.calls; |
|||
|
|||
Object.assign(response, { writableFinished: true }); |
|||
response.emit('close'); |
|||
|
|||
expect(abortSignal.aborted).toBe(false); |
|||
}); |
|||
}); |
|||
}); |
|||
@ -0,0 +1,83 @@ |
|||
import { |
|||
getDataRowCountOfCsvFile, |
|||
getMediaTypeOfActivityFile |
|||
} from './ai.helper'; |
|||
|
|||
describe('getDataRowCountOfCsvFile', () => { |
|||
it('Gives the number of the lines which are not empty, without the header', () => { |
|||
expect( |
|||
getDataRowCountOfCsvFile( |
|||
Buffer.from('Date,Symbol\n2026-01-15,AAPL\n\n2026-01-16,MSFT\n') |
|||
) |
|||
).toBe(2); |
|||
}); |
|||
|
|||
it('Counts a line with a line break in quotes as one line', () => { |
|||
expect( |
|||
getDataRowCountOfCsvFile( |
|||
Buffer.from('Date,Comment\n2026-01-15,"First\nsecond"\n') |
|||
) |
|||
).toBe(1); |
|||
}); |
|||
|
|||
it('Gives 0 for a file with a header only or without content', () => { |
|||
expect(getDataRowCountOfCsvFile(Buffer.from('Date,Symbol\n'))).toBe(0); |
|||
expect(getDataRowCountOfCsvFile(Buffer.from(''))).toBe(0); |
|||
}); |
|||
}); |
|||
|
|||
describe('getMediaTypeOfActivityFile', () => { |
|||
it('Identifies a JPEG, a PDF and a PNG file by the first bytes', () => { |
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from([0xff, 0xd8, 0xff, 0xe0]), |
|||
fileName: 'statement' |
|||
}) |
|||
).toBe('image/jpeg'); |
|||
|
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from('%PDF-1.7'), |
|||
fileName: 'statement.csv' |
|||
}) |
|||
).toBe('application/pdf'); |
|||
|
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), |
|||
fileName: 'statement.jpg' |
|||
}) |
|||
).toBe('image/png'); |
|||
}); |
|||
|
|||
it('Identifies a CSV file by the extension and the content in UTF-8', () => { |
|||
for (const fileName of ['statement.csv', 'STATEMENT.CSV']) { |
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
fileName, |
|||
buffer: Buffer.from('Datum;Stück\n15.01.2026;10\n') |
|||
}) |
|||
).toBe('text/csv'); |
|||
} |
|||
}); |
|||
|
|||
it('Refuses a CSV file which is not in UTF-8', () => { |
|||
expect( |
|||
getMediaTypeOfActivityFile({ |
|||
buffer: Buffer.from([0x53, 0x74, 0xfc, 0x63, 0x6b]), |
|||
fileName: 'statement.csv' |
|||
}) |
|||
).toBeUndefined(); |
|||
}); |
|||
|
|||
it('Refuses any other file', () => { |
|||
for (const { buffer, fileName } of [ |
|||
{ buffer: Buffer.from('Date,Symbol'), fileName: 'statement.txt' }, |
|||
{ buffer: Buffer.from('<html></html>'), fileName: 'statement.pdf' }, |
|||
{ buffer: Buffer.from([0xff, 0xd8]), fileName: 'statement.jpg' }, |
|||
{ buffer: Buffer.from(''), fileName: 'statement.png' } |
|||
]) { |
|||
expect(getMediaTypeOfActivityFile({ buffer, fileName })).toBeUndefined(); |
|||
} |
|||
}); |
|||
}); |
|||
@ -0,0 +1,52 @@ |
|||
import { isUtf8 } from 'node:buffer'; |
|||
import { parse as parseCsv } from 'papaparse'; |
|||
|
|||
const JPEG_SIGNATURE = [0xff, 0xd8, 0xff]; |
|||
const PDF_SIGNATURE = [0x25, 0x50, 0x44, 0x46, 0x2d]; |
|||
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; |
|||
|
|||
/** |
|||
* Gives the number of the lines of a CSV file which are not empty, without |
|||
* the first line. A line before the header is counted as a data row, hence |
|||
* the number is never too low. |
|||
*/ |
|||
export function getDataRowCountOfCsvFile(buffer: Buffer) { |
|||
const { data } = parseCsv(buffer.toString('utf8'), { skipEmptyLines: true }); |
|||
|
|||
return Math.max(data.length - 1, 0); |
|||
} |
|||
|
|||
/** |
|||
* Gives the media type of a file which Ghostfolio AI reads. A JPEG, a PDF or |
|||
* a PNG file is identified by its first bytes, a CSV file by its extension |
|||
* and by its content in UTF-8, as the media type which a client sends for a |
|||
* CSV file varies. |
|||
*/ |
|||
export function getMediaTypeOfActivityFile({ |
|||
buffer, |
|||
fileName |
|||
}: { |
|||
buffer: Buffer; |
|||
fileName: string; |
|||
}) { |
|||
if (startsWith(buffer, JPEG_SIGNATURE)) { |
|||
return 'image/jpeg'; |
|||
} else if (startsWith(buffer, PDF_SIGNATURE)) { |
|||
return 'application/pdf'; |
|||
} else if (startsWith(buffer, PNG_SIGNATURE)) { |
|||
return 'image/png'; |
|||
} else if (fileName.toLowerCase().endsWith('.csv') && isUtf8(buffer)) { |
|||
return 'text/csv'; |
|||
} |
|||
|
|||
return undefined; |
|||
} |
|||
|
|||
function startsWith(buffer: Buffer, signature: number[]) { |
|||
return ( |
|||
buffer.length >= signature.length && |
|||
signature.every((byte, index) => { |
|||
return buffer[index] === byte; |
|||
}) |
|||
); |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
import { Type as ActivityType } from '@ghostfolio/prisma/enums'; |
|||
|
|||
/** |
|||
* An activity as Ghostfolio AI reads it from a file, with the asset and the |
|||
* account as printed |
|||
*/ |
|||
export interface AiActivityRow { |
|||
accountName?: string | null; |
|||
currency?: string | null; |
|||
date: string; |
|||
fee: number; |
|||
isin?: string | null; |
|||
name?: string | null; |
|||
quantity: number; |
|||
symbol?: string | null; |
|||
type: ActivityType; |
|||
unitPrice: number; |
|||
} |
|||
@ -0,0 +1,8 @@ |
|||
import type { CreateOrderDto } from '@ghostfolio/common/dtos'; |
|||
|
|||
import { AiActivityRow } from '../ai-activity-row.interface'; |
|||
|
|||
export interface AiActivitiesResponse { |
|||
activities: CreateOrderDto[]; |
|||
rowsWithoutMatch: AiActivityRow[]; |
|||
} |
|||
Loading…
Reference in new issue