From cda060cfab12debb48cdaf6b5b7181730db50256 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:07:18 +0200 Subject: [PATCH] Fix repeated asset profile gathering on activity creation --- .../app/activities/activities.service.spec.ts | 70 +++++++++++++++++++ .../src/app/activities/activities.service.ts | 6 +- .../asset-profile-changed.listener.spec.ts | 54 ++++++++++++++ .../events/asset-profile-changed.listener.ts | 8 ++- .../data-gathering.service.spec.ts | 41 +++++++++++ .../data-gathering/data-gathering.service.ts | 6 +- libs/common/src/lib/config.ts | 1 + 7 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 apps/api/src/events/asset-profile-changed.listener.spec.ts diff --git a/apps/api/src/app/activities/activities.service.spec.ts b/apps/api/src/app/activities/activities.service.spec.ts index 57bd8ab665..74f7210a40 100644 --- a/apps/api/src/app/activities/activities.service.spec.ts +++ b/apps/api/src/app/activities/activities.service.spec.ts @@ -7,9 +7,12 @@ import { WHERE_ACTIVITY_NOT_DRAFT } from '@ghostfolio/api/helper/activity.helper import { AssetProfileSplitService } from '@ghostfolio/api/services/asset-profile-split/asset-profile-split.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; +import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { TagService } from '@ghostfolio/api/services/tag/tag.service'; import { + GATHER_ASSET_PROFILE_COOLDOWN_IN_MS, + GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, INVESTMENT_ACTIVITY_TYPES, NON_INVESTMENT_ACTIVITY_TYPES } from '@ghostfolio/common/config'; @@ -80,6 +83,73 @@ describe('ActivitiesService', () => { expect(assetProfile.symbol).toBe(symbol); }); + it('retains the completed asset profile job for the duration of the cooldown', async () => { + const addJobToQueue = jest.fn(); + + const service = new ActivitiesService( + null, + null, + null, + null, + { addJobToQueue } as unknown as DataGatheringService, + null, + { emit: jest.fn() } as unknown as EventEmitter2, + null, + null, + { + order: { + create: jest.fn().mockResolvedValue({ + SymbolProfile: { + currency: 'USD', + dataSource: DataSource.YAHOO, + symbol: 'VT' + }, + userId: 'user-id' + }) + } + } as unknown as PrismaService, + null, + { validateTagIds: jest.fn() } as unknown as TagService + ); + + await service.createActivity({ + currency: 'USD', + date: parseDate('2026-10-01'), + fee: 0, + quantity: 1, + SymbolProfile: { + connectOrCreate: { + create: { + currency: 'USD', + dataSource: DataSource.YAHOO, + symbol: 'VT' + }, + where: { + dataSource_symbol: { + dataSource: DataSource.YAHOO, + symbol: 'VT' + } + } + } + }, + type: 'BUY', + unitPrice: 100, + user: { connect: { id: 'user-id' } }, + userId: 'user-id' + }); + + expect(addJobToQueue).toHaveBeenCalledWith( + expect.objectContaining({ + name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, + opts: expect.objectContaining({ + removeOnComplete: { + age: GATHER_ASSET_PROFILE_COOLDOWN_IN_MS / 1000 + } + }) + }) + ); + }); + async function getCreatedAssetProfile({ existingAssetProfile, symbol diff --git a/apps/api/src/app/activities/activities.service.ts b/apps/api/src/app/activities/activities.service.ts index 0dce736f4e..20c9d673c5 100644 --- a/apps/api/src/app/activities/activities.service.ts +++ b/apps/api/src/app/activities/activities.service.ts @@ -26,6 +26,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/sy import { TagService } from '@ghostfolio/api/services/tag/tag.service'; import { DATA_GATHERING_QUEUE_PRIORITY_HIGH, + GATHER_ASSET_PROFILE_COOLDOWN_IN_MS, GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS, NON_INVESTMENT_ACTIVITY_TYPES, @@ -270,7 +271,10 @@ export class ActivitiesService { dataSource: data.SymbolProfile.connectOrCreate.create.dataSource, symbol: data.SymbolProfile.connectOrCreate.create.symbol }), - priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH, + removeOnComplete: { + age: GATHER_ASSET_PROFILE_COOLDOWN_IN_MS / 1000 + } } }); } diff --git a/apps/api/src/events/asset-profile-changed.listener.spec.ts b/apps/api/src/events/asset-profile-changed.listener.spec.ts new file mode 100644 index 0000000000..e1e24c6f79 --- /dev/null +++ b/apps/api/src/events/asset-profile-changed.listener.spec.ts @@ -0,0 +1,54 @@ +import { ActivitiesService } from '@ghostfolio/api/app/activities/activities.service'; +import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; +import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; +import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; +import { GATHER_HISTORICAL_MARKET_DATA_COOLDOWN_IN_MS } from '@ghostfolio/common/config'; +import { parseDate } from '@ghostfolio/common/helper'; + +import { DataSource } from '@prisma/client'; + +import { AssetProfileChangedListener } from './asset-profile-changed.listener'; + +describe('AssetProfileChangedListener', () => { + describe('processAssetProfileChanged', () => { + it('retains the completed exchange rate job for the duration of the cooldown', async () => { + const gatherSymbol = jest.fn(); + + const listener = new AssetProfileChangedListener( + { + getStatisticsByCurrency: jest.fn().mockResolvedValue({ + dateOfFirstActivity: parseDate('2026-01-01') + }) + } as unknown as ActivitiesService, + { + get: jest.fn().mockReturnValue(true) + } as unknown as ConfigurationService, + { gatherSymbol } as unknown as DataGatheringService, + { + getDataSourceForExchangeRates: jest + .fn() + .mockReturnValue(DataSource.YAHOO) + } as unknown as DataProviderService, + { + getCurrencies: jest.fn().mockReturnValue(['EUR', 'USD']) + } as unknown as ExchangeRateDataService + ); + + await listener['processAssetProfileChanged']({ + currency: 'EUR', + dataSource: DataSource.YAHOO, + symbol: 'PPFB.DE' + }); + + expect(gatherSymbol).toHaveBeenCalledWith({ + dataSource: DataSource.YAHOO, + date: parseDate('2026-01-01'), + removeOnComplete: { + age: GATHER_HISTORICAL_MARKET_DATA_COOLDOWN_IN_MS / 1000 + }, + symbol: 'USDEUR' + }); + }); + }); +}); diff --git a/apps/api/src/events/asset-profile-changed.listener.ts b/apps/api/src/events/asset-profile-changed.listener.ts index 15c19ff0ac..ffc4014c68 100644 --- a/apps/api/src/events/asset-profile-changed.listener.ts +++ b/apps/api/src/events/asset-profile-changed.listener.ts @@ -3,7 +3,10 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; -import { DEFAULT_CURRENCY } from '@ghostfolio/common/config'; +import { + DEFAULT_CURRENCY, + GATHER_HISTORICAL_MARKET_DATA_COOLDOWN_IN_MS +} from '@ghostfolio/common/config'; import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; @@ -91,6 +94,9 @@ export class AssetProfileChangedListener { await this.dataGatheringService.gatherSymbol({ dataSource: this.dataProviderService.getDataSourceForExchangeRates(), date: dateOfFirstActivity, + removeOnComplete: { + age: GATHER_HISTORICAL_MARKET_DATA_COOLDOWN_IN_MS / 1000 + }, symbol: `${DEFAULT_CURRENCY}${currency}` }); } diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.spec.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.spec.ts index 4fc80d3d64..95ca49efcc 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.spec.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.spec.ts @@ -186,6 +186,47 @@ describe('DataGatheringService', () => { }); }); + describe('gatherSymbol', () => { + beforeEach(() => { + jest + .spyOn(dataGatheringService as any, 'getSymbolsMax') + .mockResolvedValue([ + { + dataSource: 'YAHOO', + date: parseDate('2026-08-01'), + symbol: 'USDEUR' + } + ]); + }); + + it('does not apply the cooldown by default', async () => { + await dataGatheringService.gatherSymbol({ + dataSource: 'YAHOO', + symbol: 'USDEUR' + }); + + const [jobs] = dataGatheringQueue.addBulk.mock.calls[0]; + + expect(jobs[0].opts.removeOnComplete).toBe(true); + }); + + it('retains its completed jobs for the duration of a given cooldown', async () => { + await dataGatheringService.gatherSymbol({ + dataSource: 'YAHOO', + removeOnComplete: { + age: GATHER_HISTORICAL_MARKET_DATA_COOLDOWN_IN_MS / 1000 + }, + symbol: 'USDEUR' + }); + + const [jobs] = dataGatheringQueue.addBulk.mock.calls[0]; + + expect(jobs[0].opts.removeOnComplete).toEqual({ + age: GATHER_HISTORICAL_MARKET_DATA_COOLDOWN_IN_MS / 1000 + }); + }); + }); + describe('gatherSymbols', () => { it('does not apply the cooldown to a manually triggered gathering', async () => { await dataGatheringService.gatherSymbols({ diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index 2897855b01..5547d4cca6 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -297,8 +297,11 @@ export class DataGatheringService { dataSource, date, force = true, + removeOnComplete, symbol - }: DataGatheringItem) { + }: DataGatheringItem & { + removeOnComplete?: JobOptions['removeOnComplete']; + }) { const dataGatheringItems = (await this.getSymbolsMax()) .filter((dataGatheringItem) => { return ( @@ -314,6 +317,7 @@ export class DataGatheringService { return this.gatherSymbols({ dataGatheringItems, force, + removeOnComplete, priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH }); } diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 48f69fd403..470e3070e6 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -215,6 +215,7 @@ export const DERIVED_CURRENCIES = [ export const E_MAIL_LINE_BREAK = '%0D%0A'; +export const GATHER_ASSET_PROFILE_COOLDOWN_IN_MS = ms('12 hours'); export const GATHER_ASSET_PROFILE_PROCESS_JOB_NAME = 'GATHER_ASSET_PROFILE'; export const GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS: JobOptions = { attempts: 6, // Retries after 1, 3, 7, 15 and 31 minutes (57 minutes in total)