From 46d740e381c6852145fbe43005120ceccb5b5c36 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Mon, 22 Jun 2026 21:34:06 +0200 Subject: [PATCH] Task/consolidate exchange rates gathering together with hourly market data (#7095) * Consolidate exchange rates gathering with hourly market data * Update changelog --- CHANGELOG.md | 1 + apps/api/src/app/admin/admin.controller.ts | 4 +- apps/api/src/services/cron/cron.module.ts | 6 - apps/api/src/services/cron/cron.service.ts | 11 +- .../data-gathering/data-gathering.service.ts | 178 +++++++++--------- .../admin-market-data.component.ts | 22 +-- .../admin-market-data/admin-market-data.html | 2 +- libs/ui/src/lib/services/admin.service.ts | 8 +- 8 files changed, 113 insertions(+), 119 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bd562978..8568c74f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Consolidated the exchange rates to be gathered with hourly market data - Improved the language localization for German (`de`) - Upgraded `@openrouter/ai-sdk-provider` from version `2.9.0` to `2.9.1` - Upgraded `undici` from version `7.24.4` to `8.5.0` diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 9eb045fba..a6f6e8b6f 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -86,8 +86,8 @@ export class AdminController { @HasPermission(permissions.accessAdminControl) @Post('gather') @UseGuards(AuthGuard('jwt'), HasPermissionGuard) - public async gather7Days(): Promise { - this.dataGatheringService.gather7Days(); + public async gatherRecentMarketData(): Promise { + this.dataGatheringService.gatherRecentMarketData(); } @HasPermission(permissions.accessAdminControl) diff --git a/apps/api/src/services/cron/cron.module.ts b/apps/api/src/services/cron/cron.module.ts index 3d999e397..bc14990c1 100644 --- a/apps/api/src/services/cron/cron.module.ts +++ b/apps/api/src/services/cron/cron.module.ts @@ -2,8 +2,6 @@ import { UserModule } from '@ghostfolio/api/app/user/user.module'; import { UserService } from '@ghostfolio/api/app/user/user.service'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; -import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; -import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; @@ -21,7 +19,6 @@ import { CronService } from './cron.service'; imports: [ ConfigurationModule, DataGatheringQueueModule, - ExchangeRateDataModule, PropertyModule, StatisticsGatheringQueueModule, TwitterBotModule, @@ -32,7 +29,6 @@ import { CronService } from './cron.service'; inject: [ ConfigurationService, DataGatheringService, - ExchangeRateDataService, PropertyService, StatisticsGatheringService, TwitterBotService, @@ -42,7 +38,6 @@ import { CronService } from './cron.service'; useFactory: ( configurationService: ConfigurationService, dataGatheringService: DataGatheringService, - exchangeRateDataService: ExchangeRateDataService, propertyService: PropertyService, statisticsGatheringService: StatisticsGatheringService, twitterBotService: TwitterBotService, @@ -57,7 +52,6 @@ import { CronService } from './cron.service'; return new CronService( configurationService, dataGatheringService, - exchangeRateDataService, propertyService, statisticsGatheringService, twitterBotService, diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts index 7299cbbf4..b3e60ae59 100644 --- a/apps/api/src/services/cron/cron.service.ts +++ b/apps/api/src/services/cron/cron.service.ts @@ -1,6 +1,5 @@ import { UserService } from '@ghostfolio/api/app/user/user.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; -import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service'; @@ -24,7 +23,6 @@ export class CronService { public constructor( private readonly configurationService: ConfigurationService, private readonly dataGatheringService: DataGatheringService, - private readonly exchangeRateDataService: ExchangeRateDataService, private readonly propertyService: PropertyService, private readonly statisticsGatheringService: StatisticsGatheringService, private readonly twitterBotService: TwitterBotService, @@ -41,16 +39,11 @@ export class CronService { @Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE) public async runEveryHourAtRandomMinute() { if (await this.isDataGatheringEnabled()) { - await this.dataGatheringService.gather7Days(); - await this.dataGatheringService.gatherHourlySymbols(); + await this.dataGatheringService.gatherHourlyMarketData(); + await this.dataGatheringService.gatherRecentMarketData(); } } - @Cron(CronExpression.EVERY_12_HOURS) - public async runEveryTwelveHours() { - await this.exchangeRateDataService.loadCurrencies(); - } - @Cron(CronExpression.EVERY_DAY_AT_5PM) public async runEveryDayAtFivePm() { if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) { diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index ff65ebc83..1a2fa720e 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -69,91 +69,6 @@ export class DataGatheringService { return this.dataGatheringQueue.addBulk(jobs); } - public async gather7Days() { - await this.gatherSymbols({ - dataGatheringItems: await this.getCurrencies7D(), - priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH - }); - - await this.gatherSymbols({ - dataGatheringItems: await this.getSymbols7D({ - withUserSubscription: true - }), - priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM - }); - - await this.gatherSymbols({ - dataGatheringItems: await this.getSymbols7D({ - withUserSubscription: false - }), - priority: DATA_GATHERING_QUEUE_PRIORITY_LOW - }); - } - - public async gatherMax() { - const dataGatheringItems = await this.getSymbolsMax(); - await this.gatherSymbols({ - dataGatheringItems, - priority: DATA_GATHERING_QUEUE_PRIORITY_LOW - }); - } - - public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) { - const dataGatheringItems = (await this.getSymbolsMax()) - .filter((dataGatheringItem) => { - return ( - dataGatheringItem.dataSource === dataSource && - dataGatheringItem.symbol === symbol - ); - }) - .map((item) => ({ - ...item, - date: date ?? item.date - })); - - await this.gatherSymbols({ - dataGatheringItems, - force: true, - priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH - }); - } - - public async gatherSymbolForDate({ - dataSource, - date, - symbol - }: { date: Date } & AssetProfileIdentifier) { - try { - const historicalData = await this.dataProviderService.getHistoricalRaw({ - assetProfileIdentifiers: [{ dataSource, symbol }], - from: date, - to: date - }); - - const marketPrice = - historicalData[getAssetProfileIdentifier({ dataSource, symbol })][ - format(date, DATE_FORMAT) - ].marketPrice; - - if (marketPrice) { - return await this.prismaService.marketData.upsert({ - create: { - dataSource, - date, - marketPrice, - symbol - }, - update: { marketPrice }, - where: { dataSource_date_symbol: { dataSource, date, symbol } } - }); - } - } catch (error) { - this.logger.error(error); - } finally { - return undefined; - } - } - public async gatherAssetProfiles( aAssetProfileIdentifiers?: AssetProfileIdentifier[] ) { @@ -282,7 +197,13 @@ export class DataGatheringService { } } - public async gatherHourlySymbols() { + public async gatherHourlyMarketData() { + try { + await this.exchangeRateDataService.loadCurrencies(); + } catch (error) { + this.logger.error('Could not gather exchange rates', error); + } + const assetProfileIdentifiers = await this.getHourlyAssetProfileIdentifiers(); @@ -322,6 +243,91 @@ export class DataGatheringService { } } + public async gatherMax() { + const dataGatheringItems = await this.getSymbolsMax(); + await this.gatherSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW + }); + } + + public async gatherRecentMarketData() { + await this.gatherSymbols({ + dataGatheringItems: await this.getCurrencies7D(), + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); + + await this.gatherSymbols({ + dataGatheringItems: await this.getSymbols7D({ + withUserSubscription: true + }), + priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM + }); + + await this.gatherSymbols({ + dataGatheringItems: await this.getSymbols7D({ + withUserSubscription: false + }), + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW + }); + } + + public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) { + const dataGatheringItems = (await this.getSymbolsMax()) + .filter((dataGatheringItem) => { + return ( + dataGatheringItem.dataSource === dataSource && + dataGatheringItem.symbol === symbol + ); + }) + .map((item) => ({ + ...item, + date: date ?? item.date + })); + + await this.gatherSymbols({ + dataGatheringItems, + force: true, + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); + } + + public async gatherSymbolForDate({ + dataSource, + date, + symbol + }: { date: Date } & AssetProfileIdentifier) { + try { + const historicalData = await this.dataProviderService.getHistoricalRaw({ + assetProfileIdentifiers: [{ dataSource, symbol }], + from: date, + to: date + }); + + const marketPrice = + historicalData[getAssetProfileIdentifier({ dataSource, symbol })][ + format(date, DATE_FORMAT) + ].marketPrice; + + if (marketPrice) { + return await this.prismaService.marketData.upsert({ + create: { + dataSource, + date, + marketPrice, + symbol + }, + update: { marketPrice }, + where: { dataSource_date_symbol: { dataSource, date, symbol } } + }); + } + } catch (error) { + this.logger.error(error); + } finally { + return undefined; + } + } + public async gatherSymbols({ dataGatheringItems, force = false, diff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts b/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts index 511d6df98..ae0f8fda2 100644 --- a/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts +++ b/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts @@ -305,17 +305,6 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit { ); } - protected onGather7Days() { - this.adminService - .gather7Days() - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe(() => { - setTimeout(() => { - window.location.reload(); - }, 300); - }); - } - protected onGatherMax() { this.adminService .gatherMax() @@ -334,6 +323,17 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit { .subscribe(); } + protected onGatherRecentMarketData() { + this.adminService + .gatherRecentMarketData() + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(() => { + setTimeout(() => { + window.location.reload(); + }, 300); + }); + } + protected onOpenAssetProfileDialog({ dataSource, symbol diff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.html b/apps/client/src/app/components/admin-market-data/admin-market-data.html index e86e3e631..270722a4b 100644 --- a/apps/client/src/app/components/admin-market-data/admin-market-data.html +++ b/apps/client/src/app/components/admin-market-data/admin-market-data.html @@ -220,7 +220,7 @@ class="no-max-width" xPosition="before" > -