From d936fb62fde09a1dd0ac48430187513de1100c5d Mon Sep 17 00:00:00 2001 From: mister-ideas <36502095+mister-ideas@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:05:23 +0200 Subject: [PATCH 1/2] Feature/add environment variable to configure interval of hourly market data gathering --- CHANGELOG.md | 4 + README.md | 1 + .../configuration/configuration.service.ts | 4 + apps/api/src/services/cron/cron.module.ts | 4 + .../src/services/cron/cron.service.spec.ts | 140 ++++++++++++++++++ apps/api/src/services/cron/cron.service.ts | 68 +++++++-- .../interfaces/environment.interface.ts | 1 + libs/common/src/lib/config.ts | 1 + 8 files changed, 214 insertions(+), 9 deletions(-) create mode 100644 apps/api/src/services/cron/cron.service.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e35621f0b..4eb5ff2d54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added the `DATA_GATHERING_INTERVAL_MINUTES` environment variable to configure the interval of the market data gathering of the asset profiles with the hourly data gathering frequency + ### Changed - Improved the usability of the activities page by eliminating the reload on canceling the import activities dialog diff --git a/README.md b/README.md index 00ae13e290..d1b88fcf54 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Find answers to commonly asked questions about self-hosting Ghostfolio in our [F | `API_KEY_COINGECKO_DEMO` | `string` (optional) |   | The _CoinGecko_ Demo API key | | `API_KEY_COINGECKO_PRO` | `string` (optional) | | The _CoinGecko_ Pro API key | | `DATABASE_URL` | `string` | | The database connection URL. If using a connection pooler, use the pooled connection URL here. e.g. `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB}` | +| `DATA_GATHERING_INTERVAL_MINUTES` | `number` (optional) | `60` | The interval in minutes of the market data gathering of the asset profiles with the hourly data gathering frequency. The data providers apply their own rate limits, so keep this value as high as your use case allows | | `DIRECT_URL` | `string` (optional) | | The direct database connection URL used by the _Prisma CLI_ (e.g. for schema migrations) and seeding, bypassing any connection poolers (falls back to `DATABASE_URL`) | | `ENABLE_FEATURE_AUTH_TOKEN` | `boolean` (optional) | `true` | Enables authentication via security token | | `ENABLE_FEATURE_MCP` | `boolean` (optional) | `false` | Enables the server of the _Model Context Protocol_ (MCP) at `/mcp` (experimental) | diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts index bbd1a319b6..ae89da4775 100644 --- a/apps/api/src/services/configuration/configuration.service.ts +++ b/apps/api/src/services/configuration/configuration.service.ts @@ -2,6 +2,7 @@ import { environment } from '@ghostfolio/api/environments/environment'; import { Environment } from '@ghostfolio/api/services/interfaces/environment.interface'; import { CACHE_TTL_NO_CACHE, + DEFAULT_DATA_GATHERING_INTERVAL_MINUTES, DEFAULT_HOST, DEFAULT_PORT, DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY, @@ -55,6 +56,9 @@ export class ConfigurationService { API_KEY_RAPID_API: str({ default: '' }), CACHE_QUOTES_TTL: num({ default: ms('1 minute') }), CACHE_TTL: num({ default: CACHE_TTL_NO_CACHE }), + DATA_GATHERING_INTERVAL_MINUTES: num({ + default: DEFAULT_DATA_GATHERING_INTERVAL_MINUTES + }), DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }), DATA_SOURCE_FEAR_AND_GREED_INDEX_STOCKS: str({ default: DataSource.MANUAL diff --git a/apps/api/src/services/cron/cron.module.ts b/apps/api/src/services/cron/cron.module.ts index bc14990c15..227c7c7a46 100644 --- a/apps/api/src/services/cron/cron.module.ts +++ b/apps/api/src/services/cron/cron.module.ts @@ -12,6 +12,7 @@ import { TwitterBotModule } from '@ghostfolio/api/services/twitter-bot/twitter-b import { TwitterBotService } from '@ghostfolio/api/services/twitter-bot/twitter-bot.service'; import { Logger, Module } from '@nestjs/common'; +import { SchedulerRegistry } from '@nestjs/schedule'; import { CronService } from './cron.service'; @@ -30,6 +31,7 @@ import { CronService } from './cron.service'; ConfigurationService, DataGatheringService, PropertyService, + SchedulerRegistry, StatisticsGatheringService, TwitterBotService, UserService @@ -39,6 +41,7 @@ import { CronService } from './cron.service'; configurationService: ConfigurationService, dataGatheringService: DataGatheringService, propertyService: PropertyService, + schedulerRegistry: SchedulerRegistry, statisticsGatheringService: StatisticsGatheringService, twitterBotService: TwitterBotService, userService: UserService @@ -53,6 +56,7 @@ import { CronService } from './cron.service'; configurationService, dataGatheringService, propertyService, + schedulerRegistry, statisticsGatheringService, twitterBotService, userService diff --git a/apps/api/src/services/cron/cron.service.spec.ts b/apps/api/src/services/cron/cron.service.spec.ts new file mode 100644 index 0000000000..4f0af3a5e9 --- /dev/null +++ b/apps/api/src/services/cron/cron.service.spec.ts @@ -0,0 +1,140 @@ +import { SchedulerRegistry } from '@nestjs/schedule'; + +import { CronService } from './cron.service'; + +describe('CronService', () => { + let configurationService: { get: jest.Mock }; + let cronService: CronService; + let dataGatheringService: { + gatherHourlyMarketData: jest.Mock; + gatherRecentMarketData: jest.Mock; + }; + let propertyService: { getByKey: jest.Mock }; + let schedulerRegistry: SchedulerRegistry; + + const createCronService = (intervalInMinutes: number) => { + configurationService.get.mockReturnValue(intervalInMinutes); + + return new CronService( + configurationService as never, + dataGatheringService as never, + propertyService as never, + schedulerRegistry, + {} as never, + {} as never, + {} as never + ); + }; + + beforeEach(() => { + jest.useFakeTimers(); + + configurationService = { get: jest.fn() }; + dataGatheringService = { + gatherHourlyMarketData: jest.fn().mockResolvedValue(undefined), + gatherRecentMarketData: jest.fn().mockResolvedValue(undefined) + }; + propertyService = { getByKey: jest.fn().mockResolvedValue(undefined) }; + schedulerRegistry = new SchedulerRegistry(); + }); + + afterEach(() => { + cronService?.onModuleDestroy(); + + jest.useRealTimers(); + }); + + it('gathers the market data every hour by default', async () => { + cronService = createCronService(60); + cronService.onModuleInit(); + + await jest.advanceTimersByTimeAsync(59 * 60 * 1000); + expect(dataGatheringService.gatherHourlyMarketData).not.toHaveBeenCalled(); + + await jest.advanceTimersByTimeAsync(60 * 1000); + expect(dataGatheringService.gatherHourlyMarketData).toHaveBeenCalledTimes( + 1 + ); + expect(dataGatheringService.gatherRecentMarketData).toHaveBeenCalledTimes( + 1 + ); + }); + + it('gathers the market data in the configured interval', async () => { + cronService = createCronService(5); + cronService.onModuleInit(); + + await jest.advanceTimersByTimeAsync(15 * 60 * 1000); + + expect(dataGatheringService.gatherHourlyMarketData).toHaveBeenCalledTimes( + 3 + ); + }); + + it('does not gather more often than once per minute', async () => { + cronService = createCronService(0); + cronService.onModuleInit(); + + await jest.advanceTimersByTimeAsync(60 * 1000); + + expect(dataGatheringService.gatherHourlyMarketData).toHaveBeenCalledTimes( + 1 + ); + }); + + it('does not gather the market data if the data gathering is disabled', async () => { + propertyService.getByKey.mockResolvedValue(false); + + cronService = createCronService(60); + await cronService.runHourlyMarketDataGathering(); + + expect(dataGatheringService.gatherHourlyMarketData).not.toHaveBeenCalled(); + }); + + it('skips a run while the previous run is still in progress', async () => { + let finish: () => void = () => undefined; + + dataGatheringService.gatherHourlyMarketData.mockReturnValue( + new Promise((resolve) => { + finish = resolve; + }) + ); + + cronService = createCronService(60); + + const firstRun = cronService.runHourlyMarketDataGathering(); + + // Let the first run reach the pending data gathering + await jest.advanceTimersByTimeAsync(0); + + await cronService.runHourlyMarketDataGathering(); + + expect(dataGatheringService.gatherHourlyMarketData).toHaveBeenCalledTimes( + 1 + ); + + finish(); + await firstRun; + + await cronService.runHourlyMarketDataGathering(); + + expect(dataGatheringService.gatherHourlyMarketData).toHaveBeenCalledTimes( + 2 + ); + }); + + it('keeps running after a failed run', async () => { + dataGatheringService.gatherHourlyMarketData.mockRejectedValueOnce( + new Error('Provider unavailable') + ); + + cronService = createCronService(60); + + await cronService.runHourlyMarketDataGathering(); + await cronService.runHourlyMarketDataGathering(); + + expect(dataGatheringService.gatherHourlyMarketData).toHaveBeenCalledTimes( + 2 + ); + }); +}); diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts index b3e60ae59b..ef24ad01c8 100644 --- a/apps/api/src/services/cron/cron.service.ts +++ b/apps/api/src/services/cron/cron.service.ts @@ -12,18 +12,28 @@ import { } from '@ghostfolio/common/config'; import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; -import { Injectable } from '@nestjs/common'; -import { Cron, CronExpression } from '@nestjs/schedule'; +import { + Injectable, + Logger, + OnModuleDestroy, + OnModuleInit +} from '@nestjs/common'; +import { Cron, CronExpression, SchedulerRegistry } from '@nestjs/schedule'; +import ms from 'ms'; @Injectable() -export class CronService { - private static readonly EVERY_HOUR_AT_RANDOM_MINUTE = `${new Date().getMinutes()} * * * *`; +export class CronService implements OnModuleDestroy, OnModuleInit { + private static readonly HOURLY_MARKET_DATA_GATHERING_JOB_NAME = + 'HOURLY_MARKET_DATA_GATHERING'; private static readonly EVERY_SUNDAY_AT_LUNCH_TIME = '0 12 * * 0'; + private isHourlyMarketDataGatheringRunning = false; + public constructor( private readonly configurationService: ConfigurationService, private readonly dataGatheringService: DataGatheringService, private readonly propertyService: PropertyService, + private readonly schedulerRegistry: SchedulerRegistry, private readonly statisticsGatheringService: StatisticsGatheringService, private readonly twitterBotService: TwitterBotService, private readonly userService: UserService @@ -36,11 +46,51 @@ export class CronService { } } - @Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE) - public async runEveryHourAtRandomMinute() { - if (await this.isDataGatheringEnabled()) { - await this.dataGatheringService.gatherHourlyMarketData(); - await this.dataGatheringService.gatherRecentMarketData(); + public onModuleInit() { + const intervalInMinutes = Math.max( + 1, + this.configurationService.get('DATA_GATHERING_INTERVAL_MINUTES') + ); + + // The interval starts at the application start, which is equivalent to the + // former hourly cron job running at the minute of the application start + this.schedulerRegistry.addInterval( + CronService.HOURLY_MARKET_DATA_GATHERING_JOB_NAME, + setInterval( + () => { + void this.runHourlyMarketDataGathering(); + }, + ms(`${intervalInMinutes} minutes`) + ) + ); + } + + public onModuleDestroy() { + try { + this.schedulerRegistry.deleteInterval( + CronService.HOURLY_MARKET_DATA_GATHERING_JOB_NAME + ); + } catch {} + } + + public async runHourlyMarketDataGathering() { + // Skip the run if the previous one is still in progress to avoid + // overlapping requests to the data providers + if (this.isHourlyMarketDataGatheringRunning) { + return; + } + + this.isHourlyMarketDataGatheringRunning = true; + + try { + if (await this.isDataGatheringEnabled()) { + await this.dataGatheringService.gatherHourlyMarketData(); + await this.dataGatheringService.gatherRecentMarketData(); + } + } catch (error) { + Logger.error(error, 'CronService'); + } finally { + this.isHourlyMarketDataGatheringRunning = false; } } diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index 7f9d7813cc..b7e41f11b9 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -12,6 +12,7 @@ export interface Environment extends CleanedEnvAccessors { API_KEY_RAPID_API: string; CACHE_QUOTES_TTL: number; CACHE_TTL: number; + DATA_GATHERING_INTERVAL_MINUTES: number; DATA_SOURCE_EXCHANGE_RATES: string; DATA_SOURCE_FEAR_AND_GREED_INDEX_STOCKS: string; DATA_SOURCE_IMPORT: string; diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 61008f1fc7..7703303956 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -98,6 +98,7 @@ export const PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE_PRIORITY_LOW = export const STATISTICS_GATHERING_QUEUE = 'STATISTICS_GATHERING_QUEUE'; export const DEFAULT_CURRENCY = 'USD'; +export const DEFAULT_DATA_GATHERING_INTERVAL_MINUTES = 60; export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy'; export const DEFAULT_DATE_RANGE: DateRange = 'max'; export const DEFAULT_HOST = '0.0.0.0'; From da29d2b23667193a12c7ed52051397cf10968fc1 Mon Sep 17 00:00:00 2001 From: mister-ideas <36502095+mister-ideas@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:45:31 +0200 Subject: [PATCH 2/2] Move the changelog entry to the unreleased section --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7d45223e3..6740dae6fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added the `DATA_GATHERING_INTERVAL_MINUTES` environment variable to configure the interval of the market data gathering of the asset profiles with the hourly data gathering frequency - Added a link to the holdings tab in the holdings table on the overview tab of the home page (experimental) ### Changed @@ -36,7 +37,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added the `DATA_GATHERING_INTERVAL_MINUTES` environment variable to configure the interval of the market data gathering of the asset profiles with the hourly data gathering frequency - Added a _Storybook_ story for the file drop area component ### Changed