diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dd4209e1..671c9f935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Exposed the `ENABLE_FEATURE_CRON` environment variable to control scheduled cron job execution +- Exposed the `PROCESSOR_GATHER_STATISTICS_CONCURRENCY` environment variable to control the concurrency of the statistics gathering queue processor ### Changed diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts index 328620164..703a90c3a 100644 --- a/apps/api/src/services/configuration/configuration.service.ts +++ b/apps/api/src/services/configuration/configuration.service.ts @@ -6,6 +6,7 @@ import { DEFAULT_PORT, DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY, DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY, + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY, DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY, DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT } from '@ghostfolio/common/config'; @@ -89,6 +90,9 @@ export class ConfigurationService { PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: num({ default: DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY }), + PROCESSOR_GATHER_STATISTICS_CONCURRENCY: num({ + default: DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY + }), PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: num({ default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY }), diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index 45654a2e3..11bdeabf6 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -45,6 +45,7 @@ export interface Environment extends CleanedEnvAccessors { PORT: number; PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY: number; PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: number; + PROCESSOR_GATHER_STATISTICS_CONCURRENCY: number; PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: number; PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: number; REDIS_DB: number; diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts index 7eefc101f..82f362d25 100644 --- a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts @@ -2,6 +2,7 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY, GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, @@ -35,7 +36,14 @@ export class StatisticsGatheringProcessor { private readonly propertyService: PropertyService ) {} - @Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME) + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ?? + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(), + 10 + ), + name: GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME + }) public async gatherDockerHubPullsStatistics() { this.logger.log('Docker Hub pulls statistics gathering has been started'); @@ -49,7 +57,14 @@ export class StatisticsGatheringProcessor { this.logger.log('Docker Hub pulls statistics gathering has been completed'); } - @Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME) + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ?? + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(), + 10 + ), + name: GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME + }) public async gatherGitHubContributorsStatistics() { this.logger.log( 'GitHub contributors statistics gathering has been started' @@ -67,7 +82,14 @@ export class StatisticsGatheringProcessor { ); } - @Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME) + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ?? + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(), + 10 + ), + name: GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME + }) public async gatherGitHubStargazersStatistics() { this.logger.log('GitHub stargazers statistics gathering has been started'); @@ -83,7 +105,14 @@ export class StatisticsGatheringProcessor { ); } - @Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME) + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ?? + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(), + 10 + ), + name: GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME + }) public async gatherUptimeStatistics() { const monitorId = await this.propertyService.getByKey( PROPERTY_BETTER_UPTIME_MONITOR_ID diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 4924aaeea..e21271ca1 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -88,6 +88,7 @@ export const DEFAULT_PAGE_SIZE = 50; export const DEFAULT_PORT = 3333; export const DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY = 1; export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY = 1; +export const DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY = 1; export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY = 1; export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000;