Feature/expose PROCESSOR_GATHER_STATISTICS_CONCURRENCY environment variable (#7100)

* Expose PROCESSOR_GATHER_STATISTICS_CONCURRENCY environment variable

* Update changelog

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Exposed the `ENABLE_FEATURE_CRON` environment variable to control scheduled cron job execution
+- Exposed the `PROCESSOR_GATHER_STATISTICS_CONCURRENCY` environment variable to control the concurrency of the statistics gathering queue processor
 
 ### Changed
 

apps/api/src/services/configuration/configuration.service.ts

@@ -6,6 +6,7 @@ import {
   DEFAULT_PORT,
   DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY,
   DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY,
+  DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY,
   DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY,
   DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT
 } from '@ghostfolio/common/config';
@@ -89,6 +90,9 @@ export class ConfigurationService {
       PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: num({
         default: DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY
       }),
+      PROCESSOR_GATHER_STATISTICS_CONCURRENCY: num({
+        default: DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY
+      }),
       PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: num({
         default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY
       }),

apps/api/src/services/interfaces/environment.interface.ts

@@ -45,6 +45,7 @@ export interface Environment extends CleanedEnvAccessors {
   PORT: number;
   PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY: number;
   PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: number;
+  PROCESSOR_GATHER_STATISTICS_CONCURRENCY: number;
   PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: number;
   PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: number;
   REDIS_DB: number;

apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts

@@ -2,6 +2,7 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con
 import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import {
+  DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY,
   GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,
   GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,
   GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,
@@ -35,7 +36,14 @@ export class StatisticsGatheringProcessor {
     private readonly propertyService: PropertyService
   ) {}
 
-  @Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME)
+  @Process({
+    concurrency: parseInt(
+      process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ??
+        DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(),
+      10
+    ),
+    name: GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME
+  })
   public async gatherDockerHubPullsStatistics() {
     this.logger.log('Docker Hub pulls statistics gathering has been started');
 
@@ -49,7 +57,14 @@ export class StatisticsGatheringProcessor {
     this.logger.log('Docker Hub pulls statistics gathering has been completed');
   }
 
-  @Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME)
+  @Process({
+    concurrency: parseInt(
+      process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ??
+        DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(),
+      10
+    ),
+    name: GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME
+  })
   public async gatherGitHubContributorsStatistics() {
     this.logger.log(
       'GitHub contributors statistics gathering has been started'
@@ -67,7 +82,14 @@ export class StatisticsGatheringProcessor {
     );
   }
 
-  @Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME)
+  @Process({
+    concurrency: parseInt(
+      process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ??
+        DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(),
+      10
+    ),
+    name: GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME
+  })
   public async gatherGitHubStargazersStatistics() {
     this.logger.log('GitHub stargazers statistics gathering has been started');
 
@@ -83,7 +105,14 @@ export class StatisticsGatheringProcessor {
     );
   }
 
-  @Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME)
+  @Process({
+    concurrency: parseInt(
+      process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ??
+        DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(),
+      10
+    ),
+    name: GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME
+  })
   public async gatherUptimeStatistics() {
     const monitorId = await this.propertyService.getByKey<string>(
       PROPERTY_BETTER_UPTIME_MONITOR_ID

libs/common/src/lib/config.ts

@@ -88,6 +88,7 @@ export const DEFAULT_PAGE_SIZE = 50;
 export const DEFAULT_PORT = 3333;
 export const DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY = 1;
 export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY = 1;
+export const DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY = 1;
 export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY = 1;
 export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000;