diff --git a/apps/api/src/app/admin/queue/queue.module.ts b/apps/api/src/app/admin/queue/queue.module.ts index 22d1cefc6..a7cc37715 100644 --- a/apps/api/src/app/admin/queue/queue.module.ts +++ b/apps/api/src/app/admin/queue/queue.module.ts @@ -1,5 +1,6 @@ import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module'; +import { StatisticsGatheringModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module'; import { Module } from '@nestjs/common'; @@ -8,7 +9,11 @@ import { QueueService } from './queue.service'; @Module({ controllers: [QueueController], - imports: [DataGatheringModule, PortfolioSnapshotQueueModule], + imports: [ + DataGatheringModule, + PortfolioSnapshotQueueModule, + StatisticsGatheringModule + ], providers: [QueueService] }) export class QueueModule {} diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.ts index 747c4d6fb..f47b3d3a1 100644 --- a/apps/api/src/app/admin/queue/queue.service.ts +++ b/apps/api/src/app/admin/queue/queue.service.ts @@ -1,7 +1,8 @@ import { DATA_GATHERING_QUEUE, PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE, - QUEUE_JOB_STATUS_LIST + QUEUE_JOB_STATUS_LIST, + STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config'; import { AdminJobs } from '@ghostfolio/common/interfaces'; @@ -15,7 +16,9 @@ export class QueueService { @InjectQueue(DATA_GATHERING_QUEUE) private readonly dataGatheringQueue: Queue, @InjectQueue(PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE) - private readonly portfolioSnapshotQueue: Queue + private readonly portfolioSnapshotQueue: Queue, + @InjectQueue(STATISTICS_GATHERING_QUEUE) + private readonly statisticsGatheringQueue: Queue ) {} public async deleteJob(aId: string) { @@ -38,6 +41,7 @@ export class QueueService { await this.dataGatheringQueue.clean(300, queueStatus); await this.portfolioSnapshotQueue.clean(300, queueStatus); + await this.statisticsGatheringQueue.clean(300, queueStatus); } } @@ -58,13 +62,19 @@ export class QueueService { limit?: number; status?: JobStatus[]; }): Promise { - const [dataGatheringJobs, portfolioSnapshotJobs] = await Promise.all([ - this.dataGatheringQueue.getJobs(status), - this.portfolioSnapshotQueue.getJobs(status) - ]); + const [dataGatheringJobs, portfolioSnapshotJobs, statisticsGatheringJobs] = + await Promise.all([ + this.dataGatheringQueue.getJobs(status), + this.portfolioSnapshotQueue.getJobs(status), + this.statisticsGatheringQueue.getJobs(status) + ]); const jobsWithState = await Promise.all( - [...dataGatheringJobs, ...portfolioSnapshotJobs] + [ + ...dataGatheringJobs, + ...portfolioSnapshotJobs, + ...statisticsGatheringJobs + ] .filter((job) => { return job; }) diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.ts index 688a7d653..86630db53 100644 --- a/apps/api/src/app/info/info.service.ts +++ b/apps/api/src/app/info/info.service.ts @@ -24,6 +24,7 @@ import { permissions } from '@ghostfolio/common/permissions'; import { Injectable } from '@nestjs/common'; import { JwtService } from '@nestjs/jwt'; import { subDays } from 'date-fns'; +import { isNil } from 'lodash'; @Injectable() export class InfoService { @@ -235,10 +236,16 @@ export class InfoService { uptime: uptime ? Number.parseFloat(uptime) : undefined }; - await this.redisCacheService.set( - InfoService.CACHE_KEY_STATISTICS, - JSON.stringify(statistics) - ); + if ( + Object.values(statistics).every((value) => { + return !isNil(value); + }) + ) { + await this.redisCacheService.set( + InfoService.CACHE_KEY_STATISTICS, + JSON.stringify(statistics) + ); + } return statistics; } diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts index 6756332e2..735c83044 100644 --- a/apps/api/src/services/cron/cron.service.ts +++ b/apps/api/src/services/cron/cron.service.ts @@ -34,7 +34,7 @@ export class CronService { @Cron(CronExpression.EVERY_HOUR) public async runEveryHour() { - await this.statisticsGatheringService?.addJobToQueue(); + await this.statisticsGatheringService?.addJobsToQueue(); } @Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE) diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts index 003aa90d6..50673f121 100644 --- a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts @@ -1,7 +1,10 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { - GATHER_STATISTICS_PROCESS_JOB_NAME, + GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, + GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME, HEADER_KEY_TOKEN, PROPERTY_BETTER_UPTIME_MONITOR_ID, PROPERTY_DOCKER_HUB_PULLS, @@ -28,46 +31,82 @@ export class StatisticsGatheringProcessor { private readonly propertyService: PropertyService ) {} - @Process(GATHER_STATISTICS_PROCESS_JOB_NAME) - public async gatherStatistics() { + @Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME) + public async gatherDockerHubPullsStatistics() { Logger.log( - 'Statistics gathering has been started', + 'Docker Hub pulls statistics gathering has been started', 'StatisticsGatheringProcessor' ); - const [dockerHubPulls, gitHubContributors, gitHubStargazers, uptime] = - await Promise.all([ - this.countDockerHubPulls(), - this.countGitHubContributors(), - this.countGitHubStargazers(), - this.getUptime() - ]); - - await Promise.all([ - dockerHubPulls !== undefined && - this.propertyService.put({ - key: PROPERTY_DOCKER_HUB_PULLS, - value: String(dockerHubPulls) - }), - gitHubContributors !== undefined && - this.propertyService.put({ - key: PROPERTY_GITHUB_CONTRIBUTORS, - value: String(gitHubContributors) - }), - gitHubStargazers !== undefined && - this.propertyService.put({ - key: PROPERTY_GITHUB_STARGAZERS, - value: String(gitHubStargazers) - }), - uptime !== undefined && - this.propertyService.put({ - key: PROPERTY_UPTIME, - value: String(uptime) - }) - ]); + const dockerHubPulls = await this.countDockerHubPulls(); + + await this.propertyService.put({ + key: PROPERTY_DOCKER_HUB_PULLS, + value: String(dockerHubPulls) + }); + + Logger.log( + 'Docker Hub pulls statistics gathering has been completed', + 'StatisticsGatheringProcessor' + ); + } + + @Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME) + public async gatherGitHubContributorsStatistics() { + Logger.log( + 'GitHub contributors statistics gathering has been started', + 'StatisticsGatheringProcessor' + ); + + const gitHubContributors = await this.countGitHubContributors(); + + await this.propertyService.put({ + key: PROPERTY_GITHUB_CONTRIBUTORS, + value: String(gitHubContributors) + }); + + Logger.log( + 'GitHub contributors statistics gathering has been completed', + 'StatisticsGatheringProcessor' + ); + } + + @Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME) + public async gatherGitHubStargazersStatistics() { + Logger.log( + 'GitHub stargazers statistics gathering has been started', + 'StatisticsGatheringProcessor' + ); + + const gitHubStargazers = await this.countGitHubStargazers(); + + await this.propertyService.put({ + key: PROPERTY_GITHUB_STARGAZERS, + value: String(gitHubStargazers) + }); + + Logger.log( + 'GitHub stargazers statistics gathering has been completed', + 'StatisticsGatheringProcessor' + ); + } + + @Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME) + public async gatherUptimeStatistics() { + Logger.log( + 'Uptime statistics gathering has been started', + 'StatisticsGatheringProcessor' + ); + + const uptime = await this.getUptime(); + + await this.propertyService.put({ + key: PROPERTY_UPTIME, + value: String(uptime) + }); Logger.log( - 'Statistics gathering has been completed', + 'Uptime statistics gathering has been completed', 'StatisticsGatheringProcessor' ); } @@ -88,7 +127,7 @@ export class StatisticsGatheringProcessor { } catch (error) { Logger.error(error, 'StatisticsGatheringProcessor - DockerHub'); - return undefined; + throw error; } } @@ -102,15 +141,29 @@ export class StatisticsGatheringProcessor { const $ = cheerio.load(body); - return extractNumberFromString({ - value: $( + console.log( + $( 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter' ).text() + ); + + const value = $( + 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter' + ).text(); + + if (!value) { + throw new Error( + 'Could not find the number of contributors in the page' + ); + } + + return extractNumberFromString({ + value }); } catch (error) { Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); - return undefined; + throw error; } } @@ -130,7 +183,7 @@ export class StatisticsGatheringProcessor { } catch (error) { Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); - return undefined; + throw error; } } @@ -161,7 +214,7 @@ export class StatisticsGatheringProcessor { } catch (error) { Logger.error(error, 'StatisticsGatheringProcessor - Better Stack'); - return undefined; + throw error; } } } diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts index 7fd406082..34400b1f5 100644 --- a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts @@ -1,6 +1,9 @@ import { - GATHER_STATISTICS_PROCESS_JOB_NAME, + GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, GATHER_STATISTICS_PROCESS_JOB_OPTIONS, + GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME, STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config'; @@ -15,11 +18,40 @@ export class StatisticsGatheringService { private readonly statisticsGatheringQueue: Queue ) {} - public async addJobToQueue() { - return this.statisticsGatheringQueue.add( - GATHER_STATISTICS_PROCESS_JOB_NAME, - {}, - GATHER_STATISTICS_PROCESS_JOB_OPTIONS - ); + public async addJobsToQueue() { + return Promise.all([ + this.statisticsGatheringQueue.add( + GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, + {}, + { + ...GATHER_STATISTICS_PROCESS_JOB_OPTIONS, + jobId: GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME + } + ), + this.statisticsGatheringQueue.add( + GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, + {}, + { + ...GATHER_STATISTICS_PROCESS_JOB_OPTIONS, + jobId: GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME + } + ), + this.statisticsGatheringQueue.add( + GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, + {}, + { + ...GATHER_STATISTICS_PROCESS_JOB_OPTIONS, + jobId: GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME + } + ), + this.statisticsGatheringQueue.add( + GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME, + {}, + { + ...GATHER_STATISTICS_PROCESS_JOB_OPTIONS, + jobId: GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME + } + ) + ]); } } diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 84e8bdda7..81e8f58ce 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -185,7 +185,6 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = { removeOnComplete: true }; -export const GATHER_STATISTICS_PROCESS_JOB_NAME = 'GATHER_STATISTICS'; export const GATHER_STATISTICS_PROCESS_JOB_OPTIONS: JobOptions = { attempts: 5, backoff: { @@ -195,6 +194,18 @@ export const GATHER_STATISTICS_PROCESS_JOB_OPTIONS: JobOptions = { removeOnComplete: true }; +export const GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME = + 'GATHER_STATISTICS_DOCKER_HUB_PULLS'; + +export const GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME = + 'GATHER_STATISTICS_GITHUB_CONTRIBUTORS'; + +export const GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME = + 'GATHER_STATISTICS_GITHUB_STARGAZERS'; + +export const GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME = + 'GATHER_STATISTICS_UPTIME'; + export const INVESTMENT_ACTIVITY_TYPES = [ Type.BUY, Type.DIVIDEND,