diff --git a/CHANGELOG.md b/CHANGELOG.md index 29d71c9fe..1cc28c4eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Extended the asset profile details dialog in the admin control panel to support editing countries for all asset types - Extended the asset profile details dialog in the admin control panel to support editing sectors for all asset types +- Migrated the data collection for the _Open Startup_ (`/open`) page to the queue design pattern - Improved the language localization for German (`de`) - Upgraded `lodash` from version `4.17.23` to `4.18.1` diff --git a/apps/api/src/app/activities/activities.module.ts b/apps/api/src/app/activities/activities.module.ts index 7476ad66a..f4e592c3f 100644 --- a/apps/api/src/app/activities/activities.module.ts +++ b/apps/api/src/app/activities/activities.module.ts @@ -10,7 +10,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data- import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { Module } from '@nestjs/common'; @@ -24,7 +24,7 @@ import { ActivitiesService } from './activities.service'; imports: [ ApiModule, CacheModule, - DataGatheringModule, + DataGatheringQueueModule, DataProviderModule, ExchangeRateDataModule, ImpersonationModule, diff --git a/apps/api/src/app/admin/admin.module.ts b/apps/api/src/app/admin/admin.module.ts index 960a36629..e87df9e74 100644 --- a/apps/api/src/app/admin/admin.module.ts +++ b/apps/api/src/app/admin/admin.module.ts @@ -9,7 +9,7 @@ import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-d import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { Module } from '@nestjs/common'; @@ -24,7 +24,7 @@ import { QueueModule } from './queue/queue.module'; ApiModule, BenchmarkModule, ConfigurationModule, - DataGatheringModule, + DataGatheringQueueModule, DataProviderModule, DemoModule, ExchangeRateDataModule, diff --git a/apps/api/src/app/admin/queue/queue.module.ts b/apps/api/src/app/admin/queue/queue.module.ts index 22d1cefc6..4cfb79492 100644 --- a/apps/api/src/app/admin/queue/queue.module.ts +++ b/apps/api/src/app/admin/queue/queue.module.ts @@ -1,5 +1,6 @@ -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module'; +import { StatisticsGatheringQueueModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module'; import { Module } from '@nestjs/common'; @@ -8,7 +9,11 @@ import { QueueService } from './queue.service'; @Module({ controllers: [QueueController], - imports: [DataGatheringModule, PortfolioSnapshotQueueModule], + imports: [ + DataGatheringQueueModule, + PortfolioSnapshotQueueModule, + StatisticsGatheringQueueModule + ], providers: [QueueService] }) export class QueueModule {} diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.ts index 747c4d6fb..f47b3d3a1 100644 --- a/apps/api/src/app/admin/queue/queue.service.ts +++ b/apps/api/src/app/admin/queue/queue.service.ts @@ -1,7 +1,8 @@ import { DATA_GATHERING_QUEUE, PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE, - QUEUE_JOB_STATUS_LIST + QUEUE_JOB_STATUS_LIST, + STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config'; import { AdminJobs } from '@ghostfolio/common/interfaces'; @@ -15,7 +16,9 @@ export class QueueService { @InjectQueue(DATA_GATHERING_QUEUE) private readonly dataGatheringQueue: Queue, @InjectQueue(PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE) - private readonly portfolioSnapshotQueue: Queue + private readonly portfolioSnapshotQueue: Queue, + @InjectQueue(STATISTICS_GATHERING_QUEUE) + private readonly statisticsGatheringQueue: Queue ) {} public async deleteJob(aId: string) { @@ -38,6 +41,7 @@ export class QueueService { await this.dataGatheringQueue.clean(300, queueStatus); await this.portfolioSnapshotQueue.clean(300, queueStatus); + await this.statisticsGatheringQueue.clean(300, queueStatus); } } @@ -58,13 +62,19 @@ export class QueueService { limit?: number; status?: JobStatus[]; }): Promise { - const [dataGatheringJobs, portfolioSnapshotJobs] = await Promise.all([ - this.dataGatheringQueue.getJobs(status), - this.portfolioSnapshotQueue.getJobs(status) - ]); + const [dataGatheringJobs, portfolioSnapshotJobs, statisticsGatheringJobs] = + await Promise.all([ + this.dataGatheringQueue.getJobs(status), + this.portfolioSnapshotQueue.getJobs(status), + this.statisticsGatheringQueue.getJobs(status) + ]); const jobsWithState = await Promise.all( - [...dataGatheringJobs, ...portfolioSnapshotJobs] + [ + ...dataGatheringJobs, + ...portfolioSnapshotJobs, + ...statisticsGatheringJobs + ] .filter((job) => { return job; }) diff --git a/apps/api/src/app/app.module.ts b/apps/api/src/app/app.module.ts index 8ebe05928..3316f9ce4 100644 --- a/apps/api/src/app/app.module.ts +++ b/apps/api/src/app/app.module.ts @@ -8,7 +8,7 @@ import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-d import { I18nService } from '@ghostfolio/api/services/i18n/i18n.service'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module'; import { BULL_BOARD_ROUTE, @@ -109,7 +109,7 @@ import { UserModule } from './user/user.module'; ConfigModule.forRoot(), ConfigurationModule, CronModule, - DataGatheringModule, + DataGatheringQueueModule, DataProviderModule, EventEmitterModule.forRoot(), EventsModule, diff --git a/apps/api/src/app/endpoints/watchlist/watchlist.module.ts b/apps/api/src/app/endpoints/watchlist/watchlist.module.ts index ce9ae12bb..9b4b960a0 100644 --- a/apps/api/src/app/endpoints/watchlist/watchlist.module.ts +++ b/apps/api/src/app/endpoints/watchlist/watchlist.module.ts @@ -5,7 +5,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data- import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { Module } from '@nestjs/common'; @@ -17,7 +17,7 @@ import { WatchlistService } from './watchlist.service'; controllers: [WatchlistController], imports: [ BenchmarkModule, - DataGatheringModule, + DataGatheringQueueModule, DataProviderModule, ImpersonationModule, MarketDataModule, diff --git a/apps/api/src/app/import/import.module.ts b/apps/api/src/app/import/import.module.ts index ca9b5667b..8aebcfa08 100644 --- a/apps/api/src/app/import/import.module.ts +++ b/apps/api/src/app/import/import.module.ts @@ -12,7 +12,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data- import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { TagModule } from '@ghostfolio/api/services/tag/tag.module'; @@ -29,7 +29,7 @@ import { ImportService } from './import.service'; ApiModule, CacheModule, ConfigurationModule, - DataGatheringModule, + DataGatheringQueueModule, DataProviderModule, ExchangeRateDataModule, MarketDataModule, diff --git a/apps/api/src/app/info/info.module.ts b/apps/api/src/app/info/info.module.ts index 9ded44600..e33c5e0c2 100644 --- a/apps/api/src/app/info/info.module.ts +++ b/apps/api/src/app/info/info.module.ts @@ -8,7 +8,7 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration/conf import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { Module } from '@nestjs/common'; @@ -22,7 +22,7 @@ import { InfoService } from './info.service'; imports: [ BenchmarkModule, ConfigurationModule, - DataGatheringModule, + DataGatheringQueueModule, DataProviderModule, ExchangeRateDataModule, JwtModule.register({ diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.ts index 9b4a4d597..86630db53 100644 --- a/apps/api/src/app/info/info.service.ts +++ b/apps/api/src/app/info/info.service.ts @@ -7,26 +7,24 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate- import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DEFAULT_CURRENCY, - HEADER_KEY_TOKEN, - PROPERTY_BETTER_UPTIME_MONITOR_ID, PROPERTY_COUNTRIES_OF_SUBSCRIBERS, PROPERTY_DEMO_USER_ID, + PROPERTY_DOCKER_HUB_PULLS, + PROPERTY_GITHUB_CONTRIBUTORS, + PROPERTY_GITHUB_STARGAZERS, PROPERTY_IS_READ_ONLY_MODE, PROPERTY_SLACK_COMMUNITY_USERS, + PROPERTY_UPTIME, ghostfolioFearAndGreedIndexDataSourceStocks } from '@ghostfolio/common/config'; -import { - DATE_FORMAT, - encodeDataSource, - extractNumberFromString -} from '@ghostfolio/common/helper'; +import { encodeDataSource } from '@ghostfolio/common/helper'; import { InfoItem, Statistics } from '@ghostfolio/common/interfaces'; import { permissions } from '@ghostfolio/common/permissions'; -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { JwtService } from '@nestjs/jwt'; -import * as cheerio from 'cheerio'; -import { format, subDays } from 'date-fns'; +import { subDays } from 'date-fns'; +import { isNil } from 'lodash'; @Injectable() export class InfoService { @@ -149,68 +147,6 @@ export class InfoService { }); } - private async countDockerHubPulls(): Promise { - try { - const { pull_count } = (await fetch( - 'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio', - { - headers: { 'User-Agent': 'request' }, - signal: AbortSignal.timeout( - this.configurationService.get('REQUEST_TIMEOUT') - ) - } - ).then((res) => res.json())) as { pull_count: number }; - - return pull_count; - } catch (error) { - Logger.error(error, 'InfoService - DockerHub'); - - return undefined; - } - } - - private async countGitHubContributors(): Promise { - try { - const body = await fetch('https://github.com/ghostfolio/ghostfolio', { - signal: AbortSignal.timeout( - this.configurationService.get('REQUEST_TIMEOUT') - ) - }).then((res) => res.text()); - - const $ = cheerio.load(body); - - return extractNumberFromString({ - value: $( - 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter' - ).text() - }); - } catch (error) { - Logger.error(error, 'InfoService - GitHub'); - - return undefined; - } - } - - private async countGitHubStargazers(): Promise { - try { - const { stargazers_count } = (await fetch( - 'https://api.github.com/repos/ghostfolio/ghostfolio', - { - headers: { 'User-Agent': 'request' }, - signal: AbortSignal.timeout( - this.configurationService.get('REQUEST_TIMEOUT') - ) - } - ).then((res) => res.json())) as { stargazers_count: number }; - - return stargazers_count; - } catch (error) { - Logger.error(error, 'InfoService - GitHub'); - - return undefined; - } - } - private async countNewUsers(aDays: number) { return this.userService.count({ where: { @@ -230,12 +166,6 @@ export class InfoService { }); } - private async countSlackCommunityUsers() { - return await this.propertyService.getByKey( - PROPERTY_SLACK_COMMUNITY_USERS - ); - } - private async getDemoAuthToken() { const demoUserId = await this.propertyService.getByKey( PROPERTY_DEMO_USER_ID @@ -267,65 +197,56 @@ export class InfoService { } } catch {} - const activeUsers1d = await this.countActiveUsers(1); - const activeUsers30d = await this.countActiveUsers(30); - const newUsers30d = await this.countNewUsers(30); - - const dockerHubPulls = await this.countDockerHubPulls(); - const gitHubContributors = await this.countGitHubContributors(); - const gitHubStargazers = await this.countGitHubStargazers(); - const slackCommunityUsers = await this.countSlackCommunityUsers(); - const uptime = await this.getUptime(); - - statistics = { + const [ activeUsers1d, activeUsers30d, + newUsers30d, dockerHubPulls, gitHubContributors, gitHubStargazers, - newUsers30d, slackCommunityUsers, uptime + ] = await Promise.all([ + this.countActiveUsers(1), + this.countActiveUsers(30), + this.countNewUsers(30), + this.propertyService.getByKey(PROPERTY_DOCKER_HUB_PULLS), + this.propertyService.getByKey(PROPERTY_GITHUB_CONTRIBUTORS), + this.propertyService.getByKey(PROPERTY_GITHUB_STARGAZERS), + this.propertyService.getByKey(PROPERTY_SLACK_COMMUNITY_USERS), + this.propertyService.getByKey(PROPERTY_UPTIME) + ]); + + statistics = { + activeUsers1d, + activeUsers30d, + newUsers30d, + dockerHubPulls: dockerHubPulls + ? Number.parseInt(dockerHubPulls, 10) + : undefined, + gitHubContributors: gitHubContributors + ? Number.parseInt(gitHubContributors, 10) + : undefined, + gitHubStargazers: gitHubStargazers + ? Number.parseInt(gitHubStargazers, 10) + : undefined, + slackCommunityUsers: slackCommunityUsers + ? Number.parseInt(slackCommunityUsers, 10) + : undefined, + uptime: uptime ? Number.parseFloat(uptime) : undefined }; - await this.redisCacheService.set( - InfoService.CACHE_KEY_STATISTICS, - JSON.stringify(statistics) - ); + if ( + Object.values(statistics).every((value) => { + return !isNil(value); + }) + ) { + await this.redisCacheService.set( + InfoService.CACHE_KEY_STATISTICS, + JSON.stringify(statistics) + ); + } return statistics; } - - private async getUptime(): Promise { - { - try { - const monitorId = await this.propertyService.getByKey( - PROPERTY_BETTER_UPTIME_MONITOR_ID - ); - - const { data } = await fetch( - `https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format( - subDays(new Date(), 90), - DATE_FORMAT - )}&to${format(new Date(), DATE_FORMAT)}`, - { - headers: { - [HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get( - 'API_KEY_BETTER_UPTIME' - )}` - }, - signal: AbortSignal.timeout( - this.configurationService.get('REQUEST_TIMEOUT') - ) - } - ).then((res) => res.json()); - - return data.attributes.availability / 100; - } catch (error) { - Logger.error(error, 'InfoService - Better Stack'); - - return undefined; - } - } - } } diff --git a/apps/api/src/app/portfolio/portfolio.module.ts b/apps/api/src/app/portfolio/portfolio.module.ts index 65a9b71aa..d818195ca 100644 --- a/apps/api/src/app/portfolio/portfolio.module.ts +++ b/apps/api/src/app/portfolio/portfolio.module.ts @@ -17,7 +17,7 @@ import { I18nModule } from '@ghostfolio/api/services/i18n/i18n.module'; import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; @@ -38,7 +38,7 @@ import { RulesService } from './rules.service'; ApiModule, BenchmarkModule, ConfigurationModule, - DataGatheringModule, + DataGatheringQueueModule, DataProviderModule, ExchangeRateDataModule, I18nModule, diff --git a/apps/api/src/events/events.module.ts b/apps/api/src/events/events.module.ts index 772766945..df943a3c9 100644 --- a/apps/api/src/events/events.module.ts +++ b/apps/api/src/events/events.module.ts @@ -3,7 +3,7 @@ import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.mo import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { Module } from '@nestjs/common'; @@ -14,7 +14,7 @@ import { PortfolioChangedListener } from './portfolio-changed.listener'; imports: [ ActivitiesModule, ConfigurationModule, - DataGatheringModule, + DataGatheringQueueModule, DataProviderModule, ExchangeRateDataModule, RedisCacheModule diff --git a/apps/api/src/services/cron/cron.module.ts b/apps/api/src/services/cron/cron.module.ts index 06f9d2caa..bcc9d3360 100644 --- a/apps/api/src/services/cron/cron.module.ts +++ b/apps/api/src/services/cron/cron.module.ts @@ -2,7 +2,8 @@ import { UserModule } from '@ghostfolio/api/app/user/user.module'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { StatisticsGatheringQueueModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module'; import { TwitterBotModule } from '@ghostfolio/api/services/twitter-bot/twitter-bot.module'; import { Module } from '@nestjs/common'; @@ -12,9 +13,10 @@ import { CronService } from './cron.service'; @Module({ imports: [ ConfigurationModule, - DataGatheringModule, + DataGatheringQueueModule, ExchangeRateDataModule, PropertyModule, + StatisticsGatheringQueueModule, TwitterBotModule, UserModule ], diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts index ee91a811e..e680f0063 100644 --- a/apps/api/src/services/cron/cron.service.ts +++ b/apps/api/src/services/cron/cron.service.ts @@ -3,6 +3,7 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; +import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service'; import { TwitterBotService } from '@ghostfolio/api/services/twitter-bot/twitter-bot.service'; import { DATA_GATHERING_QUEUE_PRIORITY_LOW, @@ -25,10 +26,18 @@ export class CronService { private readonly dataGatheringService: DataGatheringService, private readonly exchangeRateDataService: ExchangeRateDataService, private readonly propertyService: PropertyService, + private readonly statisticsGatheringService: StatisticsGatheringService, private readonly twitterBotService: TwitterBotService, private readonly userService: UserService ) {} + @Cron(CronExpression.EVERY_HOUR) + public async runEveryHour() { + if (this.configurationService.get('ENABLE_FEATURE_STATISTICS')) { + await this.statisticsGatheringService.addJobsToQueue(); + } + } + @Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE) public async runEveryHourAtRandomMinute() { if (await this.isDataGatheringEnabled()) { diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.module.ts b/apps/api/src/services/queues/data-gathering/data-gathering.module.ts index f251c8d0c..d163f0d29 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.module.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.module.ts @@ -50,4 +50,4 @@ import { DataGatheringProcessor } from './data-gathering.processor'; providers: [DataGatheringProcessor, DataGatheringService], exports: [BullModule, DataEnhancerModule, DataGatheringService] }) -export class DataGatheringModule {} +export class DataGatheringQueueModule {} diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts new file mode 100644 index 000000000..1818dd4ec --- /dev/null +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts @@ -0,0 +1,36 @@ +import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; +import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; +import { STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config'; + +import { BullAdapter } from '@bull-board/api/bullAdapter'; +import { BullBoardModule } from '@bull-board/nestjs'; +import { BullModule } from '@nestjs/bull'; +import { Module } from '@nestjs/common'; + +import { StatisticsGatheringProcessor } from './statistics-gathering.processor'; +import { StatisticsGatheringService } from './statistics-gathering.service'; + +@Module({ + exports: [BullModule, StatisticsGatheringService], + imports: [ + ...(process.env.ENABLE_FEATURE_BULL_BOARD === 'true' + ? [ + BullBoardModule.forFeature({ + adapter: BullAdapter, + name: STATISTICS_GATHERING_QUEUE, + options: { + displayName: 'Statistics Gathering', + readOnlyMode: process.env.BULL_BOARD_IS_READ_ONLY !== 'false' + } + }) + ] + : []), + BullModule.registerQueue({ + name: STATISTICS_GATHERING_QUEUE + }), + ConfigurationModule, + PropertyModule + ], + providers: [StatisticsGatheringProcessor, StatisticsGatheringService] +}) +export class StatisticsGatheringQueueModule {} diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts new file mode 100644 index 000000000..07fc32585 --- /dev/null +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts @@ -0,0 +1,212 @@ +import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; +import { PropertyService } from '@ghostfolio/api/services/property/property.service'; +import { + GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, + GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME, + HEADER_KEY_TOKEN, + PROPERTY_BETTER_UPTIME_MONITOR_ID, + PROPERTY_DOCKER_HUB_PULLS, + PROPERTY_GITHUB_CONTRIBUTORS, + PROPERTY_GITHUB_STARGAZERS, + PROPERTY_UPTIME, + STATISTICS_GATHERING_QUEUE +} from '@ghostfolio/common/config'; +import { + DATE_FORMAT, + extractNumberFromString +} from '@ghostfolio/common/helper'; + +import { Process, Processor } from '@nestjs/bull'; +import { Injectable, Logger } from '@nestjs/common'; +import * as cheerio from 'cheerio'; +import { format, subDays } from 'date-fns'; + +@Injectable() +@Processor(STATISTICS_GATHERING_QUEUE) +export class StatisticsGatheringProcessor { + public constructor( + private readonly configurationService: ConfigurationService, + private readonly propertyService: PropertyService + ) {} + + @Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME) + public async gatherDockerHubPullsStatistics() { + Logger.log( + 'Docker Hub pulls statistics gathering has been started', + 'StatisticsGatheringProcessor' + ); + + const dockerHubPulls = await this.countDockerHubPulls(); + + await this.propertyService.put({ + key: PROPERTY_DOCKER_HUB_PULLS, + value: String(dockerHubPulls) + }); + + Logger.log( + 'Docker Hub pulls statistics gathering has been completed', + 'StatisticsGatheringProcessor' + ); + } + + @Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME) + public async gatherGitHubContributorsStatistics() { + Logger.log( + 'GitHub contributors statistics gathering has been started', + 'StatisticsGatheringProcessor' + ); + + const gitHubContributors = await this.countGitHubContributors(); + + await this.propertyService.put({ + key: PROPERTY_GITHUB_CONTRIBUTORS, + value: String(gitHubContributors) + }); + + Logger.log( + 'GitHub contributors statistics gathering has been completed', + 'StatisticsGatheringProcessor' + ); + } + + @Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME) + public async gatherGitHubStargazersStatistics() { + Logger.log( + 'GitHub stargazers statistics gathering has been started', + 'StatisticsGatheringProcessor' + ); + + const gitHubStargazers = await this.countGitHubStargazers(); + + await this.propertyService.put({ + key: PROPERTY_GITHUB_STARGAZERS, + value: String(gitHubStargazers) + }); + + Logger.log( + 'GitHub stargazers statistics gathering has been completed', + 'StatisticsGatheringProcessor' + ); + } + + @Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME) + public async gatherUptimeStatistics() { + Logger.log( + 'Uptime statistics gathering has been started', + 'StatisticsGatheringProcessor' + ); + + const uptime = await this.getUptime(); + + await this.propertyService.put({ + key: PROPERTY_UPTIME, + value: String(uptime) + }); + + Logger.log( + 'Uptime statistics gathering has been completed', + 'StatisticsGatheringProcessor' + ); + } + + private async countDockerHubPulls(): Promise { + try { + const { pull_count } = (await fetch( + 'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio', + { + headers: { 'User-Agent': 'request' }, + signal: AbortSignal.timeout( + this.configurationService.get('REQUEST_TIMEOUT') + ) + } + ).then((res) => res.json())) as { pull_count: number }; + + return pull_count; + } catch (error) { + Logger.error(error, 'StatisticsGatheringProcessor - DockerHub'); + + throw error; + } + } + + private async countGitHubContributors(): Promise { + try { + const body = await fetch('https://github.com/ghostfolio/ghostfolio', { + signal: AbortSignal.timeout( + this.configurationService.get('REQUEST_TIMEOUT') + ) + }).then((res) => res.text()); + + const $ = cheerio.load(body); + + const value = $( + 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter' + ).text(); + + if (!value) { + throw new Error('Could not find the contributors count in the page'); + } + + return extractNumberFromString({ + value + }); + } catch (error) { + Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); + + throw error; + } + } + + private async countGitHubStargazers(): Promise { + try { + const { stargazers_count } = (await fetch( + 'https://api.github.com/repos/ghostfolio/ghostfolio', + { + headers: { 'User-Agent': 'request' }, + signal: AbortSignal.timeout( + this.configurationService.get('REQUEST_TIMEOUT') + ) + } + ).then((res) => res.json())) as { stargazers_count: number }; + + return stargazers_count; + } catch (error) { + Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); + + throw error; + } + } + + private async getUptime(): Promise { + try { + const monitorId = await this.propertyService.getByKey( + PROPERTY_BETTER_UPTIME_MONITOR_ID + ); + + const { data } = await fetch( + `https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format( + subDays(new Date(), 90), + DATE_FORMAT + )}&to${format(new Date(), DATE_FORMAT)}`, + { + headers: { + [HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get( + 'API_KEY_BETTER_UPTIME' + )}` + }, + signal: AbortSignal.timeout( + this.configurationService.get('REQUEST_TIMEOUT') + ) + } + ).then((res) => res.json()); + + return data.attributes.availability / 100; + } catch (error) { + Logger.error(error, 'StatisticsGatheringProcessor - Better Stack'); + + throw error; + } + } +} diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts new file mode 100644 index 000000000..68c674688 --- /dev/null +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts @@ -0,0 +1,40 @@ +import { + GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, + GATHER_STATISTICS_PROCESS_JOB_OPTIONS, + GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME, + STATISTICS_GATHERING_QUEUE +} from '@ghostfolio/common/config'; + +import { InjectQueue } from '@nestjs/bull'; +import { Injectable } from '@nestjs/common'; +import { Queue } from 'bull'; + +@Injectable() +export class StatisticsGatheringService { + public constructor( + @InjectQueue(STATISTICS_GATHERING_QUEUE) + private readonly statisticsGatheringQueue: Queue + ) {} + + public async addJobsToQueue() { + return Promise.all( + [ + GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, + GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, + GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME + ].map((jobName) => { + return this.statisticsGatheringQueue.add( + jobName, + {}, + { + ...GATHER_STATISTICS_PROCESS_JOB_OPTIONS, + jobId: jobName + } + ); + }) + ); + } +} diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index b442f2b7a..42e1f6b63 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -75,6 +75,8 @@ export const PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE_PRIORITY_HIGH = 1; export const PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER; +export const STATISTICS_GATHERING_QUEUE = 'STATISTICS_GATHERING_QUEUE'; + export const DEFAULT_CURRENCY = 'USD'; export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy'; export const DEFAULT_HOST = '0.0.0.0'; @@ -183,6 +185,27 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = { removeOnComplete: true }; +export const GATHER_STATISTICS_PROCESS_JOB_OPTIONS: JobOptions = { + attempts: 5, + backoff: { + delay: ms('1 minute'), + type: 'exponential' + }, + removeOnComplete: true +}; + +export const GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME = + 'GATHER_STATISTICS_DOCKER_HUB_PULLS'; + +export const GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME = + 'GATHER_STATISTICS_GITHUB_CONTRIBUTORS'; + +export const GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME = + 'GATHER_STATISTICS_GITHUB_STARGAZERS'; + +export const GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME = + 'GATHER_STATISTICS_UPTIME'; + export const INVESTMENT_ACTIVITY_TYPES = [ Type.BUY, Type.DIVIDEND, @@ -209,6 +232,9 @@ export const PROPERTY_API_KEY_GHOSTFOLIO = 'API_KEY_GHOSTFOLIO'; export const PROPERTY_API_KEY_OPENROUTER = 'API_KEY_OPENROUTER'; export const PROPERTY_BENCHMARKS = 'BENCHMARKS'; export const PROPERTY_BETTER_UPTIME_MONITOR_ID = 'BETTER_UPTIME_MONITOR_ID'; +export const PROPERTY_DOCKER_HUB_PULLS = 'DOCKER_HUB_PULLS'; +export const PROPERTY_GITHUB_CONTRIBUTORS = 'GITHUB_CONTRIBUTORS'; +export const PROPERTY_GITHUB_STARGAZERS = 'GITHUB_STARGAZERS'; export const PROPERTY_COUNTRIES_OF_SUBSCRIBERS = 'COUNTRIES_OF_SUBSCRIBERS'; export const PROPERTY_COUPONS = 'COUPONS'; export const PROPERTY_CURRENCIES = 'CURRENCIES'; @@ -225,6 +251,7 @@ export const PROPERTY_OPENROUTER_MODEL = 'OPENROUTER_MODEL'; export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS'; export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG'; export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE'; +export const PROPERTY_UPTIME = 'UPTIME'; export const QUEUE_JOB_STATUS_LIST = [ 'active', diff --git a/libs/common/src/lib/interfaces/statistics.interface.ts b/libs/common/src/lib/interfaces/statistics.interface.ts index 2852d34ab..670cdaf4c 100644 --- a/libs/common/src/lib/interfaces/statistics.interface.ts +++ b/libs/common/src/lib/interfaces/statistics.interface.ts @@ -5,6 +5,6 @@ export interface Statistics { gitHubContributors: number; gitHubStargazers: number; newUsers30d: number; - slackCommunityUsers: string; + slackCommunityUsers: number; uptime: number; }