diff --git a/apps/api/src/app/app.module.ts b/apps/api/src/app/app.module.ts index 8ebe05928..fb2965b46 100644 --- a/apps/api/src/app/app.module.ts +++ b/apps/api/src/app/app.module.ts @@ -10,6 +10,7 @@ import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module'; +import { StatisticsGatheringModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module'; import { BULL_BOARD_ROUTE, DEFAULT_LANGUAGE_CODE, @@ -165,6 +166,9 @@ import { UserModule } from './user/user.module'; serveRoot: '/.well-known' }), SitemapModule, + ...(process.env.ENABLE_FEATURE_STATISTICS === 'true' + ? [StatisticsGatheringModule] + : []), SubscriptionModule, SymbolModule, TagsModule, diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.ts index 9b4a4d597..688a7d653 100644 --- a/apps/api/src/app/info/info.service.ts +++ b/apps/api/src/app/info/info.service.ts @@ -7,26 +7,23 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate- import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DEFAULT_CURRENCY, - HEADER_KEY_TOKEN, - PROPERTY_BETTER_UPTIME_MONITOR_ID, PROPERTY_COUNTRIES_OF_SUBSCRIBERS, PROPERTY_DEMO_USER_ID, + PROPERTY_DOCKER_HUB_PULLS, + PROPERTY_GITHUB_CONTRIBUTORS, + PROPERTY_GITHUB_STARGAZERS, PROPERTY_IS_READ_ONLY_MODE, PROPERTY_SLACK_COMMUNITY_USERS, + PROPERTY_UPTIME, ghostfolioFearAndGreedIndexDataSourceStocks } from '@ghostfolio/common/config'; -import { - DATE_FORMAT, - encodeDataSource, - extractNumberFromString -} from '@ghostfolio/common/helper'; +import { encodeDataSource } from '@ghostfolio/common/helper'; import { InfoItem, Statistics } from '@ghostfolio/common/interfaces'; import { permissions } from '@ghostfolio/common/permissions'; -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { JwtService } from '@nestjs/jwt'; -import * as cheerio from 'cheerio'; -import { format, subDays } from 'date-fns'; +import { subDays } from 'date-fns'; @Injectable() export class InfoService { @@ -149,68 +146,6 @@ export class InfoService { }); } - private async countDockerHubPulls(): Promise { - try { - const { pull_count } = (await fetch( - 'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio', - { - headers: { 'User-Agent': 'request' }, - signal: AbortSignal.timeout( - this.configurationService.get('REQUEST_TIMEOUT') - ) - } - ).then((res) => res.json())) as { pull_count: number }; - - return pull_count; - } catch (error) { - Logger.error(error, 'InfoService - DockerHub'); - - return undefined; - } - } - - private async countGitHubContributors(): Promise { - try { - const body = await fetch('https://github.com/ghostfolio/ghostfolio', { - signal: AbortSignal.timeout( - this.configurationService.get('REQUEST_TIMEOUT') - ) - }).then((res) => res.text()); - - const $ = cheerio.load(body); - - return extractNumberFromString({ - value: $( - 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter' - ).text() - }); - } catch (error) { - Logger.error(error, 'InfoService - GitHub'); - - return undefined; - } - } - - private async countGitHubStargazers(): Promise { - try { - const { stargazers_count } = (await fetch( - 'https://api.github.com/repos/ghostfolio/ghostfolio', - { - headers: { 'User-Agent': 'request' }, - signal: AbortSignal.timeout( - this.configurationService.get('REQUEST_TIMEOUT') - ) - } - ).then((res) => res.json())) as { stargazers_count: number }; - - return stargazers_count; - } catch (error) { - Logger.error(error, 'InfoService - GitHub'); - - return undefined; - } - } - private async countNewUsers(aDays: number) { return this.userService.count({ where: { @@ -230,12 +165,6 @@ export class InfoService { }); } - private async countSlackCommunityUsers() { - return await this.propertyService.getByKey( - PROPERTY_SLACK_COMMUNITY_USERS - ); - } - private async getDemoAuthToken() { const demoUserId = await this.propertyService.getByKey( PROPERTY_DEMO_USER_ID @@ -267,25 +196,43 @@ export class InfoService { } } catch {} - const activeUsers1d = await this.countActiveUsers(1); - const activeUsers30d = await this.countActiveUsers(30); - const newUsers30d = await this.countNewUsers(30); - - const dockerHubPulls = await this.countDockerHubPulls(); - const gitHubContributors = await this.countGitHubContributors(); - const gitHubStargazers = await this.countGitHubStargazers(); - const slackCommunityUsers = await this.countSlackCommunityUsers(); - const uptime = await this.getUptime(); - - statistics = { + const [ activeUsers1d, activeUsers30d, + newUsers30d, dockerHubPulls, gitHubContributors, gitHubStargazers, - newUsers30d, slackCommunityUsers, uptime + ] = await Promise.all([ + this.countActiveUsers(1), + this.countActiveUsers(30), + this.countNewUsers(30), + this.propertyService.getByKey(PROPERTY_DOCKER_HUB_PULLS), + this.propertyService.getByKey(PROPERTY_GITHUB_CONTRIBUTORS), + this.propertyService.getByKey(PROPERTY_GITHUB_STARGAZERS), + this.propertyService.getByKey(PROPERTY_SLACK_COMMUNITY_USERS), + this.propertyService.getByKey(PROPERTY_UPTIME) + ]); + + statistics = { + activeUsers1d, + activeUsers30d, + newUsers30d, + dockerHubPulls: dockerHubPulls + ? Number.parseInt(dockerHubPulls, 10) + : undefined, + gitHubContributors: gitHubContributors + ? Number.parseInt(gitHubContributors, 10) + : undefined, + gitHubStargazers: gitHubStargazers + ? Number.parseInt(gitHubStargazers, 10) + : undefined, + slackCommunityUsers: slackCommunityUsers + ? Number.parseInt(slackCommunityUsers, 10) + : undefined, + uptime: uptime ? Number.parseFloat(uptime) : undefined }; await this.redisCacheService.set( @@ -295,37 +242,4 @@ export class InfoService { return statistics; } - - private async getUptime(): Promise { - { - try { - const monitorId = await this.propertyService.getByKey( - PROPERTY_BETTER_UPTIME_MONITOR_ID - ); - - const { data } = await fetch( - `https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format( - subDays(new Date(), 90), - DATE_FORMAT - )}&to${format(new Date(), DATE_FORMAT)}`, - { - headers: { - [HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get( - 'API_KEY_BETTER_UPTIME' - )}` - }, - signal: AbortSignal.timeout( - this.configurationService.get('REQUEST_TIMEOUT') - ) - } - ).then((res) => res.json()); - - return data.attributes.availability / 100; - } catch (error) { - Logger.error(error, 'InfoService - Better Stack'); - - return undefined; - } - } - } } diff --git a/apps/api/src/services/cron/cron.module.ts b/apps/api/src/services/cron/cron.module.ts index 06f9d2caa..9867517c1 100644 --- a/apps/api/src/services/cron/cron.module.ts +++ b/apps/api/src/services/cron/cron.module.ts @@ -3,6 +3,7 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration/conf import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { StatisticsGatheringModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module'; import { TwitterBotModule } from '@ghostfolio/api/services/twitter-bot/twitter-bot.module'; import { Module } from '@nestjs/common'; @@ -15,6 +16,9 @@ import { CronService } from './cron.service'; DataGatheringModule, ExchangeRateDataModule, PropertyModule, + ...(process.env.ENABLE_FEATURE_STATISTICS === 'true' + ? [StatisticsGatheringModule] + : []), TwitterBotModule, UserModule ], diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts index ee91a811e..6756332e2 100644 --- a/apps/api/src/services/cron/cron.service.ts +++ b/apps/api/src/services/cron/cron.service.ts @@ -3,6 +3,7 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; +import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service'; import { TwitterBotService } from '@ghostfolio/api/services/twitter-bot/twitter-bot.service'; import { DATA_GATHERING_QUEUE_PRIORITY_LOW, @@ -12,7 +13,7 @@ import { } from '@ghostfolio/common/config'; import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; -import { Injectable } from '@nestjs/common'; +import { Injectable, Optional } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; @Injectable() @@ -25,10 +26,17 @@ export class CronService { private readonly dataGatheringService: DataGatheringService, private readonly exchangeRateDataService: ExchangeRateDataService, private readonly propertyService: PropertyService, + @Optional() + private readonly statisticsGatheringService: StatisticsGatheringService, private readonly twitterBotService: TwitterBotService, private readonly userService: UserService ) {} + @Cron(CronExpression.EVERY_HOUR) + public async runEveryHour() { + await this.statisticsGatheringService?.addJobToQueue(); + } + @Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE) public async runEveryHourAtRandomMinute() { if (await this.isDataGatheringEnabled()) { diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts new file mode 100644 index 000000000..f23e8c77a --- /dev/null +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts @@ -0,0 +1,36 @@ +import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; +import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; +import { STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config'; + +import { BullAdapter } from '@bull-board/api/bullAdapter'; +import { BullBoardModule } from '@bull-board/nestjs'; +import { BullModule } from '@nestjs/bull'; +import { Module } from '@nestjs/common'; + +import { StatisticsGatheringProcessor } from './statistics-gathering.processor'; +import { StatisticsGatheringService } from './statistics-gathering.service'; + +@Module({ + exports: [BullModule, StatisticsGatheringService], + imports: [ + ...(process.env.ENABLE_FEATURE_BULL_BOARD === 'true' + ? [ + BullBoardModule.forFeature({ + adapter: BullAdapter, + name: STATISTICS_GATHERING_QUEUE, + options: { + displayName: 'Statistics Gathering', + readOnlyMode: process.env.BULL_BOARD_IS_READ_ONLY !== 'false' + } + }) + ] + : []), + BullModule.registerQueue({ + name: STATISTICS_GATHERING_QUEUE + }), + ConfigurationModule, + PropertyModule + ], + providers: [StatisticsGatheringProcessor, StatisticsGatheringService] +}) +export class StatisticsGatheringModule {} diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts new file mode 100644 index 000000000..003aa90d6 --- /dev/null +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts @@ -0,0 +1,167 @@ +import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; +import { PropertyService } from '@ghostfolio/api/services/property/property.service'; +import { + GATHER_STATISTICS_PROCESS_JOB_NAME, + HEADER_KEY_TOKEN, + PROPERTY_BETTER_UPTIME_MONITOR_ID, + PROPERTY_DOCKER_HUB_PULLS, + PROPERTY_GITHUB_CONTRIBUTORS, + PROPERTY_GITHUB_STARGAZERS, + PROPERTY_UPTIME, + STATISTICS_GATHERING_QUEUE +} from '@ghostfolio/common/config'; +import { + DATE_FORMAT, + extractNumberFromString +} from '@ghostfolio/common/helper'; + +import { Process, Processor } from '@nestjs/bull'; +import { Injectable, Logger } from '@nestjs/common'; +import * as cheerio from 'cheerio'; +import { format, subDays } from 'date-fns'; + +@Injectable() +@Processor(STATISTICS_GATHERING_QUEUE) +export class StatisticsGatheringProcessor { + public constructor( + private readonly configurationService: ConfigurationService, + private readonly propertyService: PropertyService + ) {} + + @Process(GATHER_STATISTICS_PROCESS_JOB_NAME) + public async gatherStatistics() { + Logger.log( + 'Statistics gathering has been started', + 'StatisticsGatheringProcessor' + ); + + const [dockerHubPulls, gitHubContributors, gitHubStargazers, uptime] = + await Promise.all([ + this.countDockerHubPulls(), + this.countGitHubContributors(), + this.countGitHubStargazers(), + this.getUptime() + ]); + + await Promise.all([ + dockerHubPulls !== undefined && + this.propertyService.put({ + key: PROPERTY_DOCKER_HUB_PULLS, + value: String(dockerHubPulls) + }), + gitHubContributors !== undefined && + this.propertyService.put({ + key: PROPERTY_GITHUB_CONTRIBUTORS, + value: String(gitHubContributors) + }), + gitHubStargazers !== undefined && + this.propertyService.put({ + key: PROPERTY_GITHUB_STARGAZERS, + value: String(gitHubStargazers) + }), + uptime !== undefined && + this.propertyService.put({ + key: PROPERTY_UPTIME, + value: String(uptime) + }) + ]); + + Logger.log( + 'Statistics gathering has been completed', + 'StatisticsGatheringProcessor' + ); + } + + private async countDockerHubPulls(): Promise { + try { + const { pull_count } = (await fetch( + 'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio', + { + headers: { 'User-Agent': 'request' }, + signal: AbortSignal.timeout( + this.configurationService.get('REQUEST_TIMEOUT') + ) + } + ).then((res) => res.json())) as { pull_count: number }; + + return pull_count; + } catch (error) { + Logger.error(error, 'StatisticsGatheringProcessor - DockerHub'); + + return undefined; + } + } + + private async countGitHubContributors(): Promise { + try { + const body = await fetch('https://github.com/ghostfolio/ghostfolio', { + signal: AbortSignal.timeout( + this.configurationService.get('REQUEST_TIMEOUT') + ) + }).then((res) => res.text()); + + const $ = cheerio.load(body); + + return extractNumberFromString({ + value: $( + 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter' + ).text() + }); + } catch (error) { + Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); + + return undefined; + } + } + + private async countGitHubStargazers(): Promise { + try { + const { stargazers_count } = (await fetch( + 'https://api.github.com/repos/ghostfolio/ghostfolio', + { + headers: { 'User-Agent': 'request' }, + signal: AbortSignal.timeout( + this.configurationService.get('REQUEST_TIMEOUT') + ) + } + ).then((res) => res.json())) as { stargazers_count: number }; + + return stargazers_count; + } catch (error) { + Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); + + return undefined; + } + } + + private async getUptime(): Promise { + try { + const monitorId = await this.propertyService.getByKey( + PROPERTY_BETTER_UPTIME_MONITOR_ID + ); + + const { data } = await fetch( + `https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format( + subDays(new Date(), 90), + DATE_FORMAT + )}&to${format(new Date(), DATE_FORMAT)}`, + { + headers: { + [HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get( + 'API_KEY_BETTER_UPTIME' + )}` + }, + signal: AbortSignal.timeout( + this.configurationService.get('REQUEST_TIMEOUT') + ) + } + ).then((res) => res.json()); + + return data.attributes.availability / 100; + } catch (error) { + Logger.error(error, 'StatisticsGatheringProcessor - Better Stack'); + + return undefined; + } + } +} diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts new file mode 100644 index 000000000..7fd406082 --- /dev/null +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts @@ -0,0 +1,25 @@ +import { + GATHER_STATISTICS_PROCESS_JOB_NAME, + GATHER_STATISTICS_PROCESS_JOB_OPTIONS, + STATISTICS_GATHERING_QUEUE +} from '@ghostfolio/common/config'; + +import { InjectQueue } from '@nestjs/bull'; +import { Injectable } from '@nestjs/common'; +import { Queue } from 'bull'; + +@Injectable() +export class StatisticsGatheringService { + public constructor( + @InjectQueue(STATISTICS_GATHERING_QUEUE) + private readonly statisticsGatheringQueue: Queue + ) {} + + public async addJobToQueue() { + return this.statisticsGatheringQueue.add( + GATHER_STATISTICS_PROCESS_JOB_NAME, + {}, + GATHER_STATISTICS_PROCESS_JOB_OPTIONS + ); + } +} diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 08fa2f030..84e8bdda7 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -75,6 +75,8 @@ export const PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE_PRIORITY_HIGH = 1; export const PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER; +export const STATISTICS_GATHERING_QUEUE = 'STATISTICS_GATHERING_QUEUE'; + export const DEFAULT_CURRENCY = 'USD'; export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy'; export const DEFAULT_HOST = '0.0.0.0'; @@ -183,6 +185,16 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = { removeOnComplete: true }; +export const GATHER_STATISTICS_PROCESS_JOB_NAME = 'GATHER_STATISTICS'; +export const GATHER_STATISTICS_PROCESS_JOB_OPTIONS: JobOptions = { + attempts: 5, + backoff: { + delay: ms('1 minute'), + type: 'exponential' + }, + removeOnComplete: true +}; + export const INVESTMENT_ACTIVITY_TYPES = [ Type.BUY, Type.DIVIDEND, @@ -209,6 +221,9 @@ export const PROPERTY_API_KEY_GHOSTFOLIO = 'API_KEY_GHOSTFOLIO'; export const PROPERTY_API_KEY_OPENROUTER = 'API_KEY_OPENROUTER'; export const PROPERTY_BENCHMARKS = 'BENCHMARKS'; export const PROPERTY_BETTER_UPTIME_MONITOR_ID = 'BETTER_UPTIME_MONITOR_ID'; +export const PROPERTY_DOCKER_HUB_PULLS = 'DOCKER_HUB_PULLS'; +export const PROPERTY_GITHUB_CONTRIBUTORS = 'GITHUB_CONTRIBUTORS'; +export const PROPERTY_GITHUB_STARGAZERS = 'GITHUB_STARGAZERS'; export const PROPERTY_COUNTRIES_OF_SUBSCRIBERS = 'COUNTRIES_OF_SUBSCRIBERS'; export const PROPERTY_COUPONS = 'COUPONS'; export const PROPERTY_CURRENCIES = 'CURRENCIES'; @@ -225,6 +240,7 @@ export const PROPERTY_OPENROUTER_MODEL = 'OPENROUTER_MODEL'; export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS'; export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG'; export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE'; +export const PROPERTY_UPTIME = 'UPTIME'; export const QUEUE_JOB_STATUS_LIST = [ 'active', diff --git a/libs/common/src/lib/interfaces/statistics.interface.ts b/libs/common/src/lib/interfaces/statistics.interface.ts index 2852d34ab..670cdaf4c 100644 --- a/libs/common/src/lib/interfaces/statistics.interface.ts +++ b/libs/common/src/lib/interfaces/statistics.interface.ts @@ -5,6 +5,6 @@ export interface Statistics { gitHubContributors: number; gitHubStargazers: number; newUsers30d: number; - slackCommunityUsers: string; + slackCommunityUsers: number; uptime: number; }