mirror of https://github.com/ghostfolio/ghostfolio
Thomas Kaul
3 days ago
committed by
GitHub
GitHub
20 changed files with 419 additions and 156 deletions
@ -0,0 +1,36 @@ |
|||||
|
import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; |
||||
|
import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; |
||||
|
import { STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config'; |
||||
|
|
||||
|
import { BullAdapter } from '@bull-board/api/bullAdapter'; |
||||
|
import { BullBoardModule } from '@bull-board/nestjs'; |
||||
|
import { BullModule } from '@nestjs/bull'; |
||||
|
import { Module } from '@nestjs/common'; |
||||
|
|
||||
|
import { StatisticsGatheringProcessor } from './statistics-gathering.processor'; |
||||
|
import { StatisticsGatheringService } from './statistics-gathering.service'; |
||||
|
|
||||
|
@Module({ |
||||
|
exports: [BullModule, StatisticsGatheringService], |
||||
|
imports: [ |
||||
|
...(process.env.ENABLE_FEATURE_BULL_BOARD === 'true' |
||||
|
? [ |
||||
|
BullBoardModule.forFeature({ |
||||
|
adapter: BullAdapter, |
||||
|
name: STATISTICS_GATHERING_QUEUE, |
||||
|
options: { |
||||
|
displayName: 'Statistics Gathering', |
||||
|
readOnlyMode: process.env.BULL_BOARD_IS_READ_ONLY !== 'false' |
||||
|
} |
||||
|
}) |
||||
|
] |
||||
|
: []), |
||||
|
BullModule.registerQueue({ |
||||
|
name: STATISTICS_GATHERING_QUEUE |
||||
|
}), |
||||
|
ConfigurationModule, |
||||
|
PropertyModule |
||||
|
], |
||||
|
providers: [StatisticsGatheringProcessor, StatisticsGatheringService] |
||||
|
}) |
||||
|
export class StatisticsGatheringQueueModule {} |
||||
@ -0,0 +1,212 @@ |
|||||
|
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; |
||||
|
import { PropertyService } from '@ghostfolio/api/services/property/property.service'; |
||||
|
import { |
||||
|
GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, |
||||
|
GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, |
||||
|
GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, |
||||
|
GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME, |
||||
|
HEADER_KEY_TOKEN, |
||||
|
PROPERTY_BETTER_UPTIME_MONITOR_ID, |
||||
|
PROPERTY_DOCKER_HUB_PULLS, |
||||
|
PROPERTY_GITHUB_CONTRIBUTORS, |
||||
|
PROPERTY_GITHUB_STARGAZERS, |
||||
|
PROPERTY_UPTIME, |
||||
|
STATISTICS_GATHERING_QUEUE |
||||
|
} from '@ghostfolio/common/config'; |
||||
|
import { |
||||
|
DATE_FORMAT, |
||||
|
extractNumberFromString |
||||
|
} from '@ghostfolio/common/helper'; |
||||
|
|
||||
|
import { Process, Processor } from '@nestjs/bull'; |
||||
|
import { Injectable, Logger } from '@nestjs/common'; |
||||
|
import * as cheerio from 'cheerio'; |
||||
|
import { format, subDays } from 'date-fns'; |
||||
|
|
||||
|
@Injectable() |
||||
|
@Processor(STATISTICS_GATHERING_QUEUE) |
||||
|
export class StatisticsGatheringProcessor { |
||||
|
public constructor( |
||||
|
private readonly configurationService: ConfigurationService, |
||||
|
private readonly propertyService: PropertyService |
||||
|
) {} |
||||
|
|
||||
|
@Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME) |
||||
|
public async gatherDockerHubPullsStatistics() { |
||||
|
Logger.log( |
||||
|
'Docker Hub pulls statistics gathering has been started', |
||||
|
'StatisticsGatheringProcessor' |
||||
|
); |
||||
|
|
||||
|
const dockerHubPulls = await this.countDockerHubPulls(); |
||||
|
|
||||
|
await this.propertyService.put({ |
||||
|
key: PROPERTY_DOCKER_HUB_PULLS, |
||||
|
value: String(dockerHubPulls) |
||||
|
}); |
||||
|
|
||||
|
Logger.log( |
||||
|
'Docker Hub pulls statistics gathering has been completed', |
||||
|
'StatisticsGatheringProcessor' |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
@Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME) |
||||
|
public async gatherGitHubContributorsStatistics() { |
||||
|
Logger.log( |
||||
|
'GitHub contributors statistics gathering has been started', |
||||
|
'StatisticsGatheringProcessor' |
||||
|
); |
||||
|
|
||||
|
const gitHubContributors = await this.countGitHubContributors(); |
||||
|
|
||||
|
await this.propertyService.put({ |
||||
|
key: PROPERTY_GITHUB_CONTRIBUTORS, |
||||
|
value: String(gitHubContributors) |
||||
|
}); |
||||
|
|
||||
|
Logger.log( |
||||
|
'GitHub contributors statistics gathering has been completed', |
||||
|
'StatisticsGatheringProcessor' |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
@Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME) |
||||
|
public async gatherGitHubStargazersStatistics() { |
||||
|
Logger.log( |
||||
|
'GitHub stargazers statistics gathering has been started', |
||||
|
'StatisticsGatheringProcessor' |
||||
|
); |
||||
|
|
||||
|
const gitHubStargazers = await this.countGitHubStargazers(); |
||||
|
|
||||
|
await this.propertyService.put({ |
||||
|
key: PROPERTY_GITHUB_STARGAZERS, |
||||
|
value: String(gitHubStargazers) |
||||
|
}); |
||||
|
|
||||
|
Logger.log( |
||||
|
'GitHub stargazers statistics gathering has been completed', |
||||
|
'StatisticsGatheringProcessor' |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
@Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME) |
||||
|
public async gatherUptimeStatistics() { |
||||
|
Logger.log( |
||||
|
'Uptime statistics gathering has been started', |
||||
|
'StatisticsGatheringProcessor' |
||||
|
); |
||||
|
|
||||
|
const uptime = await this.getUptime(); |
||||
|
|
||||
|
await this.propertyService.put({ |
||||
|
key: PROPERTY_UPTIME, |
||||
|
value: String(uptime) |
||||
|
}); |
||||
|
|
||||
|
Logger.log( |
||||
|
'Uptime statistics gathering has been completed', |
||||
|
'StatisticsGatheringProcessor' |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
private async countDockerHubPulls(): Promise<number> { |
||||
|
try { |
||||
|
const { pull_count } = (await fetch( |
||||
|
'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio', |
||||
|
{ |
||||
|
headers: { 'User-Agent': 'request' }, |
||||
|
signal: AbortSignal.timeout( |
||||
|
this.configurationService.get('REQUEST_TIMEOUT') |
||||
|
) |
||||
|
} |
||||
|
).then((res) => res.json())) as { pull_count: number }; |
||||
|
|
||||
|
return pull_count; |
||||
|
} catch (error) { |
||||
|
Logger.error(error, 'StatisticsGatheringProcessor - DockerHub'); |
||||
|
|
||||
|
throw error; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private async countGitHubContributors(): Promise<number> { |
||||
|
try { |
||||
|
const body = await fetch('https://github.com/ghostfolio/ghostfolio', { |
||||
|
signal: AbortSignal.timeout( |
||||
|
this.configurationService.get('REQUEST_TIMEOUT') |
||||
|
) |
||||
|
}).then((res) => res.text()); |
||||
|
|
||||
|
const $ = cheerio.load(body); |
||||
|
|
||||
|
const value = $( |
||||
|
'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter' |
||||
|
).text(); |
||||
|
|
||||
|
if (!value) { |
||||
|
throw new Error('Could not find the contributors count in the page'); |
||||
|
} |
||||
|
|
||||
|
return extractNumberFromString({ |
||||
|
value |
||||
|
}); |
||||
|
} catch (error) { |
||||
|
Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); |
||||
|
|
||||
|
throw error; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private async countGitHubStargazers(): Promise<number> { |
||||
|
try { |
||||
|
const { stargazers_count } = (await fetch( |
||||
|
'https://api.github.com/repos/ghostfolio/ghostfolio', |
||||
|
{ |
||||
|
headers: { 'User-Agent': 'request' }, |
||||
|
signal: AbortSignal.timeout( |
||||
|
this.configurationService.get('REQUEST_TIMEOUT') |
||||
|
) |
||||
|
} |
||||
|
).then((res) => res.json())) as { stargazers_count: number }; |
||||
|
|
||||
|
return stargazers_count; |
||||
|
} catch (error) { |
||||
|
Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); |
||||
|
|
||||
|
throw error; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private async getUptime(): Promise<number> { |
||||
|
try { |
||||
|
const monitorId = await this.propertyService.getByKey<string>( |
||||
|
PROPERTY_BETTER_UPTIME_MONITOR_ID |
||||
|
); |
||||
|
|
||||
|
const { data } = await fetch( |
||||
|
`https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format( |
||||
|
subDays(new Date(), 90), |
||||
|
DATE_FORMAT |
||||
|
)}&to${format(new Date(), DATE_FORMAT)}`,
|
||||
|
{ |
||||
|
headers: { |
||||
|
[HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get( |
||||
|
'API_KEY_BETTER_UPTIME' |
||||
|
)}` |
||||
|
}, |
||||
|
signal: AbortSignal.timeout( |
||||
|
this.configurationService.get('REQUEST_TIMEOUT') |
||||
|
) |
||||
|
} |
||||
|
).then((res) => res.json()); |
||||
|
|
||||
|
return data.attributes.availability / 100; |
||||
|
} catch (error) { |
||||
|
Logger.error(error, 'StatisticsGatheringProcessor - Better Stack'); |
||||
|
|
||||
|
throw error; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,40 @@ |
|||||
|
import { |
||||
|
GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, |
||||
|
GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, |
||||
|
GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, |
||||
|
GATHER_STATISTICS_PROCESS_JOB_OPTIONS, |
||||
|
GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME, |
||||
|
STATISTICS_GATHERING_QUEUE |
||||
|
} from '@ghostfolio/common/config'; |
||||
|
|
||||
|
import { InjectQueue } from '@nestjs/bull'; |
||||
|
import { Injectable } from '@nestjs/common'; |
||||
|
import { Queue } from 'bull'; |
||||
|
|
||||
|
@Injectable() |
||||
|
export class StatisticsGatheringService { |
||||
|
public constructor( |
||||
|
@InjectQueue(STATISTICS_GATHERING_QUEUE) |
||||
|
private readonly statisticsGatheringQueue: Queue |
||||
|
) {} |
||||
|
|
||||
|
public async addJobsToQueue() { |
||||
|
return Promise.all( |
||||
|
[ |
||||
|
GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, |
||||
|
GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, |
||||
|
GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, |
||||
|
GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME |
||||
|
].map((jobName) => { |
||||
|
return this.statisticsGatheringQueue.add( |
||||
|
jobName, |
||||
|
{}, |
||||
|
{ |
||||
|
...GATHER_STATISTICS_PROCESS_JOB_OPTIONS, |
||||
|
jobId: jobName |
||||
|
} |
||||
|
); |
||||
|
}) |
||||
|
); |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue