mirror of https://github.com/ghostfolio/ghostfolio
Thomas Kaul
3 days ago
committed by
GitHub
GitHub
20 changed files with 419 additions and 156 deletions
@ -0,0 +1,36 @@ |
|||
import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; |
|||
import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; |
|||
import { STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config'; |
|||
|
|||
import { BullAdapter } from '@bull-board/api/bullAdapter'; |
|||
import { BullBoardModule } from '@bull-board/nestjs'; |
|||
import { BullModule } from '@nestjs/bull'; |
|||
import { Module } from '@nestjs/common'; |
|||
|
|||
import { StatisticsGatheringProcessor } from './statistics-gathering.processor'; |
|||
import { StatisticsGatheringService } from './statistics-gathering.service'; |
|||
|
|||
@Module({ |
|||
exports: [BullModule, StatisticsGatheringService], |
|||
imports: [ |
|||
...(process.env.ENABLE_FEATURE_BULL_BOARD === 'true' |
|||
? [ |
|||
BullBoardModule.forFeature({ |
|||
adapter: BullAdapter, |
|||
name: STATISTICS_GATHERING_QUEUE, |
|||
options: { |
|||
displayName: 'Statistics Gathering', |
|||
readOnlyMode: process.env.BULL_BOARD_IS_READ_ONLY !== 'false' |
|||
} |
|||
}) |
|||
] |
|||
: []), |
|||
BullModule.registerQueue({ |
|||
name: STATISTICS_GATHERING_QUEUE |
|||
}), |
|||
ConfigurationModule, |
|||
PropertyModule |
|||
], |
|||
providers: [StatisticsGatheringProcessor, StatisticsGatheringService] |
|||
}) |
|||
export class StatisticsGatheringQueueModule {} |
|||
@ -0,0 +1,212 @@ |
|||
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; |
|||
import { PropertyService } from '@ghostfolio/api/services/property/property.service'; |
|||
import { |
|||
GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, |
|||
GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, |
|||
GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, |
|||
GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME, |
|||
HEADER_KEY_TOKEN, |
|||
PROPERTY_BETTER_UPTIME_MONITOR_ID, |
|||
PROPERTY_DOCKER_HUB_PULLS, |
|||
PROPERTY_GITHUB_CONTRIBUTORS, |
|||
PROPERTY_GITHUB_STARGAZERS, |
|||
PROPERTY_UPTIME, |
|||
STATISTICS_GATHERING_QUEUE |
|||
} from '@ghostfolio/common/config'; |
|||
import { |
|||
DATE_FORMAT, |
|||
extractNumberFromString |
|||
} from '@ghostfolio/common/helper'; |
|||
|
|||
import { Process, Processor } from '@nestjs/bull'; |
|||
import { Injectable, Logger } from '@nestjs/common'; |
|||
import * as cheerio from 'cheerio'; |
|||
import { format, subDays } from 'date-fns'; |
|||
|
|||
@Injectable() |
|||
@Processor(STATISTICS_GATHERING_QUEUE) |
|||
export class StatisticsGatheringProcessor { |
|||
public constructor( |
|||
private readonly configurationService: ConfigurationService, |
|||
private readonly propertyService: PropertyService |
|||
) {} |
|||
|
|||
@Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME) |
|||
public async gatherDockerHubPullsStatistics() { |
|||
Logger.log( |
|||
'Docker Hub pulls statistics gathering has been started', |
|||
'StatisticsGatheringProcessor' |
|||
); |
|||
|
|||
const dockerHubPulls = await this.countDockerHubPulls(); |
|||
|
|||
await this.propertyService.put({ |
|||
key: PROPERTY_DOCKER_HUB_PULLS, |
|||
value: String(dockerHubPulls) |
|||
}); |
|||
|
|||
Logger.log( |
|||
'Docker Hub pulls statistics gathering has been completed', |
|||
'StatisticsGatheringProcessor' |
|||
); |
|||
} |
|||
|
|||
@Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME) |
|||
public async gatherGitHubContributorsStatistics() { |
|||
Logger.log( |
|||
'GitHub contributors statistics gathering has been started', |
|||
'StatisticsGatheringProcessor' |
|||
); |
|||
|
|||
const gitHubContributors = await this.countGitHubContributors(); |
|||
|
|||
await this.propertyService.put({ |
|||
key: PROPERTY_GITHUB_CONTRIBUTORS, |
|||
value: String(gitHubContributors) |
|||
}); |
|||
|
|||
Logger.log( |
|||
'GitHub contributors statistics gathering has been completed', |
|||
'StatisticsGatheringProcessor' |
|||
); |
|||
} |
|||
|
|||
@Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME) |
|||
public async gatherGitHubStargazersStatistics() { |
|||
Logger.log( |
|||
'GitHub stargazers statistics gathering has been started', |
|||
'StatisticsGatheringProcessor' |
|||
); |
|||
|
|||
const gitHubStargazers = await this.countGitHubStargazers(); |
|||
|
|||
await this.propertyService.put({ |
|||
key: PROPERTY_GITHUB_STARGAZERS, |
|||
value: String(gitHubStargazers) |
|||
}); |
|||
|
|||
Logger.log( |
|||
'GitHub stargazers statistics gathering has been completed', |
|||
'StatisticsGatheringProcessor' |
|||
); |
|||
} |
|||
|
|||
@Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME) |
|||
public async gatherUptimeStatistics() { |
|||
Logger.log( |
|||
'Uptime statistics gathering has been started', |
|||
'StatisticsGatheringProcessor' |
|||
); |
|||
|
|||
const uptime = await this.getUptime(); |
|||
|
|||
await this.propertyService.put({ |
|||
key: PROPERTY_UPTIME, |
|||
value: String(uptime) |
|||
}); |
|||
|
|||
Logger.log( |
|||
'Uptime statistics gathering has been completed', |
|||
'StatisticsGatheringProcessor' |
|||
); |
|||
} |
|||
|
|||
private async countDockerHubPulls(): Promise<number> { |
|||
try { |
|||
const { pull_count } = (await fetch( |
|||
'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio', |
|||
{ |
|||
headers: { 'User-Agent': 'request' }, |
|||
signal: AbortSignal.timeout( |
|||
this.configurationService.get('REQUEST_TIMEOUT') |
|||
) |
|||
} |
|||
).then((res) => res.json())) as { pull_count: number }; |
|||
|
|||
return pull_count; |
|||
} catch (error) { |
|||
Logger.error(error, 'StatisticsGatheringProcessor - DockerHub'); |
|||
|
|||
throw error; |
|||
} |
|||
} |
|||
|
|||
private async countGitHubContributors(): Promise<number> { |
|||
try { |
|||
const body = await fetch('https://github.com/ghostfolio/ghostfolio', { |
|||
signal: AbortSignal.timeout( |
|||
this.configurationService.get('REQUEST_TIMEOUT') |
|||
) |
|||
}).then((res) => res.text()); |
|||
|
|||
const $ = cheerio.load(body); |
|||
|
|||
const value = $( |
|||
'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter' |
|||
).text(); |
|||
|
|||
if (!value) { |
|||
throw new Error('Could not find the contributors count in the page'); |
|||
} |
|||
|
|||
return extractNumberFromString({ |
|||
value |
|||
}); |
|||
} catch (error) { |
|||
Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); |
|||
|
|||
throw error; |
|||
} |
|||
} |
|||
|
|||
private async countGitHubStargazers(): Promise<number> { |
|||
try { |
|||
const { stargazers_count } = (await fetch( |
|||
'https://api.github.com/repos/ghostfolio/ghostfolio', |
|||
{ |
|||
headers: { 'User-Agent': 'request' }, |
|||
signal: AbortSignal.timeout( |
|||
this.configurationService.get('REQUEST_TIMEOUT') |
|||
) |
|||
} |
|||
).then((res) => res.json())) as { stargazers_count: number }; |
|||
|
|||
return stargazers_count; |
|||
} catch (error) { |
|||
Logger.error(error, 'StatisticsGatheringProcessor - GitHub'); |
|||
|
|||
throw error; |
|||
} |
|||
} |
|||
|
|||
private async getUptime(): Promise<number> { |
|||
try { |
|||
const monitorId = await this.propertyService.getByKey<string>( |
|||
PROPERTY_BETTER_UPTIME_MONITOR_ID |
|||
); |
|||
|
|||
const { data } = await fetch( |
|||
`https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format( |
|||
subDays(new Date(), 90), |
|||
DATE_FORMAT |
|||
)}&to${format(new Date(), DATE_FORMAT)}`,
|
|||
{ |
|||
headers: { |
|||
[HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get( |
|||
'API_KEY_BETTER_UPTIME' |
|||
)}` |
|||
}, |
|||
signal: AbortSignal.timeout( |
|||
this.configurationService.get('REQUEST_TIMEOUT') |
|||
) |
|||
} |
|||
).then((res) => res.json()); |
|||
|
|||
return data.attributes.availability / 100; |
|||
} catch (error) { |
|||
Logger.error(error, 'StatisticsGatheringProcessor - Better Stack'); |
|||
|
|||
throw error; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
import { |
|||
GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, |
|||
GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, |
|||
GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, |
|||
GATHER_STATISTICS_PROCESS_JOB_OPTIONS, |
|||
GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME, |
|||
STATISTICS_GATHERING_QUEUE |
|||
} from '@ghostfolio/common/config'; |
|||
|
|||
import { InjectQueue } from '@nestjs/bull'; |
|||
import { Injectable } from '@nestjs/common'; |
|||
import { Queue } from 'bull'; |
|||
|
|||
@Injectable() |
|||
export class StatisticsGatheringService { |
|||
public constructor( |
|||
@InjectQueue(STATISTICS_GATHERING_QUEUE) |
|||
private readonly statisticsGatheringQueue: Queue |
|||
) {} |
|||
|
|||
public async addJobsToQueue() { |
|||
return Promise.all( |
|||
[ |
|||
GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, |
|||
GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, |
|||
GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, |
|||
GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME |
|||
].map((jobName) => { |
|||
return this.statisticsGatheringQueue.add( |
|||
jobName, |
|||
{}, |
|||
{ |
|||
...GATHER_STATISTICS_PROCESS_JOB_OPTIONS, |
|||
jobId: jobName |
|||
} |
|||
); |
|||
}) |
|||
); |
|||
} |
|||
} |
|||
Loading…
Reference in new issue