From ae3289fa1f220a887f04330b1631c315009c0364 Mon Sep 17 00:00:00 2001 From: Thomas <4159106+dtslvr@users.noreply.github.com> Date: Tue, 7 Jun 2022 20:11:08 +0200 Subject: [PATCH] Clean up --- apps/api/src/app/admin/admin.controller.ts | 18 ++ apps/api/src/app/admin/admin.service.ts | 22 -- apps/api/src/app/app.controller.ts | 17 +- apps/api/src/app/cache/cache.controller.ts | 4 +- apps/api/src/app/cache/cache.service.ts | 11 +- apps/api/src/app/info/info.service.ts | 8 - apps/api/src/services/cron.service.ts | 4 +- .../src/services/data-gathering.service.ts | 211 +----------------- .../admin-overview.component.ts | 91 +++----- .../admin-overview/admin-overview.html | 38 ++-- apps/client/src/app/services/admin.service.ts | 4 + libs/common/src/lib/config.ts | 2 - .../lib/interfaces/admin-data.interface.ts | 2 - 13 files changed, 93 insertions(+), 339 deletions(-) diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 384da8a47..3b10bb0f0 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -66,6 +66,24 @@ export class AdminController { return this.adminService.get(); } + @Post('gather') + @UseGuards(AuthGuard('jwt')) + public async gather7Days(): Promise { + if ( + !hasPermission( + this.request.user.permissions, + permissions.accessAdminControl + ) + ) { + throw new HttpException( + getReasonPhrase(StatusCodes.FORBIDDEN), + StatusCodes.FORBIDDEN + ); + } + + this.dataGatheringService.gather7Days(); + } + @Post('gather/max') @UseGuards(AuthGuard('jwt')) public async gatherMax(): Promise { diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts index 3b2392bfc..5e240a473 100644 --- a/apps/api/src/app/admin/admin.service.ts +++ b/apps/api/src/app/admin/admin.service.ts @@ -42,8 +42,6 @@ export class AdminService { public async get(): Promise { return { - dataGatheringProgress: - await this.dataGatheringService.getDataGatheringProgress(), exchangeRates: this.exchangeRateDataService .getCurrencies() .filter((currency) => { @@ -60,7 +58,6 @@ export class AdminService { ) }; }), - lastDataGathering: await this.getLastDataGathering(), settings: await this.propertyService.get(), transactionCount: await this.prismaService.order.count(), userCount: await this.prismaService.user.count(), @@ -161,30 +158,11 @@ export class AdminService { if (key === PROPERTY_CURRENCIES) { await this.exchangeRateDataService.initialize(); - await this.dataGatheringService.reset(); } return response; } - private async getLastDataGathering() { - const lastDataGathering = - await this.dataGatheringService.getLastDataGathering(); - - if (lastDataGathering) { - return lastDataGathering; - } - - const dataGatheringInProgress = - await this.dataGatheringService.getIsInProgress(); - - if (dataGatheringInProgress) { - return 'IN_PROGRESS'; - } - - return undefined; - } - private async getUsersWithAnalytics(): Promise { const usersWithAnalytics = await this.prismaService.user.findMany({ orderBy: { diff --git a/apps/api/src/app/app.controller.ts b/apps/api/src/app/app.controller.ts index 4d024f117..f7704f9e2 100644 --- a/apps/api/src/app/app.controller.ts +++ b/apps/api/src/app/app.controller.ts @@ -1,21 +1,6 @@ -import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; import { Controller } from '@nestjs/common'; @Controller() export class AppController { - public constructor( - private readonly dataGatheringService: DataGatheringService - ) { - this.initialize(); - } - - private async initialize() { - const isDataGatheringInProgress = - await this.dataGatheringService.getIsInProgress(); - - if (isDataGatheringInProgress) { - // Prepare for automatical data gathering, if hung up in progress state - await this.dataGatheringService.reset(); - } - } + public constructor() {} } diff --git a/apps/api/src/app/cache/cache.controller.ts b/apps/api/src/app/cache/cache.controller.ts index 4783266b4..cd8a1e62b 100644 --- a/apps/api/src/app/cache/cache.controller.ts +++ b/apps/api/src/app/cache/cache.controller.ts @@ -36,8 +36,6 @@ export class CacheController { ); } - this.redisCacheService.reset(); - - return this.cacheService.flush(); + return this.redisCacheService.reset(); } } diff --git a/apps/api/src/app/cache/cache.service.ts b/apps/api/src/app/cache/cache.service.ts index 69c5fe66a..b907e14c3 100644 --- a/apps/api/src/app/cache/cache.service.ts +++ b/apps/api/src/app/cache/cache.service.ts @@ -1,15 +1,6 @@ -import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; import { Injectable } from '@nestjs/common'; @Injectable() export class CacheService { - public constructor( - private readonly dataGaterhingService: DataGatheringService - ) {} - - public async flush(): Promise { - await this.dataGaterhingService.reset(); - - return; - } + public constructor() {} } diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.ts index 440f90aa1..04651dd86 100644 --- a/apps/api/src/app/info/info.service.ts +++ b/apps/api/src/app/info/info.service.ts @@ -106,7 +106,6 @@ export class InfoService { baseCurrency: this.configurationService.get('BASE_CURRENCY'), currencies: this.exchangeRateDataService.getCurrencies(), demoAuthToken: this.getDemoAuthToken(), - lastDataGathering: await this.getLastDataGathering(), statistics: await this.getStatistics(), subscriptions: await this.getSubscriptions(), tags: await this.tagService.get() @@ -215,13 +214,6 @@ export class InfoService { }); } - private async getLastDataGathering() { - const lastDataGathering = - await this.dataGatheringService.getLastDataGathering(); - - return lastDataGathering ?? null; - } - private async getStatistics() { if (!this.configurationService.get('ENABLE_FEATURE_STATISTICS')) { return undefined; diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index b19d676dc..4aaef3dbe 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -23,8 +23,8 @@ export class CronService { private readonly twitterBotService: TwitterBotService ) {} - @Cron(CronExpression.EVERY_MINUTE) - public async runEveryMinute() { + @Cron(CronExpression.EVERY_HOUR) + public async runEveryHour() { await this.dataGatheringService.gather7Days(); } diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 020013c85..50ce2e892 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -2,9 +2,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.se import { DATA_GATHERING_QUEUE, DATA_GATHERING_QUEUE_PRIORITY_LOW, - GATHER_HISTORICAL_MARKET_DATA_PROCESS, - PROPERTY_LAST_DATA_GATHERING, - PROPERTY_LOCKED_DATA_GATHERING + GATHER_HISTORICAL_MARKET_DATA_PROCESS } from '@ghostfolio/common/config'; import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; @@ -12,7 +10,7 @@ import { InjectQueue } from '@nestjs/bull'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { DataSource } from '@prisma/client'; import { Queue } from 'bull'; -import { differenceInHours, format, subDays } from 'date-fns'; +import { format, subDays } from 'date-fns'; import ms from 'ms'; import { DataProviderService } from './data-provider/data-provider.service'; @@ -37,155 +35,23 @@ export class DataGatheringService { ) {} public async gather7Days() { - const isDataGatheringNeeded = await this.isDataGatheringNeeded(); - - if (isDataGatheringNeeded) { - Logger.log('7d data gathering has been started.', 'DataGatheringService'); - console.time('data-gathering-7d'); - - await this.prismaService.property.create({ - data: { - key: PROPERTY_LOCKED_DATA_GATHERING, - value: new Date().toISOString() - } - }); - - const dataGatheringItems = await this.getSymbols7D(); - - try { - await this.gatherSymbols(dataGatheringItems); - - await this.prismaService.property.upsert({ - create: { - key: PROPERTY_LAST_DATA_GATHERING, - value: new Date().toISOString() - }, - update: { value: new Date().toISOString() }, - where: { key: PROPERTY_LAST_DATA_GATHERING } - }); - } catch (error) { - Logger.error(error, 'DataGatheringService'); - } - - await this.prismaService.property.delete({ - where: { - key: PROPERTY_LOCKED_DATA_GATHERING - } - }); - - Logger.log( - '7d data gathering has been completed.', - 'DataGatheringService' - ); - console.timeEnd('data-gathering-7d'); - } + const dataGatheringItems = await this.getSymbols7D(); + await this.gatherSymbols(dataGatheringItems); } public async gatherMax() { - const isDataGatheringLocked = await this.prismaService.property.findUnique({ - where: { key: PROPERTY_LOCKED_DATA_GATHERING } - }); - - if (!isDataGatheringLocked) { - Logger.log( - 'Max data gathering has been started.', - 'DataGatheringService' - ); - console.time('data-gathering-max'); - - await this.prismaService.property.create({ - data: { - key: PROPERTY_LOCKED_DATA_GATHERING, - value: new Date().toISOString() - } - }); - - const symbols = await this.getSymbolsMax(); - - try { - await this.gatherSymbols(symbols); - - await this.prismaService.property.upsert({ - create: { - key: PROPERTY_LAST_DATA_GATHERING, - value: new Date().toISOString() - }, - update: { value: new Date().toISOString() }, - where: { key: PROPERTY_LAST_DATA_GATHERING } - }); - } catch (error) { - Logger.error(error, 'DataGatheringService'); - } - - await this.prismaService.property.delete({ - where: { - key: PROPERTY_LOCKED_DATA_GATHERING - } - }); - - Logger.log( - 'Max data gathering has been completed.', - 'DataGatheringService' - ); - console.timeEnd('data-gathering-max'); - } + const dataGatheringItems = await this.getSymbolsMax(); + await this.gatherSymbols(dataGatheringItems); } public async gatherSymbol({ dataSource, symbol }: UniqueAsset) { - const isDataGatheringLocked = await this.prismaService.property.findUnique({ - where: { key: PROPERTY_LOCKED_DATA_GATHERING } - }); - - if (!isDataGatheringLocked) { - Logger.log( - `Symbol data gathering for ${symbol} has been started.`, - 'DataGatheringService' + const symbols = (await this.getSymbolsMax()).filter((dataGatheringItem) => { + return ( + dataGatheringItem.dataSource === dataSource && + dataGatheringItem.symbol === symbol ); - console.time('data-gathering-symbol'); - - await this.prismaService.property.create({ - data: { - key: PROPERTY_LOCKED_DATA_GATHERING, - value: new Date().toISOString() - } - }); - - const symbols = (await this.getSymbolsMax()).filter( - (dataGatheringItem) => { - return ( - dataGatheringItem.dataSource === dataSource && - dataGatheringItem.symbol === symbol - ); - } - ); - - try { - await this.gatherSymbols(symbols); - - await this.prismaService.property.upsert({ - create: { - key: PROPERTY_LAST_DATA_GATHERING, - value: new Date().toISOString() - }, - update: { value: new Date().toISOString() }, - where: { key: PROPERTY_LAST_DATA_GATHERING } - }); - } catch (error) { - Logger.error(error, 'DataGatheringService'); - } - - await this.prismaService.property.delete({ - where: { - key: PROPERTY_LOCKED_DATA_GATHERING - } - }); - - Logger.log( - `Symbol data gathering for ${symbol} has been completed.`, - 'DataGatheringService' - ); - console.timeEnd('data-gathering-symbol'); - } + }); + await this.gatherSymbols(symbols); } public async gatherSymbolForDate({ @@ -358,34 +224,6 @@ export class DataGatheringService { } } - public async getDataGatheringProgress() { - const isInProgress = await this.getIsInProgress(); - - if (isInProgress) { - return this.dataGatheringProgress; - } - - return undefined; - } - - public async getIsInProgress() { - return await this.prismaService.property.findUnique({ - where: { key: PROPERTY_LOCKED_DATA_GATHERING } - }); - } - - public async getLastDataGathering() { - const lastDataGathering = await this.prismaService.property.findUnique({ - where: { key: PROPERTY_LAST_DATA_GATHERING } - }); - - if (lastDataGathering?.value) { - return new Date(lastDataGathering.value); - } - - return undefined; - } - public async getSymbolsMax(): Promise { const startDate = ( @@ -454,19 +292,6 @@ export class DataGatheringService { }); } - public async reset() { - Logger.log('Data gathering has been reset.', 'DataGatheringService'); - - await this.prismaService.property.deleteMany({ - where: { - OR: [ - { key: PROPERTY_LAST_DATA_GATHERING }, - { key: PROPERTY_LOCKED_DATA_GATHERING } - ] - } - }); - } - private async getSymbols7D(): Promise { const startDate = subDays(resetHours(new Date()), 7); @@ -529,16 +354,4 @@ export class DataGatheringService { return [...currencyPairsToGather, ...symbolProfilesToGather]; } - - private async isDataGatheringNeeded() { - const lastDataGathering = await this.getLastDataGathering(); - - const isDataGatheringLocked = await this.prismaService.property.findUnique({ - where: { key: PROPERTY_LOCKED_DATA_GATHERING } - }); - - const diffInHours = differenceInHours(new Date(), lastDataGathering); - - return (diffInHours >= 1 || !lastDataGathering) && !isDataGatheringLocked; - } } diff --git a/apps/client/src/app/components/admin-overview/admin-overview.component.ts b/apps/client/src/app/components/admin-overview/admin-overview.component.ts index f115c4ca9..f5ac65178 100644 --- a/apps/client/src/app/components/admin-overview/admin-overview.component.ts +++ b/apps/client/src/app/components/admin-overview/admin-overview.component.ts @@ -15,7 +15,6 @@ import { hasPermission, permissions } from '@ghostfolio/common/permissions'; import { differenceInSeconds, formatDistanceToNowStrict, - isValid, parseISO } from 'date-fns'; import { uniq } from 'lodash'; @@ -32,14 +31,11 @@ export class AdminOverviewComponent implements OnDestroy, OnInit { public couponDuration: StringValue = '30 days'; public coupons: Coupon[]; public customCurrencies: string[]; - public dataGatheringInProgress: boolean; - public dataGatheringProgress: number; public exchangeRates: { label1: string; label2: string; value: number }[]; public hasPermissionForSubscription: boolean; public hasPermissionForSystemMessage: boolean; public hasPermissionToToggleReadOnlyMode: boolean; public info: InfoItem; - public lastDataGathering: string; public transactionCount: number; public userCount: number; public user: User; @@ -128,7 +124,7 @@ export class AdminOverviewComponent implements OnDestroy, OnInit { public onDeleteCoupon(aCouponCode: string) { const confirmation = confirm('Do you really want to delete this coupon?'); - if (confirmation) { + if (confirmation === true) { const coupons = this.coupons.filter((coupon) => { return coupon.code !== aCouponCode; }); @@ -139,7 +135,7 @@ export class AdminOverviewComponent implements OnDestroy, OnInit { public onDeleteCurrency(aCurrency: string) { const confirmation = confirm('Do you really want to delete this currency?'); - if (confirmation) { + if (confirmation === true) { const currencies = this.customCurrencies.filter((currency) => { return currency !== aCurrency; }); @@ -152,24 +148,11 @@ export class AdminOverviewComponent implements OnDestroy, OnInit { } public onFlushCache() { - this.cacheService - .flush() - .pipe(takeUntil(this.unsubscribeSubject)) - .subscribe(() => { - setTimeout(() => { - window.location.reload(); - }, 300); - }); - } - - public onGatherMax() { - const confirmation = confirm( - 'This action may take some time. Do you want to proceed?' - ); + const confirmation = confirm('Do you really want to flush the cache?'); if (confirmation === true) { - this.adminService - .gatherMax() + this.cacheService + .flush() .pipe(takeUntil(this.unsubscribeSubject)) .subscribe(() => { setTimeout(() => { @@ -179,6 +162,28 @@ export class AdminOverviewComponent implements OnDestroy, OnInit { } } + public onGather7Days() { + this.adminService + .gather7Days() + .pipe(takeUntil(this.unsubscribeSubject)) + .subscribe(() => { + setTimeout(() => { + window.location.reload(); + }, 300); + }); + } + + public onGatherMax() { + this.adminService + .gatherMax() + .pipe(takeUntil(this.unsubscribeSubject)) + .subscribe(() => { + setTimeout(() => { + window.location.reload(); + }, 300); + }); + } + public onGatherProfileData() { this.adminService .gatherProfileData() @@ -207,39 +212,15 @@ export class AdminOverviewComponent implements OnDestroy, OnInit { this.dataService .fetchAdminData() .pipe(takeUntil(this.unsubscribeSubject)) - .subscribe( - ({ - dataGatheringProgress, - exchangeRates, - lastDataGathering, - settings, - transactionCount, - userCount - }) => { - this.coupons = (settings[PROPERTY_COUPONS] as Coupon[]) ?? []; - this.customCurrencies = settings[PROPERTY_CURRENCIES] as string[]; - this.dataGatheringProgress = dataGatheringProgress; - this.exchangeRates = exchangeRates; - - if (isValid(parseISO(lastDataGathering?.toString()))) { - this.lastDataGathering = formatDistanceToNowStrict( - new Date(lastDataGathering), - { - addSuffix: true - } - ); - } else if (lastDataGathering === 'IN_PROGRESS') { - this.dataGatheringInProgress = true; - } else { - this.lastDataGathering = 'Starting soon...'; - } - - this.transactionCount = transactionCount; - this.userCount = userCount; - - this.changeDetectorRef.markForCheck(); - } - ); + .subscribe(({ exchangeRates, settings, transactionCount, userCount }) => { + this.coupons = (settings[PROPERTY_COUPONS] as Coupon[]) ?? []; + this.customCurrencies = settings[PROPERTY_CURRENCIES] as string[]; + this.exchangeRates = exchangeRates; + this.transactionCount = transactionCount; + this.userCount = userCount; + + this.changeDetectorRef.markForCheck(); + }); } private generateCouponCode(aLength: number) { diff --git a/apps/client/src/app/components/admin-overview/admin-overview.html b/apps/client/src/app/components/admin-overview/admin-overview.html index 6118fccaa..76cab6c86 100644 --- a/apps/client/src/app/components/admin-overview/admin-overview.html +++ b/apps/client/src/app/components/admin-overview/admin-overview.html @@ -19,37 +19,30 @@
Data Gathering
-
- {{ lastDataGathering }} - In Progress ({{ dataGatheringProgress | percent : '1.2-2' - }}) -
-
+
@@ -58,7 +51,6 @@ class="mb-2 mr-2" color="accent" mat-flat-button - [disabled]="dataGatheringInProgress" (click)="onGatherProfileData()" > @@ -109,7 +100,6 @@
+
+
Housekeeping
+
+ +
+
diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.ts index 7b99acc82..d97939b93 100644 --- a/apps/client/src/app/services/admin.service.ts +++ b/apps/client/src/app/services/admin.service.ts @@ -51,6 +51,10 @@ export class AdminService { return this.http.get(`/api/v1/admin/queue/job`); } + public gather7Days() { + return this.http.post(`/api/v1/admin/gather`, {}); + } + public gatherMax() { return this.http.post(`/api/v1/admin/gather/max`, {}); } diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index b0207f309..47e5d375f 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -56,8 +56,6 @@ export const PROPERTY_BENCHMARKS = 'BENCHMARKS'; export const PROPERTY_COUPONS = 'COUPONS'; export const PROPERTY_CURRENCIES = 'CURRENCIES'; export const PROPERTY_IS_READ_ONLY_MODE = 'IS_READ_ONLY_MODE'; -export const PROPERTY_LAST_DATA_GATHERING = 'LAST_DATA_GATHERING'; -export const PROPERTY_LOCKED_DATA_GATHERING = 'LOCKED_DATA_GATHERING'; export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS'; export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG'; export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE'; diff --git a/libs/common/src/lib/interfaces/admin-data.interface.ts b/libs/common/src/lib/interfaces/admin-data.interface.ts index ce90dccc5..1e0c6e532 100644 --- a/libs/common/src/lib/interfaces/admin-data.interface.ts +++ b/libs/common/src/lib/interfaces/admin-data.interface.ts @@ -1,7 +1,5 @@ export interface AdminData { - dataGatheringProgress?: number; exchangeRates: { label1: string; label2: string; value: number }[]; - lastDataGathering?: Date | 'IN_PROGRESS'; settings: { [key: string]: boolean | object | string | string[] }; transactionCount: number; userCount: number;