Clean up

apps/api/src/app/admin/admin.controller.ts

@@ -66,6 +66,24 @@ export class AdminController {
     return this.adminService.get();
   }
 
+  @Post('gather')
+  @UseGuards(AuthGuard('jwt'))
+  public async gather7Days(): Promise<void> {
+    if (
+      !hasPermission(
+        this.request.user.permissions,
+        permissions.accessAdminControl
+      )
+    ) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+
+    this.dataGatheringService.gather7Days();
+  }
+
   @Post('gather/max')
   @UseGuards(AuthGuard('jwt'))
   public async gatherMax(): Promise<void> {

apps/api/src/app/admin/admin.service.ts

@@ -42,8 +42,6 @@ export class AdminService {
 
   public async get(): Promise<AdminData> {
     return {
-      dataGatheringProgress:
-        await this.dataGatheringService.getDataGatheringProgress(),
       exchangeRates: this.exchangeRateDataService
         .getCurrencies()
         .filter((currency) => {
@@ -60,7 +58,6 @@ export class AdminService {
             )
           };
         }),
-      lastDataGathering: await this.getLastDataGathering(),
       settings: await this.propertyService.get(),
       transactionCount: await this.prismaService.order.count(),
       userCount: await this.prismaService.user.count(),
@@ -161,30 +158,11 @@ export class AdminService {
 
     if (key === PROPERTY_CURRENCIES) {
       await this.exchangeRateDataService.initialize();
-      await this.dataGatheringService.reset();
     }
 
     return response;
   }
 
-  private async getLastDataGathering() {
-    const lastDataGathering =
-      await this.dataGatheringService.getLastDataGathering();
-
-    if (lastDataGathering) {
-      return lastDataGathering;
-    }
-
-    const dataGatheringInProgress =
-      await this.dataGatheringService.getIsInProgress();
-
-    if (dataGatheringInProgress) {
-      return 'IN_PROGRESS';
-    }
-
-    return undefined;
-  }
-
   private async getUsersWithAnalytics(): Promise<AdminData['users']> {
     const usersWithAnalytics = await this.prismaService.user.findMany({
       orderBy: {

apps/api/src/app/app.controller.ts

@@ -1,21 +1,6 @@
-import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { Controller } from '@nestjs/common';
 
 @Controller()
 export class AppController {
-  public constructor(
-    private readonly dataGatheringService: DataGatheringService
-  ) {
-    this.initialize();
-  }
-
-  private async initialize() {
-    const isDataGatheringInProgress =
-      await this.dataGatheringService.getIsInProgress();
-
-    if (isDataGatheringInProgress) {
-      // Prepare for automatical data gathering, if hung up in progress state
-      await this.dataGatheringService.reset();
-    }
-  }
+  public constructor() {}
 }

apps/api/src/app/cache/cache.controller.ts

@@ -36,8 +36,6 @@ export class CacheController {
       );
     }
 
-    this.redisCacheService.reset();
-
-    return this.cacheService.flush();
+    return this.redisCacheService.reset();
   }
 }

apps/api/src/app/cache/cache.service.ts

@@ -1,15 +1,6 @@
-import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { Injectable } from '@nestjs/common';
 
 @Injectable()
 export class CacheService {
-  public constructor(
-    private readonly dataGaterhingService: DataGatheringService
-  ) {}
-
-  public async flush(): Promise<void> {
-    await this.dataGaterhingService.reset();
-
-    return;
-  }
+  public constructor() {}
 }

apps/api/src/app/info/info.service.ts

@@ -106,7 +106,6 @@ export class InfoService {
       baseCurrency: this.configurationService.get('BASE_CURRENCY'),
       currencies: this.exchangeRateDataService.getCurrencies(),
       demoAuthToken: this.getDemoAuthToken(),
-      lastDataGathering: await this.getLastDataGathering(),
       statistics: await this.getStatistics(),
       subscriptions: await this.getSubscriptions(),
       tags: await this.tagService.get()
@@ -215,13 +214,6 @@ export class InfoService {
     });
   }
 
-  private async getLastDataGathering() {
-    const lastDataGathering =
-      await this.dataGatheringService.getLastDataGathering();
-
-    return lastDataGathering ?? null;
-  }
-
   private async getStatistics() {
     if (!this.configurationService.get('ENABLE_FEATURE_STATISTICS')) {
       return undefined;

apps/api/src/services/cron.service.ts

@@ -23,8 +23,8 @@ export class CronService {
     private readonly twitterBotService: TwitterBotService
   ) {}
 
-  @Cron(CronExpression.EVERY_MINUTE)
-  public async runEveryMinute() {
+  @Cron(CronExpression.EVERY_HOUR)
+  public async runEveryHour() {
     await this.dataGatheringService.gather7Days();
   }
 

apps/api/src/services/data-gathering.service.ts

@@ -2,9 +2,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.se
 import {
   DATA_GATHERING_QUEUE,
   DATA_GATHERING_QUEUE_PRIORITY_LOW,
-  GATHER_HISTORICAL_MARKET_DATA_PROCESS,
-  PROPERTY_LAST_DATA_GATHERING,
-  PROPERTY_LOCKED_DATA_GATHERING
+  GATHER_HISTORICAL_MARKET_DATA_PROCESS
 } from '@ghostfolio/common/config';
 import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
@@ -12,7 +10,7 @@ import { InjectQueue } from '@nestjs/bull';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
 import { Queue } from 'bull';
-import { differenceInHours, format, subDays } from 'date-fns';
+import { format, subDays } from 'date-fns';
 import ms from 'ms';
 
 import { DataProviderService } from './data-provider/data-provider.service';
@@ -37,155 +35,23 @@ export class DataGatheringService {
   ) {}
 
   public async gather7Days() {
-    const isDataGatheringNeeded = await this.isDataGatheringNeeded();
-
-    if (isDataGatheringNeeded) {
-      Logger.log('7d data gathering has been started.', 'DataGatheringService');
-      console.time('data-gathering-7d');
-
-      await this.prismaService.property.create({
-        data: {
-          key: PROPERTY_LOCKED_DATA_GATHERING,
-          value: new Date().toISOString()
-        }
-      });
-
     const dataGatheringItems = await this.getSymbols7D();
-
-      try {
     await this.gatherSymbols(dataGatheringItems);
-
-        await this.prismaService.property.upsert({
-          create: {
-            key: PROPERTY_LAST_DATA_GATHERING,
-            value: new Date().toISOString()
-          },
-          update: { value: new Date().toISOString() },
-          where: { key: PROPERTY_LAST_DATA_GATHERING }
-        });
-      } catch (error) {
-        Logger.error(error, 'DataGatheringService');
-      }
-
-      await this.prismaService.property.delete({
-        where: {
-          key: PROPERTY_LOCKED_DATA_GATHERING
-        }
-      });
-
-      Logger.log(
-        '7d data gathering has been completed.',
-        'DataGatheringService'
-      );
-      console.timeEnd('data-gathering-7d');
-    }
   }
 
   public async gatherMax() {
-    const isDataGatheringLocked = await this.prismaService.property.findUnique({
-      where: { key: PROPERTY_LOCKED_DATA_GATHERING }
-    });
-
-    if (!isDataGatheringLocked) {
-      Logger.log(
-        'Max data gathering has been started.',
-        'DataGatheringService'
-      );
-      console.time('data-gathering-max');
-
-      await this.prismaService.property.create({
-        data: {
-          key: PROPERTY_LOCKED_DATA_GATHERING,
-          value: new Date().toISOString()
-        }
-      });
-
-      const symbols = await this.getSymbolsMax();
-
-      try {
-        await this.gatherSymbols(symbols);
-
-        await this.prismaService.property.upsert({
-          create: {
-            key: PROPERTY_LAST_DATA_GATHERING,
-            value: new Date().toISOString()
-          },
-          update: { value: new Date().toISOString() },
-          where: { key: PROPERTY_LAST_DATA_GATHERING }
-        });
-      } catch (error) {
-        Logger.error(error, 'DataGatheringService');
-      }
-
-      await this.prismaService.property.delete({
-        where: {
-          key: PROPERTY_LOCKED_DATA_GATHERING
-        }
-      });
-
-      Logger.log(
-        'Max data gathering has been completed.',
-        'DataGatheringService'
-      );
-      console.timeEnd('data-gathering-max');
-    }
+    const dataGatheringItems = await this.getSymbolsMax();
+    await this.gatherSymbols(dataGatheringItems);
   }
 
   public async gatherSymbol({ dataSource, symbol }: UniqueAsset) {
-    const isDataGatheringLocked = await this.prismaService.property.findUnique({
-      where: { key: PROPERTY_LOCKED_DATA_GATHERING }
-    });
-
-    if (!isDataGatheringLocked) {
-      Logger.log(
-        `Symbol data gathering for ${symbol} has been started.`,
-        'DataGatheringService'
-      );
-      console.time('data-gathering-symbol');
-
-      await this.prismaService.property.create({
-        data: {
-          key: PROPERTY_LOCKED_DATA_GATHERING,
-          value: new Date().toISOString()
-        }
-      });
-
-      const symbols = (await this.getSymbolsMax()).filter(
-        (dataGatheringItem) => {
+    const symbols = (await this.getSymbolsMax()).filter((dataGatheringItem) => {
       return (
         dataGatheringItem.dataSource === dataSource &&
         dataGatheringItem.symbol === symbol
       );
-        }
-      );
-
-      try {
-        await this.gatherSymbols(symbols);
-
-        await this.prismaService.property.upsert({
-          create: {
-            key: PROPERTY_LAST_DATA_GATHERING,
-            value: new Date().toISOString()
-          },
-          update: { value: new Date().toISOString() },
-          where: { key: PROPERTY_LAST_DATA_GATHERING }
     });
-      } catch (error) {
-        Logger.error(error, 'DataGatheringService');
-      }
-
-      await this.prismaService.property.delete({
-        where: {
-          key: PROPERTY_LOCKED_DATA_GATHERING
-        }
-      });
-
-      Logger.log(
-        `Symbol data gathering for ${symbol} has been completed.`,
-        'DataGatheringService'
-      );
-      console.timeEnd('data-gathering-symbol');
-    }
+    await this.gatherSymbols(symbols);
   }
 
   public async gatherSymbolForDate({
@@ -358,34 +224,6 @@ export class DataGatheringService {
     }
   }
 
-  public async getDataGatheringProgress() {
-    const isInProgress = await this.getIsInProgress();
-
-    if (isInProgress) {
-      return this.dataGatheringProgress;
-    }
-
-    return undefined;
-  }
-
-  public async getIsInProgress() {
-    return await this.prismaService.property.findUnique({
-      where: { key: PROPERTY_LOCKED_DATA_GATHERING }
-    });
-  }
-
-  public async getLastDataGathering() {
-    const lastDataGathering = await this.prismaService.property.findUnique({
-      where: { key: PROPERTY_LAST_DATA_GATHERING }
-    });
-
-    if (lastDataGathering?.value) {
-      return new Date(lastDataGathering.value);
-    }
-
-    return undefined;
-  }
-
   public async getSymbolsMax(): Promise<IDataGatheringItem[]> {
     const startDate =
       (
@@ -454,19 +292,6 @@ export class DataGatheringService {
       });
   }
 
-  public async reset() {
-    Logger.log('Data gathering has been reset.', 'DataGatheringService');
-
-    await this.prismaService.property.deleteMany({
-      where: {
-        OR: [
-          { key: PROPERTY_LAST_DATA_GATHERING },
-          { key: PROPERTY_LOCKED_DATA_GATHERING }
-        ]
-      }
-    });
-  }
-
   private async getSymbols7D(): Promise<IDataGatheringItem[]> {
     const startDate = subDays(resetHours(new Date()), 7);
 
@@ -529,16 +354,4 @@ export class DataGatheringService {
 
     return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
-
-  private async isDataGatheringNeeded() {
-    const lastDataGathering = await this.getLastDataGathering();
-
-    const isDataGatheringLocked = await this.prismaService.property.findUnique({
-      where: { key: PROPERTY_LOCKED_DATA_GATHERING }
-    });
-
-    const diffInHours = differenceInHours(new Date(), lastDataGathering);
-
-    return (diffInHours >= 1 || !lastDataGathering) && !isDataGatheringLocked;
-  }
 }

apps/client/src/app/components/admin-overview/admin-overview.component.ts

@@ -15,7 +15,6 @@ import { hasPermission, permissions } from '@ghostfolio/common/permissions';
 import {
   differenceInSeconds,
   formatDistanceToNowStrict,
-  isValid,
   parseISO
 } from 'date-fns';
 import { uniq } from 'lodash';
@@ -32,14 +31,11 @@ export class AdminOverviewComponent implements OnDestroy, OnInit {
   public couponDuration: StringValue = '30 days';
   public coupons: Coupon[];
   public customCurrencies: string[];
-  public dataGatheringInProgress: boolean;
-  public dataGatheringProgress: number;
   public exchangeRates: { label1: string; label2: string; value: number }[];
   public hasPermissionForSubscription: boolean;
   public hasPermissionForSystemMessage: boolean;
   public hasPermissionToToggleReadOnlyMode: boolean;
   public info: InfoItem;
-  public lastDataGathering: string;
   public transactionCount: number;
   public userCount: number;
   public user: User;
@@ -128,7 +124,7 @@ export class AdminOverviewComponent implements OnDestroy, OnInit {
   public onDeleteCoupon(aCouponCode: string) {
     const confirmation = confirm('Do you really want to delete this coupon?');
 
-    if (confirmation) {
+    if (confirmation === true) {
       const coupons = this.coupons.filter((coupon) => {
         return coupon.code !== aCouponCode;
       });
@@ -139,7 +135,7 @@ export class AdminOverviewComponent implements OnDestroy, OnInit {
   public onDeleteCurrency(aCurrency: string) {
     const confirmation = confirm('Do you really want to delete this currency?');
 
-    if (confirmation) {
+    if (confirmation === true) {
       const currencies = this.customCurrencies.filter((currency) => {
         return currency !== aCurrency;
       });
@@ -152,6 +148,9 @@ export class AdminOverviewComponent implements OnDestroy, OnInit {
   }
 
   public onFlushCache() {
+    const confirmation = confirm('Do you really want to flush the cache?');
+
+    if (confirmation === true) {
       this.cacheService
         .flush()
         .pipe(takeUntil(this.unsubscribeSubject))
@@ -161,13 +160,20 @@ export class AdminOverviewComponent implements OnDestroy, OnInit {
           }, 300);
         });
     }
+  }
 
-  public onGatherMax() {
-    const confirmation = confirm(
-      'This action may take some time. Do you want to proceed?'
-    );
+  public onGather7Days() {
+    this.adminService
+      .gather7Days()
+      .pipe(takeUntil(this.unsubscribeSubject))
+      .subscribe(() => {
+        setTimeout(() => {
+          window.location.reload();
+        }, 300);
+      });
+  }
 
-    if (confirmation === true) {
+  public onGatherMax() {
     this.adminService
       .gatherMax()
       .pipe(takeUntil(this.unsubscribeSubject))
@@ -177,7 +183,6 @@ export class AdminOverviewComponent implements OnDestroy, OnInit {
         }, 300);
       });
   }
-  }
 
   public onGatherProfileData() {
     this.adminService
@@ -207,39 +212,15 @@ export class AdminOverviewComponent implements OnDestroy, OnInit {
     this.dataService
       .fetchAdminData()
       .pipe(takeUntil(this.unsubscribeSubject))
-      .subscribe(
-        ({
-          dataGatheringProgress,
-          exchangeRates,
-          lastDataGathering,
-          settings,
-          transactionCount,
-          userCount
-        }) => {
+      .subscribe(({ exchangeRates, settings, transactionCount, userCount }) => {
         this.coupons = (settings[PROPERTY_COUPONS] as Coupon[]) ?? [];
         this.customCurrencies = settings[PROPERTY_CURRENCIES] as string[];
-          this.dataGatheringProgress = dataGatheringProgress;
         this.exchangeRates = exchangeRates;
-
-          if (isValid(parseISO(lastDataGathering?.toString()))) {
-            this.lastDataGathering = formatDistanceToNowStrict(
-              new Date(lastDataGathering),
-              {
-                addSuffix: true
-              }
-            );
-          } else if (lastDataGathering === 'IN_PROGRESS') {
-            this.dataGatheringInProgress = true;
-          } else {
-            this.lastDataGathering = 'Starting soon...';
-          }
-
         this.transactionCount = transactionCount;
         this.userCount = userCount;
 
         this.changeDetectorRef.markForCheck();
-        }
-      );
+      });
   }
 
   private generateCouponCode(aLength: number) {

apps/client/src/app/components/admin-overview/admin-overview.html

@@ -19,37 +19,30 @@
           <div class="d-flex my-3">
             <div class="w-50" i18n>Data Gathering</div>
             <div class="w-50">
-              <div>
-                <ng-container *ngIf="lastDataGathering"
-                  >{{ lastDataGathering }}</ng-container
-                >
-                <ng-container *ngIf="dataGatheringInProgress" i18n
-                  >In Progress ({{ dataGatheringProgress | percent : '1.2-2'
-                  }})</ng-container
-                >
-              </div>
-              <div class="mt-2 overflow-hidden">
+              <div class="overflow-hidden">
                 <div class="mb-2">
                   <button
                     color="accent"
                     mat-flat-button
-                    (click)="onFlushCache()"
+                    (click)="onGather7Days()"
                   >
                     <ion-icon
                       class="mr-1"
-                      name="close-circle-outline"
+                      name="cloud-download-outline"
                     ></ion-icon>
-                    <span i18n>Reset Data Gathering</span>
+                    <span i18n>Start Data Gathering</span>
                   </button>
                 </div>
                 <div class="mb-2">
                   <button
-                    color="warn"
+                    color="accent"
                     mat-flat-button
-                    [disabled]="dataGatheringInProgress"
                     (click)="onGatherMax()"
                   >
-                    <ion-icon class="mr-1" name="warning-outline"></ion-icon>
+                    <ion-icon
+                      class="mr-1"
+                      name="cloud-download-outline"
+                    ></ion-icon>
                     <span i18n>Gather All Data</span>
                   </button>
                 </div>
@@ -58,7 +51,6 @@
                     class="mb-2 mr-2"
                     color="accent"
                     mat-flat-button
-                    [disabled]="dataGatheringInProgress"
                     (click)="onGatherProfileData()"
                   >
                     <ion-icon
@@ -97,7 +89,6 @@
                       *ngIf="customCurrencies.includes(exchangeRate.label2)"
                       class="mini-icon mx-1 no-min-width px-2"
                       mat-button
-                      [disabled]="dataGatheringInProgress"
                       (click)="onDeleteCurrency(exchangeRate.label2)"
                     >
                       <ion-icon name="trash-outline"></ion-icon>
@@ -109,7 +100,6 @@
                 <button
                   color="primary"
                   mat-flat-button
-                  [disabled]="dataGatheringInProgress"
                   (click)="onAddCurrency()"
                 >
                   <ion-icon class="mr-1" name="add-outline"></ion-icon>
@@ -126,7 +116,6 @@
                 <button
                   class="mini-icon mx-1 no-min-width px-2"
                   mat-button
-                  [disabled]="dataGatheringInProgress"
                   (click)="onDeleteSystemMessage()"
                 >
                   <ion-icon name="trash-outline"></ion-icon>
@@ -197,6 +186,15 @@
               </div>
             </div>
           </div>
+          <div class="d-flex my-3">
+            <div class="w-50" i18n>Housekeeping</div>
+            <div class="w-50">
+              <button color="warn" mat-flat-button (click)="onFlushCache()">
+                <ion-icon class="mr-1" name="close-circle-outline"></ion-icon>
+                <span i18n>Flush Cache</span>
+              </button>
+            </div>
+          </div>
         </mat-card-content>
       </mat-card>
     </div>

apps/client/src/app/services/admin.service.ts

@@ -51,6 +51,10 @@ export class AdminService {
     return this.http.get<AdminJobs>(`/api/v1/admin/queue/job`);
   }
 
+  public gather7Days() {
+    return this.http.post<void>(`/api/v1/admin/gather`, {});
+  }
+
   public gatherMax() {
     return this.http.post<void>(`/api/v1/admin/gather/max`, {});
   }

libs/common/src/lib/config.ts

@@ -56,8 +56,6 @@ export const PROPERTY_BENCHMARKS = 'BENCHMARKS';
 export const PROPERTY_COUPONS = 'COUPONS';
 export const PROPERTY_CURRENCIES = 'CURRENCIES';
 export const PROPERTY_IS_READ_ONLY_MODE = 'IS_READ_ONLY_MODE';
-export const PROPERTY_LAST_DATA_GATHERING = 'LAST_DATA_GATHERING';
-export const PROPERTY_LOCKED_DATA_GATHERING = 'LOCKED_DATA_GATHERING';
 export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS';
 export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG';
 export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE';

libs/common/src/lib/interfaces/admin-data.interface.ts

@@ -1,7 +1,5 @@
 export interface AdminData {
-  dataGatheringProgress?: number;
   exchangeRates: { label1: string; label2: string; value: number }[];
-  lastDataGathering?: Date | 'IN_PROGRESS';
   settings: { [key: string]: boolean | object | string | string[] };
   transactionCount: number;
   userCount: number;