Feature/introduce data gathering progress (#467)

* Add data gathering progress

* Update changelog

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added a logo to the log on the server start
+- Added the data gathering progress to the log and the admin control panel
 
 ## 1.74.0 - 11.11.2021
 

apps/api/src/app/admin/admin.service.ts

@@ -20,6 +20,8 @@ export class AdminService {
 
   public async get(): Promise<AdminData> {
     return {
+      dataGatheringProgress:
+        await this.dataGatheringService.getDataGatheringProgress(),
       exchangeRates: this.exchangeRateDataService
         .getCurrencies()
         .filter((currency) => {
@@ -58,7 +60,7 @@ export class AdminService {
       return 'IN_PROGRESS';
     }
 
-    return null;
+    return undefined;
   }
 
   private async getUsersWithAnalytics(): Promise<AdminData['users']> {

apps/api/src/services/data-gathering.service.ts

@@ -18,7 +18,6 @@ import {
 
 import { ConfigurationService } from './configuration.service';
 import { DataProviderService } from './data-provider/data-provider.service';
-import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface';
 import { ExchangeRateDataService } from './exchange-rate-data.service';
 import { IDataGatheringItem } from './interfaces/interfaces';
@@ -26,13 +25,14 @@ import { PrismaService } from './prisma.service';
 
 @Injectable()
 export class DataGatheringService {
+  private dataGatheringProgress: number;
+
   public constructor(
     private readonly configurationService: ConfigurationService,
     @Inject('DataEnhancers')
     private readonly dataEnhancers: DataEnhancerInterface[],
     private readonly dataProviderService: DataProviderService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
-    private readonly ghostfolioScraperApi: GhostfolioScraperApiService,
     private readonly prismaService: PrismaService,
     private readonly symbolProfileService: SymbolProfileService
   ) {}
@@ -204,8 +204,11 @@ export class DataGatheringService {
 
   public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
     let hasError = false;
+    let symbolCounter = 0;
 
     for (const { dataSource, date, symbol } of aSymbolsWithStartDate) {
+      this.dataGatheringProgress = symbolCounter / aSymbolsWithStartDate.length;
+
       try {
         const historicalData = await this.dataProviderService.getHistoricalRaw(
           [{ dataSource, symbol }],
@@ -263,6 +266,16 @@ export class DataGatheringService {
         hasError = true;
         Logger.error(error);
       }
+
+      if (symbolCounter > 0 && symbolCounter % 100 === 0) {
+        Logger.log(
+          `Data gathering progress: ${(
+            this.dataGatheringProgress * 100
+          ).toFixed(2)}%`
+        );
+      }
+
+      symbolCounter += 1;
     }
 
     await this.exchangeRateDataService.initialize();
@@ -272,6 +285,16 @@ export class DataGatheringService {
     }
   }
 
+  public async getDataGatheringProgress() {
+    const isInProgress = await this.getIsInProgress();
+
+    if (isInProgress) {
+      return this.dataGatheringProgress;
+    }
+
+    return undefined;
+  }
+
   public async getIsInProgress() {
     return await this.prismaService.property.findUnique({
       where: { key: 'LOCKED_DATA_GATHERING' }

apps/client/src/app/pages/admin/admin-page.component.ts

@@ -22,6 +22,7 @@ import { takeUntil } from 'rxjs/operators';
 })
 export class AdminPageComponent implements OnDestroy, OnInit {
   public dataGatheringInProgress: boolean;
+  public dataGatheringProgress: number;
   public defaultDateFormat = DEFAULT_DATE_FORMAT;
   public exchangeRates: { label1: string; label2: string; value: number }[];
   public lastDataGathering: string;
@@ -134,12 +135,14 @@ export class AdminPageComponent implements OnDestroy, OnInit {
       .pipe(takeUntil(this.unsubscribeSubject))
       .subscribe(
         ({
+          dataGatheringProgress,
           exchangeRates,
           lastDataGathering,
           transactionCount,
           userCount,
           users
         }) => {
+          this.dataGatheringProgress = dataGatheringProgress;
           this.exchangeRates = exchangeRates;
           this.users = users;
 

apps/client/src/app/pages/admin/admin-page.html

@@ -35,14 +35,15 @@
             </div>
           </div>
           <div class="d-flex my-3">
-            <div class="w-50" i18n>Last Data Gathering</div>
+            <div class="w-50" i18n>Data Gathering</div>
             <div class="w-50">
               <div>
                 <ng-container *ngIf="lastDataGathering"
                   >{{ lastDataGathering }}</ng-container
                 >
                 <ng-container *ngIf="dataGatheringInProgress" i18n
-                  >In Progress</ng-container
+                  >In Progress ({{ dataGatheringProgress | percent : '1.2-2'
+                  }})</ng-container
                 >
               </div>
               <div class="mt-2 overflow-hidden">

libs/common/src/lib/interfaces/admin-data.interface.ts

@@ -1,6 +1,7 @@
 export interface AdminData {
+  dataGatheringProgress?: number;
   exchangeRates: { label1: string; label2: string; value: number }[];
-  lastDataGathering: Date | 'IN_PROGRESS';
+  lastDataGathering?: Date | 'IN_PROGRESS';
   transactionCount: number;
   userCount: number;
   users: {