Feature/improve data gathering (#576)

* Eliminate benchmarks to gather

* Optimize 7d data gathering

* Update changelog

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Increased the historical data chart of the _Fear & Greed Index_ (market mood) to 30 days
 - Made the import functionality for transactions by `csv` files more flexible
+- Optimized the 7d data gathering (only consider symbols with incomplete market data)
 - Upgraded `prettier` from version `2.3.2` to `2.5.1`
 
 ## 1.93.0 - 21.12.2021

apps/api/src/services/data-gathering.service.ts

@@ -1,12 +1,11 @@
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
 import {
   PROPERTY_LAST_DATA_GATHERING,
-  PROPERTY_LOCKED_DATA_GATHERING,
+  PROPERTY_LOCKED_DATA_GATHERING
-  ghostfolioFearAndGreedIndexSymbol
 } from '@ghostfolio/common/config';
 import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
 import { Inject, Injectable, Logger } from '@nestjs/common';
-import { DataSource, MarketData } from '@prisma/client';
+import { DataSource } from '@prisma/client';
 import {
   differenceInHours,
   format,
@@ -17,7 +16,6 @@ import {
   subDays
 } from 'date-fns';
 
-import { ConfigurationService } from './configuration.service';
 import { DataProviderService } from './data-provider/data-provider.service';
 import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface';
 import { ExchangeRateDataService } from './exchange-rate-data.service';
@@ -29,7 +27,6 @@ export class DataGatheringService {
   private dataGatheringProgress: number;
 
   public constructor(
-    private readonly configurationService: ConfigurationService,
     @Inject('DataEnhancers')
     private readonly dataEnhancers: DataEnhancerInterface[],
     private readonly dataProviderService: DataProviderService,
@@ -448,11 +445,7 @@ export class DataGatheringService {
       };
     });
 
-    return [
+    return [...currencyPairsToGather, ...symbolProfilesToGather];
-      ...this.getBenchmarksToGather(startDate),
-      ...currencyPairsToGather,
-      ...symbolProfilesToGather
-    ];
   }
 
   public async reset() {
@@ -468,23 +461,27 @@ export class DataGatheringService {
     });
   }
 
-  private getBenchmarksToGather(startDate: Date): IDataGatheringItem[] {
-    const benchmarksToGather: IDataGatheringItem[] = [];
-
-    if (this.configurationService.get('ENABLE_FEATURE_FEAR_AND_GREED_INDEX')) {
-      benchmarksToGather.push({
-        dataSource: DataSource.RAKUTEN,
-        date: startDate,
-        symbol: ghostfolioFearAndGreedIndexSymbol
-      });
-    }
-
-    return benchmarksToGather;
-  }
-
   private async getSymbols7D(): Promise<IDataGatheringItem[]> {
     const startDate = subDays(resetHours(new Date()), 7);
 
+    // Only consider symbols with incomplete market data for the last
+    // 7 days
+    const symbolsToGather = (
+      await this.prismaService.marketData.groupBy({
+        _count: true,
+        by: ['symbol'],
+        where: {
+          date: { gt: startDate }
+        }
+      })
+    )
+      .filter((group) => {
+        return group._count < 6;
+      })
+      .map((group) => {
+        return group.symbol;
+      });
+
     const symbolProfilesToGather = (
       await this.prismaService.symbolProfile.findMany({
         orderBy: [{ symbol: 'asc' }],
@@ -494,12 +491,16 @@ export class DataGatheringService {
           symbol: true
         }
       })
-    ).map((symbolProfile) => {
+    )
-      return {
+      .filter((symbolProfile) => {
-        ...symbolProfile,
+        return symbolsToGather.includes(symbolProfile.symbol);
-        date: startDate
+      })
-      };
+      .map((symbolProfile) => {
-    });
+        return {
+          ...symbolProfile,
+          date: startDate
+        };
+      });
 
     const currencyPairsToGather = this.exchangeRateDataService
       .getCurrencyPairs()
@@ -511,30 +512,22 @@ export class DataGatheringService {
         };
       });
 
-    return [
+    return [...currencyPairsToGather, ...symbolProfilesToGather];
-      ...this.getBenchmarksToGather(startDate),
-      ...currencyPairsToGather,
-      ...symbolProfilesToGather
-    ];
   }
 
   private async getSymbolsProfileData(): Promise<IDataGatheringItem[]> {
-    const startDate = subDays(resetHours(new Date()), 7);
-
     const distinctOrders = await this.prismaService.order.findMany({
       distinct: ['symbol'],
       orderBy: [{ symbol: 'asc' }],
       select: { dataSource: true, symbol: true }
     });
 
-    return [...this.getBenchmarksToGather(startDate), ...distinctOrders].filter(
+    return distinctOrders.filter((distinctOrder) => {
-      (distinctOrder) => {
+      return (
-        return (
+        distinctOrder.dataSource !== DataSource.GHOSTFOLIO &&
-          distinctOrder.dataSource !== DataSource.GHOSTFOLIO &&
+        distinctOrder.dataSource !== DataSource.RAKUTEN
-          distinctOrder.dataSource !== DataSource.RAKUTEN
+      );
-        );
+    });
-      }
-    );
   }
 
   private async isDataGatheringNeeded() {