Optimize 7d data gathering by prioritization

apps/api/src/services/data-gathering/data-gathering.service.ts

@@ -10,6 +10,7 @@ import {
   DATA_GATHERING_QUEUE,
   DATA_GATHERING_QUEUE_PRIORITY_HIGH,
   DATA_GATHERING_QUEUE_PRIORITY_LOW,
+  DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
   PROPERTY_BENCHMARKS
@@ -62,9 +63,22 @@ export class DataGatheringService {
   }
 
   public async gather7Days() {
-    const dataGatheringItems = await this.getSymbols7D();
     await this.gatherSymbols({
-      dataGatheringItems,
+      dataGatheringItems: await this.getCurrencies7D(),
+      priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+    });
+
+    await this.gatherSymbols({
+      dataGatheringItems: await this.getSymbols7D({
+        withUserSubscription: true
+      }),
+      priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
+    });
+
+    await this.gatherSymbols({
+      dataGatheringItems: await this.getSymbols7D({
+        withUserSubscription: false
+      }),
       priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
     });
   }
@@ -290,43 +304,46 @@ export class DataGatheringService {
       });
   }
 
+  private async getCurrencies7D(): Promise<IDataGatheringItem[]> {
+    const startDate = subDays(resetHours(new Date()), 7);
+
+    const symbolsWithCompleteMarketData =
+      await this.getSymbolsWithCompleteMarketData();
+
+    return this.exchangeRateDataService
+      .getCurrencyPairs()
+      .filter(({ symbol }) => {
+        return !symbolsWithCompleteMarketData.includes(symbol);
+      })
+      .map(({ dataSource, symbol }) => {
+        return {
+          dataSource,
+          symbol,
+          date: startDate
+        };
+      });
+  }
+
   private getEarliestDate(aStartDate: Date) {
     return min([aStartDate, subYears(new Date(), 10)]);
   }
 
-  private async getSymbols7D(): Promise<IDataGatheringItem[]> {
+  private async getSymbols7D({
+    withUserSubscription = false
+  }: {
+    withUserSubscription?: boolean;
+  }): Promise<IDataGatheringItem[]> {
     const startDate = subDays(resetHours(new Date()), 7);
 
-    const symbolProfiles = await this.prismaService.symbolProfile.findMany({
-      orderBy: [{ symbol: 'asc' }],
-      select: {
-        dataSource: true,
-        scraperConfiguration: true,
-        symbol: true
-      }
-    });
-
-    // Only consider symbols with incomplete market data for the last
-    // 7 days
-    const symbolsWithCompleteMarketData = (
-      await this.prismaService.marketData.groupBy({
-        _count: true,
-        by: ['symbol'],
-        orderBy: [{ symbol: 'asc' }],
-        where: {
-          date: { gt: startDate },
-          state: 'CLOSE'
-        }
-      })
-    )
-      .filter((group) => {
-        return group._count >= 6;
-      })
-      .map((group) => {
-        return group.symbol;
+    const symbolProfiles =
+      await this.symbolProfileService.getSymbolProfilesByUserSubscription({
+        withUserSubscription
       });
 
-    const symbolProfilesToGather = symbolProfiles
+    const symbolsWithCompleteMarketData =
+      await this.getSymbolsWithCompleteMarketData();
+
+    return symbolProfiles
       .filter(({ dataSource, scraperConfiguration, symbol }) => {
         const manualDataSourceWithScraperConfiguration =
           dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);
@@ -342,21 +359,6 @@ export class DataGatheringService {
           date: startDate
         };
       });
-
-    const currencyPairsToGather = this.exchangeRateDataService
-      .getCurrencyPairs()
-      .filter(({ symbol }) => {
-        return !symbolsWithCompleteMarketData.includes(symbol);
-      })
-      .map(({ dataSource, symbol }) => {
-        return {
-          dataSource,
-          symbol,
-          date: startDate
-        };
-      });
-
-    return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 
   private async getSymbolsMax(): Promise<IDataGatheringItem[]> {
@@ -426,4 +428,26 @@ export class DataGatheringService {
 
     return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
+
+  private async getSymbolsWithCompleteMarketData() {
+    const startDate = subDays(resetHours(new Date()), 7);
+
+    return (
+      await this.prismaService.marketData.groupBy({
+        _count: true,
+        by: ['symbol'],
+        orderBy: [{ symbol: 'asc' }],
+        where: {
+          date: { gt: startDate },
+          state: 'CLOSE'
+        }
+      })
+    )
+      .filter(({ _count }) => {
+        return _count >= 6;
+      })
+      .map(({ symbol }) => {
+        return symbol;
+      });
+  }
 }

apps/api/src/services/data-provider/data-provider.service.ts

@@ -9,7 +9,6 @@ import {
 import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import {
   DEFAULT_CURRENCY,
   DERIVED_CURRENCIES,
@@ -37,8 +36,7 @@ export class DataProviderService {
     private readonly marketDataService: MarketDataService,
     private readonly prismaService: PrismaService,
     private readonly propertyService: PropertyService,
-    private readonly redisCacheService: RedisCacheService,
-    private readonly symbolProfileService: SymbolProfileService
+    private readonly redisCacheService: RedisCacheService
   ) {
     this.initialize();
   }
@@ -48,26 +46,6 @@ export class DataProviderService {
       ((await this.propertyService.getByKey(PROPERTY_DATA_SOURCE_MAPPING)) as {
         [dataProviderName: string]: string;
       }) ?? {};
-
-    const withSubscription =
-      await this.symbolProfileService.getSymbolProfilesWithSubscription(true);
-    console.log('withSubscription', withSubscription.length);
-
-    // const vwrl = withSubscription.find(({ symbol }) => {
-    //   return symbol === 'VWRL.SW';
-    // });
-
-    // console.log({ vwrl });
-
-    const withoutSubscription =
-      await this.symbolProfileService.getSymbolProfilesWithSubscription(false);
-    console.log('withoutSubscription', withoutSubscription.length);
-
-    console.log(
-      'Total: ',
-      withSubscription.length + withoutSubscription.length,
-      await this.prismaService.symbolProfile.count()
-    );
   }
 
   public async checkQuote(dataSource: DataSource) {

apps/api/src/services/symbol-profile/symbol-profile.service.ts

@@ -91,7 +91,11 @@ export class SymbolProfileService {
       });
   }
 
-  public async getSymbolProfilesWithSubscription(withSubscription = true) {
+  public async getSymbolProfilesByUserSubscription({
+    withUserSubscription = false
+  }: {
+    withUserSubscription?: boolean;
+  }) {
     return this.prismaService.symbolProfile.findMany({
       include: {
         Order: {
@@ -100,8 +104,9 @@ export class SymbolProfileService {
           }
         }
       },
+      orderBy: [{ symbol: 'asc' }],
       where: {
-        Order: withSubscription
+        Order: withUserSubscription
           ? {
               some: {
                 User: {