From 4da7505b4fda566e85f327fc978c2525f7ec7d39 Mon Sep 17 00:00:00 2001
From: Thomas <4159106+dtslvr@users.noreply.github.com>
Date: Sat, 25 Dec 2021 13:53:44 +0100
Subject: [PATCH] Optimize 7d data gathering

---
 .../src/services/data-gathering.service.ts    | 34 +++++++++++++++----
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts
index 87d5f72a0..1ee6ee221 100644
--- a/apps/api/src/services/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering.service.ts
@@ -464,6 +464,24 @@ export class DataGatheringService {
   private async getSymbols7D(): Promise<IDataGatheringItem[]> {
     const startDate = subDays(resetHours(new Date()), 7);
 
+    // Only consider symbols with incomplete market data for the last
+    // 7 days
+    const symbolsToGather = (
+      await this.prismaService.marketData.groupBy({
+        _count: true,
+        by: ['symbol'],
+        where: {
+          date: { gt: startDate }
+        }
+      })
+    )
+      .filter((group) => {
+        return group._count < 6;
+      })
+      .map((group) => {
+        return group.symbol;
+      });
+
     const symbolProfilesToGather = (
       await this.prismaService.symbolProfile.findMany({
         orderBy: [{ symbol: 'asc' }],
@@ -473,12 +491,16 @@ export class DataGatheringService {
           symbol: true
         }
       })
-    ).map((symbolProfile) => {
-      return {
-        ...symbolProfile,
-        date: startDate
-      };
-    });
+    )
+      .filter((symbolProfile) => {
+        return symbolsToGather.includes(symbolProfile.symbol);
+      })
+      .map((symbolProfile) => {
+        return {
+          ...symbolProfile,
+          date: startDate
+        };
+      });
 
     const currencyPairsToGather = this.exchangeRateDataService
       .getCurrencyPairs()