Improve market data gathering robustness

apps/api/src/services/market-data/market-data.service.ts

@@ -1,6 +1,7 @@
 import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.interface';
 import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
+import { DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_TIMEOUT } from '@ghostfolio/common/config';
 import { UpdateMarketDataDto } from '@ghostfolio/common/dtos';
 import { resetHours } from '@ghostfolio/common/helper';
 import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
@@ -155,49 +156,52 @@ export class MarketDataService {
     dataSource,
     symbol
   }: AssetProfileIdentifier & { data: Prisma.MarketDataUpdateInput[] }) {
-    await this.prismaService.$transaction(async (prisma) => {
+    await this.prismaService.$transaction(
-      if (data.length > 0) {
+      async (prisma) => {
-        let minTime = Infinity;
+        if (data.length > 0) {
-        let maxTime = -Infinity;
+          let minTime = Infinity;
+          let maxTime = -Infinity;
 
-        for (const { date } of data) {
+          for (const { date } of data) {
-          const time = (date as Date).getTime();
+            const time = (date as Date).getTime();
 
-          if (time < minTime) {
+            if (time < minTime) {
-            minTime = time;
+              minTime = time;
-          }
+            }
 
-          if (time > maxTime) {
+            if (time > maxTime) {
-            maxTime = time;
+              maxTime = time;
+            }
           }
-        }
 
-        const minDate = new Date(minTime);
+          const minDate = new Date(minTime);
-        const maxDate = new Date(maxTime);
+          const maxDate = new Date(maxTime);
 
-        await prisma.marketData.deleteMany({
+          await prisma.marketData.deleteMany({
-          where: {
+            where: {
-            dataSource,
+              dataSource,
-            symbol,
+              symbol,
-            date: {
+              date: {
-              gte: minDate,
+                gte: minDate,
-              lte: maxDate
+                lte: maxDate
+              }
             }
-          }
+          });
-        });
 
-        await prisma.marketData.createMany({
+          await prisma.marketData.createMany({
-          data: data.map(({ date, marketPrice, state }) => ({
+            data: data.map(({ date, marketPrice, state }) => ({
-            dataSource,
+              dataSource,
-            symbol,
+              symbol,
-            date: date as Date,
+              date: date as Date,
-            marketPrice: marketPrice as number,
+              marketPrice: marketPrice as number,
-            state: state as MarketDataState
+              state: state as MarketDataState
-          })),
+            })),
-          skipDuplicates: true
+            skipDuplicates: true
-        });
+          });
-      }
+        }
-    });
+      },
+      { timeout: DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_TIMEOUT }
+    );
   }
 
   public async updateAssetProfileIdentifier(

libs/common/src/lib/config.ts

@@ -90,6 +90,7 @@ export const DEFAULT_PAGE_SIZE = 50;
 export const DEFAULT_PORT = 3333;
 export const DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY = 1;
 export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY = 1;
+export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_TIMEOUT = 60000;
 export const DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY = 1;
 export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY = 1;
 export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000;