From be52395a93f0313da0000cb75260be29d32cfb26 Mon Sep 17 00:00:00 2001
From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com>
Date: Tue, 16 Apr 2024 20:56:58 +0200
Subject: [PATCH] Let data gathering queue jobs fail by throwing errors

---
 .../src/app/portfolio/portfolio.service.ts    | 18 +++++++++++----
 apps/api/src/app/symbol/symbol.service.ts     | 20 ++++++++++++----
 .../data-gathering.processor.ts               | 20 ++++++++++++----
 .../data-gathering/data-gathering.service.ts  | 23 ++++++++-----------
 .../data-provider/data-provider.service.ts    | 18 +++++++++------
 5 files changed, 63 insertions(+), 36 deletions(-)

diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts
index 95a68eaae..4c65d2eb9 100644
--- a/apps/api/src/app/portfolio/portfolio.service.ts
+++ b/apps/api/src/app/portfolio/portfolio.service.ts
@@ -822,11 +822,19 @@ export class PortfolioService {
       );
 
       if (isEmpty(historicalData)) {
-        historicalData = await this.dataProviderService.getHistoricalRaw(
-          [{ dataSource: DataSource.YAHOO, symbol: aSymbol }],
-          portfolioStart,
-          new Date()
-        );
+        try {
+          historicalData = await this.dataProviderService.getHistoricalRaw({
+            dataGatheringItems: [
+              { dataSource: DataSource.YAHOO, symbol: aSymbol }
+            ],
+            from: portfolioStart,
+            to: new Date()
+          });
+        } catch {
+          historicalData = {
+            [aSymbol]: {}
+          };
+        }
       }
 
       const historicalDataArray: HistoricalDataItem[] = [];
diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts
index 9a3f7a3a0..90259a776 100644
--- a/apps/api/src/app/symbol/symbol.service.ts
+++ b/apps/api/src/app/symbol/symbol.service.ts
@@ -74,11 +74,21 @@ export class SymbolService {
     date = new Date(),
     symbol
   }: IDataGatheringItem): Promise<IDataProviderHistoricalResponse> {
-    const historicalData = await this.dataProviderService.getHistoricalRaw(
-      [{ dataSource, symbol }],
-      date,
-      date
-    );
+    let historicalData: {
+      [symbol: string]: {
+        [date: string]: IDataProviderHistoricalResponse;
+      };
+    } = {
+      [symbol]: {}
+    };
+
+    try {
+      historicalData = await this.dataProviderService.getHistoricalRaw({
+        dataGatheringItems: [{ dataSource, symbol }],
+        from: date,
+        to: date
+      });
+    } catch {}
 
     return {
       marketPrice:
diff --git a/apps/api/src/services/data-gathering/data-gathering.processor.ts b/apps/api/src/services/data-gathering/data-gathering.processor.ts
index bf960048c..11eda2e7a 100644
--- a/apps/api/src/services/data-gathering/data-gathering.processor.ts
+++ b/apps/api/src/services/data-gathering/data-gathering.processor.ts
@@ -37,7 +37,17 @@ export class DataGatheringProcessor {
   @Process({ concurrency: 1, name: GATHER_ASSET_PROFILE_PROCESS })
   public async gatherAssetProfile(job: Job<UniqueAsset>) {
     try {
+      Logger.log(
+        `Asset profile data gathering has been started for ${job.data.symbol} (${job.data.dataSource})`,
+        `DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS})`
+      );
+
       await this.dataGatheringService.gatherAssetProfiles([job.data]);
+
+      Logger.log(
+        `Asset profile data gathering has been completed for ${job.data.symbol} (${job.data.dataSource})`,
+        `DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS})`
+      );
     } catch (error) {
       Logger.error(
         error,
@@ -62,11 +72,11 @@ export class DataGatheringProcessor {
         `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS})`
       );
 
-      const historicalData = await this.dataProviderService.getHistoricalRaw(
-        [{ dataSource, symbol }],
-        currentDate,
-        new Date()
-      );
+      const historicalData = await this.dataProviderService.getHistoricalRaw({
+        dataGatheringItems: [{ dataSource, symbol }],
+        from: currentDate,
+        to: new Date()
+      });
 
       const data: Prisma.MarketDataUpdateInput[] = [];
       let lastMarketPrice: number;
diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.ts
index b2b0c371c..31a0040a5 100644
--- a/apps/api/src/services/data-gathering/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering/data-gathering.service.ts
@@ -104,11 +104,11 @@ export class DataGatheringService {
     symbol: string;
   }) {
     try {
-      const historicalData = await this.dataProviderService.getHistoricalRaw(
-        [{ dataSource, symbol }],
-        date,
-        date
-      );
+      const historicalData = await this.dataProviderService.getHistoricalRaw({
+        dataGatheringItems: [{ dataSource, symbol }],
+        from: date,
+        to: date
+      });
 
       const marketPrice =
         historicalData[symbol][format(date, DATE_FORMAT)].marketPrice;
@@ -230,17 +230,12 @@ export class DataGatheringService {
           error,
           'DataGatheringService'
         );
+
+        if (uniqueAssets.length === 1) {
+          throw error;
+        }
       }
     }
-
-    Logger.log(
-      `Asset profile data gathering has been completed for ${uniqueAssets
-        .map(({ dataSource, symbol }) => {
-          return `${symbol} (${dataSource})`;
-        })
-        .join(',')}.`,
-      'DataGatheringService'
-    );
   }
 
   public async gatherSymbols({
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index 9afdab1dc..675c13377 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -233,15 +233,17 @@ export class DataProviderService {
     }
   }
 
-  public async getHistoricalRaw(
-    aDataGatheringItems: UniqueAsset[],
-    from: Date,
-    to: Date
-  ): Promise<{
+  public async getHistoricalRaw({
+    dataGatheringItems,
+    from,
+    to
+  }: {
+    dataGatheringItems: UniqueAsset[];
+    from: Date;
+    to: Date;
+  }): Promise<{
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }> {
-    let dataGatheringItems = aDataGatheringItems;
-
     for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
       if (
         this.hasCurrency({
@@ -330,6 +332,8 @@ export class DataProviderService {
       }
     } catch (error) {
       Logger.error(error, 'DataProviderService');
+
+      throw error;
     }
 
     return result;