Feature/improve error handling in data providers (part 2) (#5413)

* Improve error handling

* Update changelog

CHANGELOG.md

@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Moved the support to customize rules in the _X-ray_ section from experimental to general availability
 - Improved the create or update activity dialog’s asset sub class selector for valuables to update the options dynamically based on the selected asset class
+- Improved the error handling in data providers
 - Randomized the minutes of the hourly data gathering cron job
 - Refactored the dialog footer component to standalone
 - Refactored the dialog header component to standalone

apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts

@@ -163,7 +163,7 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface {
   public async getAssetProfile(
     aSymbol: string
   ): Promise<Partial<SymbolProfile>> {
-    const response: Partial<SymbolProfile> = {};
+    let response: Partial<SymbolProfile> = {};
 
     try {
       let symbol = aSymbol;
@@ -241,10 +241,13 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface {
       }
 
       const url = assetProfile.summaryProfile?.website;
+
       if (url) {
         response.url = url;
       }
     } catch (error) {
+      response = undefined;
+
       if (error.message === `Quote not found for symbol: ${aSymbol}`) {
         throw new AssetProfileDelistedError(
           `No data found, ${aSymbol} (${this.getName()}) may be delisted`

apps/api/src/services/data-provider/data-provider.service.ts

@@ -107,7 +107,9 @@ export class DataProviderService implements OnModuleInit {
 
         promises.push(
           promise.then((symbolProfile) => {
-            response[symbol] = symbolProfile;
+            if (symbolProfile) {
+              response[symbol] = symbolProfile;
+            }
           })
         );
       }

apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts

@@ -55,14 +55,18 @@ export class EodHistoricalDataService implements DataProviderInterface {
   }: GetAssetProfileParams): Promise<Partial<SymbolProfile>> {
     const [searchResult] = await this.getSearchResult(symbol);
 
+    if (!searchResult) {
+      return undefined;
+    }
+
     return {
       symbol,
-      assetClass: searchResult?.assetClass,
-      assetSubClass: searchResult?.assetSubClass,
-      currency: this.convertCurrency(searchResult?.currency),
+      assetClass: searchResult.assetClass,
+      assetSubClass: searchResult.assetSubClass,
+      currency: this.convertCurrency(searchResult.currency),
       dataSource: this.getName(),
-      isin: searchResult?.isin,
-      name: searchResult?.name
+      isin: searchResult.isin,
+      name: searchResult.name
     };
   }
 

apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts

@@ -64,7 +64,7 @@ export class FinancialModelingPrepService implements DataProviderInterface {
     requestTimeout = this.configurationService.get('REQUEST_TIMEOUT'),
     symbol
   }: GetAssetProfileParams): Promise<Partial<SymbolProfile>> {
-    const response: Partial<SymbolProfile> = {
+    let response: Partial<SymbolProfile> = {
       symbol,
       dataSource: this.getName()
     };
@@ -201,6 +201,7 @@ export class FinancialModelingPrepService implements DataProviderInterface {
       }
     } catch (error) {
       let message = error;
+      response = undefined;
 
       if (['AbortError', 'TimeoutError'].includes(error?.name)) {
         message = `RequestError: The operation to get the asset profile for ${symbol} was aborted because the request to the data provider took more than ${(

apps/api/src/services/data-provider/ghostfolio/ghostfolio.service.ts

@@ -51,20 +51,26 @@ export class GhostfolioService implements DataProviderInterface {
     requestTimeout = this.configurationService.get('REQUEST_TIMEOUT'),
     symbol
   }: GetAssetProfileParams): Promise<Partial<SymbolProfile>> {
-    let response: DataProviderGhostfolioAssetProfileResponse = {};
+    let assetProfile: DataProviderGhostfolioAssetProfileResponse;
 
     try {
-      const assetProfile = (await fetch(
+      const response = await fetch(
         `${this.URL}/v1/data-providers/ghostfolio/asset-profile/${symbol}`,
         {
           headers: await this.getRequestHeaders(),
           signal: AbortSignal.timeout(requestTimeout)
         }
-      ).then((res) =>
-        res.json()
-      )) as DataProviderGhostfolioAssetProfileResponse;
+      );
+
+      if (!response.ok) {
+        throw new Response(await response.text(), {
+          status: response.status,
+          statusText: response.statusText
+        });
+      }
 
-      response = assetProfile;
+      assetProfile =
+        (await response.json()) as DataProviderGhostfolioAssetProfileResponse;
     } catch (error) {
       let message = error;
 
@@ -72,24 +78,21 @@ export class GhostfolioService implements DataProviderInterface {
         message = `RequestError: The operation to get the asset profile for ${symbol} was aborted because the request to the data provider took more than ${(
           requestTimeout / 1000
         ).toFixed(3)} seconds`;
+      } else if (error?.status === StatusCodes.TOO_MANY_REQUESTS) {
+        message = 'RequestError: The daily request limit has been exceeded';
       } else if (
-        error?.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS
+        [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes(
+          error?.status
+        )
       ) {
-        message = 'RequestError: The daily request limit has been exceeded';
-      } else if (error?.response?.statusCode === StatusCodes.UNAUTHORIZED) {
-        if (!error?.request?.options?.headers?.authorization?.includes('-')) {
-          message =
-            'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.';
-        } else {
-          message =
-            'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.';
-        }
+        message =
+          'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.';
       }
 
       Logger.error(message, 'GhostfolioService');
     }
 
-    return response;
+    return assetProfile;
   }
 
   public getDataProviderInfo(): DataProviderInfo {
@@ -110,12 +113,12 @@ export class GhostfolioService implements DataProviderInterface {
   }: GetDividendsParams): Promise<{
     [date: string]: IDataProviderHistoricalResponse;
   }> {
-    let response: {
+    let dividends: {
       [date: string]: IDataProviderHistoricalResponse;
     } = {};
 
     try {
-      const { dividends } = (await fetch(
+      const response = await fetch(
         `${this.URL}/v2/data-providers/ghostfolio/dividends/${symbol}?from=${format(from, DATE_FORMAT)}&granularity=${granularity}&to=${format(
           to,
           DATE_FORMAT
@@ -124,28 +127,34 @@ export class GhostfolioService implements DataProviderInterface {
           headers: await this.getRequestHeaders(),
           signal: AbortSignal.timeout(requestTimeout)
         }
-      ).then((res) => res.json())) as DividendsResponse;
+      );
+
+      if (!response.ok) {
+        throw new Response(await response.text(), {
+          status: response.status,
+          statusText: response.statusText
+        });
+      }
 
-      response = dividends;
+      dividends = ((await response.json()) as DividendsResponse).dividends;
     } catch (error) {
       let message = error;
 
-      if (error.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS) {
+      if (error?.status === StatusCodes.TOO_MANY_REQUESTS) {
         message = 'RequestError: The daily request limit has been exceeded';
-      } else if (error.response?.statusCode === StatusCodes.UNAUTHORIZED) {
-        if (!error.request?.options?.headers?.authorization?.includes('-')) {
-          message =
-            'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.';
-        } else {
-          message =
-            'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.';
-        }
+      } else if (
+        [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes(
+          error?.status
+        )
+      ) {
+        message =
+          'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.';
       }
 
       Logger.error(message, 'GhostfolioService');
     }
 
-    return response;
+    return dividends;
   }
 
   public async getHistorical({
@@ -158,7 +167,7 @@ export class GhostfolioService implements DataProviderInterface {
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }> {
     try {
-      const { historicalData } = (await fetch(
+      const response = await fetch(
         `${this.URL}/v2/data-providers/ghostfolio/historical/${symbol}?from=${format(from, DATE_FORMAT)}&granularity=${granularity}&to=${format(
           to,
           DATE_FORMAT
@@ -167,27 +176,36 @@ export class GhostfolioService implements DataProviderInterface {
           headers: await this.getRequestHeaders(),
           signal: AbortSignal.timeout(requestTimeout)
         }
-      ).then((res) => res.json())) as HistoricalResponse;
+      );
+
+      if (!response.ok) {
+        throw new Response(await response.text(), {
+          status: response.status,
+          statusText: response.statusText
+        });
+      }
+
+      const { historicalData } = (await response.json()) as HistoricalResponse;
 
       return {
         [symbol]: historicalData
       };
     } catch (error) {
-      let message = error;
-
-      if (error.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS) {
-        message = 'RequestError: The daily request limit has been exceeded';
-      } else if (error.response?.statusCode === StatusCodes.UNAUTHORIZED) {
-        if (!error.request?.options?.headers?.authorization?.includes('-')) {
-          message =
-            'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.';
-        } else {
-          message =
-            'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.';
-        }
+      if (error?.status === StatusCodes.TOO_MANY_REQUESTS) {
+        error.name = 'RequestError';
+        error.message =
+          'RequestError: The daily request limit has been exceeded';
+      } else if (
+        [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes(
+          error?.status
+        )
+      ) {
+        error.name = 'RequestError';
+        error.message =
+          'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.';
       }
 
-      Logger.error(message, 'GhostfolioService');
+      Logger.error(error.message, 'GhostfolioService');
 
       throw new Error(
         `Could not get historical market data for ${symbol} (${this.getName()}) from ${format(
@@ -212,22 +230,29 @@ export class GhostfolioService implements DataProviderInterface {
   }: GetQuotesParams): Promise<{
     [symbol: string]: IDataProviderResponse;
   }> {
-    let response: { [symbol: string]: IDataProviderResponse } = {};
+    let quotes: { [symbol: string]: IDataProviderResponse } = {};
 
     if (symbols.length <= 0) {
-      return response;
+      return quotes;
     }
 
     try {
-      const { quotes } = (await fetch(
+      const response = await fetch(
         `${this.URL}/v2/data-providers/ghostfolio/quotes?symbols=${symbols.join(',')}`,
         {
           headers: await this.getRequestHeaders(),
           signal: AbortSignal.timeout(requestTimeout)
         }
-      ).then((res) => res.json())) as QuotesResponse;
+      );
+
+      if (!response.ok) {
+        throw new Response(await response.text(), {
+          status: response.status,
+          statusText: response.statusText
+        });
+      }
 
-      response = quotes;
+      quotes = ((await response.json()) as QuotesResponse).quotes;
     } catch (error) {
       let message = error;
 
@@ -237,24 +262,21 @@ export class GhostfolioService implements DataProviderInterface {
         )} was aborted because the request to the data provider took more than ${(
           requestTimeout / 1000
         ).toFixed(3)} seconds`;
+      } else if (error?.status === StatusCodes.TOO_MANY_REQUESTS) {
+        message = 'RequestError: The daily request limit has been exceeded';
       } else if (
-        error?.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS
+        [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes(
+          error?.status
+        )
       ) {
-        message = 'RequestError: The daily request limit has been exceeded';
-      } else if (error?.response?.statusCode === StatusCodes.UNAUTHORIZED) {
-        if (!error?.request?.options?.headers?.authorization?.includes('-')) {
-          message =
-            'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.';
-        } else {
-          message =
-            'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.';
-        }
+        message =
+          'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.';
       }
 
       Logger.error(message, 'GhostfolioService');
     }
 
-    return response;
+    return quotes;
   }
 
   public getTestSymbol() {
@@ -268,13 +290,22 @@ export class GhostfolioService implements DataProviderInterface {
     let searchResult: LookupResponse = { items: [] };
 
     try {
-      searchResult = (await fetch(
+      const response = await fetch(
         `${this.URL}/v2/data-providers/ghostfolio/lookup?query=${query}`,
         {
           headers: await this.getRequestHeaders(),
           signal: AbortSignal.timeout(requestTimeout)
         }
-      ).then((res) => res.json())) as LookupResponse;
+      );
+
+      if (!response.ok) {
+        throw new Response(await response.text(), {
+          status: response.status,
+          statusText: response.statusText
+        });
+      }
+
+      searchResult = (await response.json()) as LookupResponse;
     } catch (error) {
       let message = error;
 
@@ -282,18 +313,15 @@ export class GhostfolioService implements DataProviderInterface {
         message = `RequestError: The operation to search for ${query} was aborted because the request to the data provider took more than ${(
           requestTimeout / 1000
         ).toFixed(3)} seconds`;
+      } else if (error?.status === StatusCodes.TOO_MANY_REQUESTS) {
+        message = 'RequestError: The daily request limit has been exceeded';
       } else if (
-        error?.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS
+        [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes(
+          error?.status
+        )
       ) {
-        message = 'RequestError: The daily request limit has been exceeded';
-      } else if (error.response?.statusCode === StatusCodes.UNAUTHORIZED) {
-        if (!error?.request?.options?.headers?.authorization?.includes('-')) {
-          message =
-            'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.';
-        } else {
-          message =
-            'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.';
-        }
+        message =
+          'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.';
       }
 
       Logger.error(message, 'GhostfolioService');

apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts

@@ -36,13 +36,10 @@ export class GoogleSheetsService implements DataProviderInterface {
     return true;
   }
 
-  public async getAssetProfile({
-    symbol
-  }: GetAssetProfileParams): Promise<Partial<SymbolProfile>> {
-    return {
-      symbol,
-      dataSource: this.getName()
-    };
+  public async getAssetProfile({}: GetAssetProfileParams): Promise<
+    Partial<SymbolProfile>
+  > {
+    return undefined;
   }
 
   public getDataProviderInfo(): DataProviderInfo {

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -45,21 +45,20 @@ export class ManualService implements DataProviderInterface {
   public async getAssetProfile({
     symbol
   }: GetAssetProfileParams): Promise<Partial<SymbolProfile>> {
-    const assetProfile: Partial<SymbolProfile> = {
-      symbol,
-      dataSource: this.getName()
-    };
-
     const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles([
       { symbol, dataSource: this.getName() }
     ]);
 
-    if (symbolProfile) {
-      assetProfile.currency = symbolProfile.currency;
-      assetProfile.name = symbolProfile.name;
+    if (!symbolProfile) {
+      return undefined;
     }
 
-    return assetProfile;
+    return {
+      symbol,
+      currency: symbolProfile.currency,
+      dataSource: this.getName(),
+      name: symbolProfile.name
+    };
   }
 
   public getDataProviderInfo(): DataProviderInfo {

apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts

@@ -35,13 +35,10 @@ export class RapidApiService implements DataProviderInterface {
     return !!this.configurationService.get('API_KEY_RAPID_API');
   }
 
-  public async getAssetProfile({
-    symbol
-  }: GetAssetProfileParams): Promise<Partial<SymbolProfile>> {
-    return {
-      symbol,
-      dataSource: this.getName()
-    };
+  public async getAssetProfile({}: GetAssetProfileParams): Promise<
+    Partial<SymbolProfile>
+  > {
+    return undefined;
   }
 
   public getDataProviderInfo(): DataProviderInfo {