From af903321a22fb585a5d961f79a34a709e8381c42 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Wed, 27 Aug 2025 18:35:19 +0200 Subject: [PATCH] Feature/improve error handling in data providers (part 2) (#5413) * Improve error handling * Update changelog --- CHANGELOG.md | 1 + .../yahoo-finance/yahoo-finance.service.ts | 5 +- .../data-provider/data-provider.service.ts | 4 +- .../eod-historical-data.service.ts | 14 +- .../financial-modeling-prep.service.ts | 3 +- .../ghostfolio/ghostfolio.service.ts | 176 ++++++++++-------- .../google-sheets/google-sheets.service.ts | 11 +- .../data-provider/manual/manual.service.ts | 17 +- .../rapid-api/rapid-api.service.ts | 11 +- 9 files changed, 137 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb3bd6d3c..6a225c51f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Moved the support to customize rules in the _X-ray_ section from experimental to general availability - Improved the create or update activity dialog’s asset sub class selector for valuables to update the options dynamically based on the selected asset class +- Improved the error handling in data providers - Randomized the minutes of the hourly data gathering cron job - Refactored the dialog footer component to standalone - Refactored the dialog header component to standalone diff --git a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts index 54167bc5b..ecbc38256 100644 --- a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts @@ -163,7 +163,7 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface { public async getAssetProfile( aSymbol: string ): Promise> { - const response: Partial = {}; + let response: Partial = {}; try { let symbol = aSymbol; @@ -241,10 +241,13 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface { } const url = assetProfile.summaryProfile?.website; + if (url) { response.url = url; } } catch (error) { + response = undefined; + if (error.message === `Quote not found for symbol: ${aSymbol}`) { throw new AssetProfileDelistedError( `No data found, ${aSymbol} (${this.getName()}) may be delisted` diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 631f90d63..8754c3537 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -107,7 +107,9 @@ export class DataProviderService implements OnModuleInit { promises.push( promise.then((symbolProfile) => { - response[symbol] = symbolProfile; + if (symbolProfile) { + response[symbol] = symbolProfile; + } }) ); } diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts index d06071ac3..58b4d0d3e 100644 --- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts +++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts @@ -55,14 +55,18 @@ export class EodHistoricalDataService implements DataProviderInterface { }: GetAssetProfileParams): Promise> { const [searchResult] = await this.getSearchResult(symbol); + if (!searchResult) { + return undefined; + } + return { symbol, - assetClass: searchResult?.assetClass, - assetSubClass: searchResult?.assetSubClass, - currency: this.convertCurrency(searchResult?.currency), + assetClass: searchResult.assetClass, + assetSubClass: searchResult.assetSubClass, + currency: this.convertCurrency(searchResult.currency), dataSource: this.getName(), - isin: searchResult?.isin, - name: searchResult?.name + isin: searchResult.isin, + name: searchResult.name }; } diff --git a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts index ed2aa5f25..d1e591d5d 100644 --- a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts +++ b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts @@ -64,7 +64,7 @@ export class FinancialModelingPrepService implements DataProviderInterface { requestTimeout = this.configurationService.get('REQUEST_TIMEOUT'), symbol }: GetAssetProfileParams): Promise> { - const response: Partial = { + let response: Partial = { symbol, dataSource: this.getName() }; @@ -201,6 +201,7 @@ export class FinancialModelingPrepService implements DataProviderInterface { } } catch (error) { let message = error; + response = undefined; if (['AbortError', 'TimeoutError'].includes(error?.name)) { message = `RequestError: The operation to get the asset profile for ${symbol} was aborted because the request to the data provider took more than ${( diff --git a/apps/api/src/services/data-provider/ghostfolio/ghostfolio.service.ts b/apps/api/src/services/data-provider/ghostfolio/ghostfolio.service.ts index 48ba42bd4..c4a6996c1 100644 --- a/apps/api/src/services/data-provider/ghostfolio/ghostfolio.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio/ghostfolio.service.ts @@ -51,20 +51,26 @@ export class GhostfolioService implements DataProviderInterface { requestTimeout = this.configurationService.get('REQUEST_TIMEOUT'), symbol }: GetAssetProfileParams): Promise> { - let response: DataProviderGhostfolioAssetProfileResponse = {}; + let assetProfile: DataProviderGhostfolioAssetProfileResponse; try { - const assetProfile = (await fetch( + const response = await fetch( `${this.URL}/v1/data-providers/ghostfolio/asset-profile/${symbol}`, { headers: await this.getRequestHeaders(), signal: AbortSignal.timeout(requestTimeout) } - ).then((res) => - res.json() - )) as DataProviderGhostfolioAssetProfileResponse; + ); + + if (!response.ok) { + throw new Response(await response.text(), { + status: response.status, + statusText: response.statusText + }); + } - response = assetProfile; + assetProfile = + (await response.json()) as DataProviderGhostfolioAssetProfileResponse; } catch (error) { let message = error; @@ -72,24 +78,21 @@ export class GhostfolioService implements DataProviderInterface { message = `RequestError: The operation to get the asset profile for ${symbol} was aborted because the request to the data provider took more than ${( requestTimeout / 1000 ).toFixed(3)} seconds`; + } else if (error?.status === StatusCodes.TOO_MANY_REQUESTS) { + message = 'RequestError: The daily request limit has been exceeded'; } else if ( - error?.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS + [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes( + error?.status + ) ) { - message = 'RequestError: The daily request limit has been exceeded'; - } else if (error?.response?.statusCode === StatusCodes.UNAUTHORIZED) { - if (!error?.request?.options?.headers?.authorization?.includes('-')) { - message = - 'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.'; - } else { - message = - 'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.'; - } + message = + 'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.'; } Logger.error(message, 'GhostfolioService'); } - return response; + return assetProfile; } public getDataProviderInfo(): DataProviderInfo { @@ -110,12 +113,12 @@ export class GhostfolioService implements DataProviderInterface { }: GetDividendsParams): Promise<{ [date: string]: IDataProviderHistoricalResponse; }> { - let response: { + let dividends: { [date: string]: IDataProviderHistoricalResponse; } = {}; try { - const { dividends } = (await fetch( + const response = await fetch( `${this.URL}/v2/data-providers/ghostfolio/dividends/${symbol}?from=${format(from, DATE_FORMAT)}&granularity=${granularity}&to=${format( to, DATE_FORMAT @@ -124,28 +127,34 @@ export class GhostfolioService implements DataProviderInterface { headers: await this.getRequestHeaders(), signal: AbortSignal.timeout(requestTimeout) } - ).then((res) => res.json())) as DividendsResponse; + ); + + if (!response.ok) { + throw new Response(await response.text(), { + status: response.status, + statusText: response.statusText + }); + } - response = dividends; + dividends = ((await response.json()) as DividendsResponse).dividends; } catch (error) { let message = error; - if (error.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS) { + if (error?.status === StatusCodes.TOO_MANY_REQUESTS) { message = 'RequestError: The daily request limit has been exceeded'; - } else if (error.response?.statusCode === StatusCodes.UNAUTHORIZED) { - if (!error.request?.options?.headers?.authorization?.includes('-')) { - message = - 'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.'; - } else { - message = - 'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.'; - } + } else if ( + [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes( + error?.status + ) + ) { + message = + 'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.'; } Logger.error(message, 'GhostfolioService'); } - return response; + return dividends; } public async getHistorical({ @@ -158,7 +167,7 @@ export class GhostfolioService implements DataProviderInterface { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { try { - const { historicalData } = (await fetch( + const response = await fetch( `${this.URL}/v2/data-providers/ghostfolio/historical/${symbol}?from=${format(from, DATE_FORMAT)}&granularity=${granularity}&to=${format( to, DATE_FORMAT @@ -167,27 +176,36 @@ export class GhostfolioService implements DataProviderInterface { headers: await this.getRequestHeaders(), signal: AbortSignal.timeout(requestTimeout) } - ).then((res) => res.json())) as HistoricalResponse; + ); + + if (!response.ok) { + throw new Response(await response.text(), { + status: response.status, + statusText: response.statusText + }); + } + + const { historicalData } = (await response.json()) as HistoricalResponse; return { [symbol]: historicalData }; } catch (error) { - let message = error; - - if (error.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS) { - message = 'RequestError: The daily request limit has been exceeded'; - } else if (error.response?.statusCode === StatusCodes.UNAUTHORIZED) { - if (!error.request?.options?.headers?.authorization?.includes('-')) { - message = - 'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.'; - } else { - message = - 'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.'; - } + if (error?.status === StatusCodes.TOO_MANY_REQUESTS) { + error.name = 'RequestError'; + error.message = + 'RequestError: The daily request limit has been exceeded'; + } else if ( + [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes( + error?.status + ) + ) { + error.name = 'RequestError'; + error.message = + 'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.'; } - Logger.error(message, 'GhostfolioService'); + Logger.error(error.message, 'GhostfolioService'); throw new Error( `Could not get historical market data for ${symbol} (${this.getName()}) from ${format( @@ -212,22 +230,29 @@ export class GhostfolioService implements DataProviderInterface { }: GetQuotesParams): Promise<{ [symbol: string]: IDataProviderResponse; }> { - let response: { [symbol: string]: IDataProviderResponse } = {}; + let quotes: { [symbol: string]: IDataProviderResponse } = {}; if (symbols.length <= 0) { - return response; + return quotes; } try { - const { quotes } = (await fetch( + const response = await fetch( `${this.URL}/v2/data-providers/ghostfolio/quotes?symbols=${symbols.join(',')}`, { headers: await this.getRequestHeaders(), signal: AbortSignal.timeout(requestTimeout) } - ).then((res) => res.json())) as QuotesResponse; + ); + + if (!response.ok) { + throw new Response(await response.text(), { + status: response.status, + statusText: response.statusText + }); + } - response = quotes; + quotes = ((await response.json()) as QuotesResponse).quotes; } catch (error) { let message = error; @@ -237,24 +262,21 @@ export class GhostfolioService implements DataProviderInterface { )} was aborted because the request to the data provider took more than ${( requestTimeout / 1000 ).toFixed(3)} seconds`; + } else if (error?.status === StatusCodes.TOO_MANY_REQUESTS) { + message = 'RequestError: The daily request limit has been exceeded'; } else if ( - error?.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS + [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes( + error?.status + ) ) { - message = 'RequestError: The daily request limit has been exceeded'; - } else if (error?.response?.statusCode === StatusCodes.UNAUTHORIZED) { - if (!error?.request?.options?.headers?.authorization?.includes('-')) { - message = - 'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.'; - } else { - message = - 'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.'; - } + message = + 'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.'; } Logger.error(message, 'GhostfolioService'); } - return response; + return quotes; } public getTestSymbol() { @@ -268,13 +290,22 @@ export class GhostfolioService implements DataProviderInterface { let searchResult: LookupResponse = { items: [] }; try { - searchResult = (await fetch( + const response = await fetch( `${this.URL}/v2/data-providers/ghostfolio/lookup?query=${query}`, { headers: await this.getRequestHeaders(), signal: AbortSignal.timeout(requestTimeout) } - ).then((res) => res.json())) as LookupResponse; + ); + + if (!response.ok) { + throw new Response(await response.text(), { + status: response.status, + statusText: response.statusText + }); + } + + searchResult = (await response.json()) as LookupResponse; } catch (error) { let message = error; @@ -282,18 +313,15 @@ export class GhostfolioService implements DataProviderInterface { message = `RequestError: The operation to search for ${query} was aborted because the request to the data provider took more than ${( requestTimeout / 1000 ).toFixed(3)} seconds`; + } else if (error?.status === StatusCodes.TOO_MANY_REQUESTS) { + message = 'RequestError: The daily request limit has been exceeded'; } else if ( - error?.response?.statusCode === StatusCodes.TOO_MANY_REQUESTS + [StatusCodes.FORBIDDEN, StatusCodes.UNAUTHORIZED].includes( + error?.status + ) ) { - message = 'RequestError: The daily request limit has been exceeded'; - } else if (error.response?.statusCode === StatusCodes.UNAUTHORIZED) { - if (!error?.request?.options?.headers?.authorization?.includes('-')) { - message = - 'RequestError: The provided API key is invalid. Please update it in the Settings section of the Admin Control panel.'; - } else { - message = - 'RequestError: The provided API key has expired. Please request a new one and update it in the Settings section of the Admin Control panel.'; - } + message = + 'RequestError: The API key is invalid. Please update it in the Settings section of the Admin Control panel.'; } Logger.error(message, 'GhostfolioService'); diff --git a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts index f067f042c..111f2d004 100644 --- a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts +++ b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts @@ -36,13 +36,10 @@ export class GoogleSheetsService implements DataProviderInterface { return true; } - public async getAssetProfile({ - symbol - }: GetAssetProfileParams): Promise> { - return { - symbol, - dataSource: this.getName() - }; + public async getAssetProfile({}: GetAssetProfileParams): Promise< + Partial + > { + return undefined; } public getDataProviderInfo(): DataProviderInfo { diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 1c1c4c2da..c411f678b 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -45,21 +45,20 @@ export class ManualService implements DataProviderInterface { public async getAssetProfile({ symbol }: GetAssetProfileParams): Promise> { - const assetProfile: Partial = { - symbol, - dataSource: this.getName() - }; - const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles([ { symbol, dataSource: this.getName() } ]); - if (symbolProfile) { - assetProfile.currency = symbolProfile.currency; - assetProfile.name = symbolProfile.name; + if (!symbolProfile) { + return undefined; } - return assetProfile; + return { + symbol, + currency: symbolProfile.currency, + dataSource: this.getName(), + name: symbolProfile.name + }; } public getDataProviderInfo(): DataProviderInfo { diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts index 62b3ed71c..824f44328 100644 --- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts @@ -35,13 +35,10 @@ export class RapidApiService implements DataProviderInterface { return !!this.configurationService.get('API_KEY_RAPID_API'); } - public async getAssetProfile({ - symbol - }: GetAssetProfileParams): Promise> { - return { - symbol, - dataSource: this.getName() - }; + public async getAssetProfile({}: GetAssetProfileParams): Promise< + Partial + > { + return undefined; } public getDataProviderInfo(): DataProviderInfo {