From 42734dff754f28a84c2e8a96cee977760cc8e498 Mon Sep 17 00:00:00 2001 From: csehatt741 <77381875+csehatt741@users.noreply.github.com> Date: Wed, 16 Apr 2025 07:44:42 +0200 Subject: [PATCH] Feature/deactivate asset profile on delisting (Yahoo Finance) (#4524) * Deactivate asset profile on delisting (Yahoo Finance) * Update changelog --------- Co-authored-by: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> --- CHANGELOG.md | 6 ++ apps/api/src/app/admin/admin.controller.ts | 16 ++--- apps/api/src/app/order/order.service.ts | 8 +-- apps/api/src/services/cron.service.ts | 8 +-- .../yahoo-finance/yahoo-finance.service.ts | 9 ++- .../data-provider/data-provider.service.ts | 8 ++- .../errors/asset-profile-delisted.error.ts | 7 ++ .../yahoo-finance/yahoo-finance.service.ts | 19 ++++-- .../data-gathering.processor.ts | 66 +++++++++++++++---- .../asset-profile-dialog.html | 8 ++- libs/common/src/lib/config.ts | 4 +- 11 files changed, 120 insertions(+), 39 deletions(-) create mode 100644 apps/api/src/services/data-provider/errors/asset-profile-delisted.error.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 642c64392..693a17089 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Deactivated asset profiles automatically on delisting in the _Yahoo Finance_ service + ## 2.151.0 - 2025-04-11 ### Added diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 2df1d98ae..e9952ea08 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -9,8 +9,8 @@ import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathe import { DATA_GATHERING_QUEUE_PRIORITY_HIGH, DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, - GATHER_ASSET_PROFILE_PROCESS, - GATHER_ASSET_PROFILE_PROCESS_OPTIONS + GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, + GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS } from '@ghostfolio/common/config'; import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { @@ -92,9 +92,9 @@ export class AdminController { dataSource, symbol }, - name: GATHER_ASSET_PROFILE_PROCESS, + name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, opts: { - ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + ...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS, jobId: getAssetProfileIdentifier({ dataSource, symbol }), priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM } @@ -119,9 +119,9 @@ export class AdminController { dataSource, symbol }, - name: GATHER_ASSET_PROFILE_PROCESS, + name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, opts: { - ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + ...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS, jobId: getAssetProfileIdentifier({ dataSource, symbol }), priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM } @@ -142,9 +142,9 @@ export class AdminController { dataSource, symbol }, - name: GATHER_ASSET_PROFILE_PROCESS, + name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, opts: { - ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + ...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS, jobId: getAssetProfileIdentifier({ dataSource, symbol }), priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH } diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index a26099e9d..aa5ac4630 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -7,8 +7,8 @@ import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathe import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DATA_GATHERING_QUEUE_PRIORITY_HIGH, - GATHER_ASSET_PROFILE_PROCESS, - GATHER_ASSET_PROFILE_PROCESS_OPTIONS + GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, + GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS } from '@ghostfolio/common/config'; import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { @@ -144,9 +144,9 @@ export class OrderService { dataSource: data.SymbolProfile.connectOrCreate.create.dataSource, symbol: data.SymbolProfile.connectOrCreate.create.symbol }, - name: GATHER_ASSET_PROFILE_PROCESS, + name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, opts: { - ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + ...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS, jobId: getAssetProfileIdentifier({ dataSource: data.SymbolProfile.connectOrCreate.create.dataSource, symbol: data.SymbolProfile.connectOrCreate.create.symbol diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index 088348d85..3d4aff08b 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -1,8 +1,8 @@ import { UserService } from '@ghostfolio/api/app/user/user.service'; import { DATA_GATHERING_QUEUE_PRIORITY_LOW, - GATHER_ASSET_PROFILE_PROCESS, - GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, + GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS, PROPERTY_IS_DATA_GATHERING_ENABLED } from '@ghostfolio/common/config'; import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; @@ -66,9 +66,9 @@ export class CronService { dataSource, symbol }, - name: GATHER_ASSET_PROFILE_PROCESS, + name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, opts: { - ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + ...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS, jobId: getAssetProfileIdentifier({ dataSource, symbol }), priority: DATA_GATHERING_QUEUE_PRIORITY_LOW } diff --git a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts index d876b42ca..64bbeebb5 100644 --- a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts @@ -1,4 +1,5 @@ import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service'; +import { AssetProfileDelistedError } from '@ghostfolio/api/services/data-provider/errors/asset-profile-delisted.error'; import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface'; import { DEFAULT_CURRENCY, @@ -236,7 +237,13 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface { response.url = url; } } catch (error) { - Logger.error(error, 'YahooFinanceService'); + if (error.message === `Quote not found for symbol: ${aSymbol}`) { + throw new AssetProfileDelistedError( + `No data found, ${aSymbol} (${this.getName()}) may be delisted` + ); + } else { + Logger.error(error, 'YahooFinanceService'); + } } return response; diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 88ae136ae..3f02cf109 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -114,7 +114,13 @@ export class DataProviderService { } } - await Promise.all(promises); + try { + await Promise.all(promises); + } catch (error) { + Logger.error(error, 'DataProviderService'); + + throw error; + } return response; } diff --git a/apps/api/src/services/data-provider/errors/asset-profile-delisted.error.ts b/apps/api/src/services/data-provider/errors/asset-profile-delisted.error.ts new file mode 100644 index 000000000..dabe0aa5b --- /dev/null +++ b/apps/api/src/services/data-provider/errors/asset-profile-delisted.error.ts @@ -0,0 +1,7 @@ +export class AssetProfileDelistedError extends Error { + public constructor(message: string) { + super(message); + + this.name = 'AssetProfileDelistedError'; + } +} diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index 72ae1ff97..6b42c9283 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -1,5 +1,6 @@ import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service'; import { YahooFinanceDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service'; +import { AssetProfileDelistedError } from '@ghostfolio/api/services/data-provider/errors/asset-profile-delisted.error'; import { DataProviderInterface, GetAssetProfileParams, @@ -143,12 +144,18 @@ export class YahooFinanceService implements DataProviderInterface { return response; } catch (error) { - throw new Error( - `Could not get historical market data for ${symbol} (${this.getName()}) from ${format( - from, - DATE_FORMAT - )} to ${format(to, DATE_FORMAT)}: [${error.name}] ${error.message}` - ); + if (error.message === 'No data found, symbol may be delisted') { + throw new AssetProfileDelistedError( + `No data found, ${symbol} (${this.getName()}) may be delisted` + ); + } else { + throw new Error( + `Could not get historical market data for ${symbol} (${this.getName()}) from ${format( + from, + DATE_FORMAT + )} to ${format(to, DATE_FORMAT)}: [${error.name}] ${error.message}` + ); + } } } diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts index eedad7475..9cf6f63e6 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts @@ -1,11 +1,13 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; +import { AssetProfileDelistedError } from '@ghostfolio/api/services/data-provider/errors/asset-profile-delisted.error'; import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; +import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DATA_GATHERING_QUEUE, DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY, DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY, - GATHER_ASSET_PROFILE_PROCESS, + GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME } from '@ghostfolio/common/config'; import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; @@ -33,7 +35,8 @@ export class DataGatheringProcessor { public constructor( private readonly dataGatheringService: DataGatheringService, private readonly dataProviderService: DataProviderService, - private readonly marketDataService: MarketDataService + private readonly marketDataService: MarketDataService, + private readonly symbolProfileService: SymbolProfileService ) {} @Process({ @@ -42,28 +45,49 @@ export class DataGatheringProcessor { DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY.toString(), 10 ), - name: GATHER_ASSET_PROFILE_PROCESS + name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME }) public async gatherAssetProfile(job: Job) { + const { dataSource, symbol } = job.data; + try { Logger.log( - `Asset profile data gathering has been started for ${job.data.symbol} (${job.data.dataSource})`, - `DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS})` + `Asset profile data gathering has been started for ${symbol} (${dataSource})`, + `DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS_JOB_NAME})` ); await this.dataGatheringService.gatherAssetProfiles([job.data]); Logger.log( - `Asset profile data gathering has been completed for ${job.data.symbol} (${job.data.dataSource})`, - `DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS})` + `Asset profile data gathering has been completed for ${symbol} (${dataSource})`, + `DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS_JOB_NAME})` ); } catch (error) { + if (error instanceof AssetProfileDelistedError) { + await this.symbolProfileService.updateSymbolProfile( + { + dataSource, + symbol + }, + { + isActive: false + } + ); + + Logger.log( + `Asset profile data gathering has been discarded for ${symbol} (${dataSource})`, + `DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS_JOB_NAME})` + ); + + return job.discard(); + } + Logger.error( error, - `DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS})` + `DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS_JOB_NAME})` ); - throw new Error(error); + throw error; } } @@ -76,8 +100,9 @@ export class DataGatheringProcessor { name: GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME }) public async gatherHistoricalMarketData(job: Job) { + const { dataSource, date, symbol } = job.data; + try { - const { dataSource, date, symbol } = job.data; let currentDate = parseISO(date as unknown as string); Logger.log( @@ -142,12 +167,31 @@ export class DataGatheringProcessor { `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` ); } catch (error) { + if (error instanceof AssetProfileDelistedError) { + await this.symbolProfileService.updateSymbolProfile( + { + dataSource, + symbol + }, + { + isActive: false + } + ); + + Logger.log( + `Historical market data gathering has been discarded for ${symbol} (${dataSource})`, + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` + ); + + return job.discard(); + } + Logger.error( error, `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` ); - throw new Error(error); + throw error; } } } diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html index 042c63594..32f946ad8 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html @@ -19,7 +19,9 @@