From af9889f0ff0a06b73a48a9ce3aa2d515108c235b Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 21 Jun 2026 17:38:48 +0200 Subject: [PATCH] Bugfix/missing asset profiles and historical data for symbols across data sources (#7089) * Fix missing asset profiles and historical data * Update changelog --- CHANGELOG.md | 4 ++ apps/api/src/app/admin/admin.service.ts | 12 +++- .../ghostfolio/ghostfolio.service.ts | 9 ++- .../endpoints/watchlist/watchlist.service.ts | 7 +- .../src/app/portfolio/portfolio.service.ts | 7 +- apps/api/src/app/symbol/symbol.service.ts | 12 +++- .../data-provider/data-provider.service.ts | 71 ++++++++++++------- .../exchange-rate-data.service.ts | 18 ++++- .../data-gathering.processor.ts | 21 ++++-- .../data-gathering/data-gathering.service.ts | 8 ++- 10 files changed, 125 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41bf233c0..e76ae0198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Improved the language localization for German (`de`) +### Fixed + +- Fixed an issue in the data provider service where asset profiles and historical data could be missing for symbols that exist in multiple data sources by keying the responses by the asset profile identifier + ## 3.13.0 - 2026-06-20 ### Added diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts index 6dddbbca7..e50c0c77f 100644 --- a/apps/api/src/app/admin/admin.service.ts +++ b/apps/api/src/app/admin/admin.service.ts @@ -11,7 +11,10 @@ import { PROPERTY_IS_READ_ONLY_MODE, PROPERTY_IS_USER_SIGNUP_ENABLED } from '@ghostfolio/common/config'; -import { getCurrencyFromSymbol } from '@ghostfolio/common/helper'; +import { + getAssetProfileIdentifier, + getCurrencyFromSymbol +} from '@ghostfolio/common/helper'; import { AdminData, AdminUserResponse, @@ -68,14 +71,17 @@ export class AdminService { { dataSource, symbol } ]); - if (!assetProfiles[symbol]?.currency) { + const assetProfile = + assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (!assetProfile?.currency) { throw new BadRequestException( `Asset profile not found for ${symbol} (${dataSource})` ); } return this.symbolProfileService.add( - assetProfiles[symbol] as Prisma.SymbolProfileCreateInput + assetProfile as Prisma.SymbolProfileCreateInput ); } catch (error) { if ( diff --git a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts index b84ca881f..caff86ae0 100644 --- a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts +++ b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts @@ -16,6 +16,7 @@ import { DERIVED_CURRENCIES } from '@ghostfolio/common/config'; import { PROPERTY_DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER_MAX_REQUESTS } from '@ghostfolio/common/config'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { DataProviderGhostfolioAssetProfileResponse, DataProviderHistoricalResponse, @@ -60,7 +61,13 @@ export class GhostfolioService { } ]) .then(async (assetProfiles) => { - const assetProfile = assetProfiles[symbol]; + const assetProfile = + assetProfiles[ + getAssetProfileIdentifier({ + symbol, + dataSource: dataProviderService.getName() + }) + ]; const dataSourceOrigin = DataSource.GHOSTFOLIO; if (assetProfile) { diff --git a/apps/api/src/app/endpoints/watchlist/watchlist.service.ts b/apps/api/src/app/endpoints/watchlist/watchlist.service.ts index 78786c00b..88702da00 100644 --- a/apps/api/src/app/endpoints/watchlist/watchlist.service.ts +++ b/apps/api/src/app/endpoints/watchlist/watchlist.service.ts @@ -40,14 +40,17 @@ export class WatchlistService { { dataSource, symbol } ]); - if (!assetProfiles[symbol]?.currency) { + const assetProfile = + assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (!assetProfile?.currency) { throw new BadRequestException( `Asset profile not found for ${symbol} (${dataSource})` ); } await this.symbolProfileService.add( - assetProfiles[symbol] as Prisma.SymbolProfileCreateInput + assetProfile as Prisma.SymbolProfileCreateInput ); } diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index d6a54b180..418e60401 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -889,10 +889,13 @@ export class PortfolioService { marketPrice ); - if (historicalData[symbol]) { + const historicalDataItems = + historicalData[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (historicalDataItems) { let j = -1; for (const [date, { marketPrice }] of Object.entries( - historicalData[symbol] + historicalDataItems )) { while ( j + 1 < transactionPoints.length && diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index 732fcb23d..39d279de1 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -94,12 +94,17 @@ export class SymbolService { date = new Date(), symbol }: DataGatheringItem): Promise { + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + let historicalData: { - [symbol: string]: { + [assetProfileIdentifier: string]: { [date: string]: DataProviderHistoricalResponse; }; } = { - [symbol]: {} + [assetProfileIdentifier]: {} }; try { @@ -112,7 +117,8 @@ export class SymbolService { return { marketPrice: - historicalData?.[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice + historicalData?.[assetProfileIdentifier]?.[format(date, DATE_FORMAT)] + ?.marketPrice }; } diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 6780f58f0..5d49848fa 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -86,12 +86,11 @@ export class DataProviderService implements OnModuleInit { return false; } - // TODO: Change symbol in response to assetProfileIdentifier public async getAssetProfiles(items: AssetProfileIdentifier[]): Promise<{ - [symbol: string]: Partial; + [assetProfileIdentifier: string]: Partial; }> { const response: { - [symbol: string]: Partial; + [assetProfileIdentifier: string]: Partial; } = {}; const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => { @@ -117,7 +116,12 @@ export class DataProviderService implements OnModuleInit { promises.push( promise.then((assetProfile) => { if (isCurrency(assetProfile?.currency)) { - response[symbol] = assetProfile; + response[ + getAssetProfileIdentifier({ + symbol, + dataSource: DataSource[dataSource] + }) + ] = { ...assetProfile, symbol }; } }) ); @@ -283,7 +287,7 @@ export class DataProviderService implements OnModuleInit { symbol } ]) - )?.[symbol]; + )?.[assetProfileIdentifier]; } catch {} if (!assetProfile?.name) { @@ -333,17 +337,20 @@ export class DataProviderService implements OnModuleInit { }); } - // TODO: Change symbol in response to assetProfileIdentifier public async getHistorical( aItems: AssetProfileIdentifier[], aGranularity: Granularity = 'month', from: Date, to: Date ): Promise<{ - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; }> { let response: { - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; } = {}; if (isEmpty(aItems) || !isValid(from) || !isValid(to)) { @@ -383,13 +390,20 @@ export class DataProviderService implements OnModuleInit { ORDER BY date;`; response = marketDataByGranularity.reduce((r, marketData) => { - const { date, marketPrice, symbol } = marketData; + const { dataSource, date, marketPrice, symbol } = marketData; - if (!r[symbol]) { - r[symbol] = {}; + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + + if (!r[assetProfileIdentifier]) { + r[assetProfileIdentifier] = {}; } - r[symbol][format(new Date(date), DATE_FORMAT)] = { marketPrice }; + r[assetProfileIdentifier][format(new Date(date), DATE_FORMAT)] = { + marketPrice + }; return r; }, {}); @@ -400,7 +414,6 @@ export class DataProviderService implements OnModuleInit { } } - // TODO: Change symbol in response to assetProfileIdentifier public async getHistoricalRaw({ assetProfileIdentifiers, from, @@ -410,7 +423,9 @@ export class DataProviderService implements OnModuleInit { from: Date; to: Date; }): Promise<{ - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; }> { for (const { currency, rootCurrency } of DERIVED_CURRENCIES) { if ( @@ -443,11 +458,14 @@ export class DataProviderService implements OnModuleInit { ); const result: { - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; } = {}; const promises: Promise<{ data: { [date: string]: DataProviderHistoricalResponse }; + dataSource: DataSource; symbol: string; }>[] = []; for (const { dataSource, symbol } of assetProfileIdentifiers) { @@ -465,6 +483,7 @@ export class DataProviderService implements OnModuleInit { promises.push( Promise.resolve({ data, + dataSource, symbol }) ); @@ -478,7 +497,7 @@ export class DataProviderService implements OnModuleInit { requestTimeout: ms('30 seconds') }) .then((data) => { - return { symbol, data: data?.[symbol] }; + return { dataSource, symbol, data: data?.[symbol] }; }) ); } @@ -488,22 +507,26 @@ export class DataProviderService implements OnModuleInit { try { const allData = await Promise.all(promises); - for (const { data, symbol } of allData) { + for (const { data, dataSource, symbol } of allData) { const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => { return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol; }); if (currency) { // Add derived currency - result[`${DEFAULT_CURRENCY}${currency.currency}`] = - this.transformHistoricalData({ - allData, - currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`, - factor: currency.factor - }); + result[ + getAssetProfileIdentifier({ + dataSource, + symbol: `${DEFAULT_CURRENCY}${currency.currency}` + }) + ] = this.transformHistoricalData({ + allData, + currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`, + factor: currency.factor + }); } - result[symbol] = data; + result[getAssetProfileIdentifier({ dataSource, symbol })] = data; } } catch (error) { this.logger.error(error); diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts index d8a08c1c4..3b48ce292 100644 --- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts @@ -15,6 +15,7 @@ import { getYesterday, resetHours } from '@ghostfolio/common/helper'; +import { DataProviderHistoricalResponse } from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { @@ -163,7 +164,7 @@ export class ExchangeRateDataService { } public async loadCurrencies() { - const result = await this.dataProviderService.getHistorical( + const historicalData = await this.dataProviderService.getHistorical( this.currencyPairs, 'day', getYesterday(), @@ -177,8 +178,21 @@ export class ExchangeRateDataService { requestTimeout: ms('30 seconds') }); + const result: { + [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + } = {}; + for (const { dataSource, symbol } of this.currencyPairs) { - const quote = quotes[getAssetProfileIdentifier({ dataSource, symbol })]; + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + + if (historicalData[assetProfileIdentifier]) { + result[symbol] = historicalData[assetProfileIdentifier]; + } + + const quote = quotes[assetProfileIdentifier]; if (isNumber(quote?.marketPrice)) { result[symbol] = { diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts index ee5cb838a..8b7e3489f 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts @@ -10,7 +10,11 @@ import { GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME } from '@ghostfolio/common/config'; -import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; +import { + DATE_FORMAT, + getAssetProfileIdentifier, + getStartOfUtcDate +} from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { Process, Processor } from '@nestjs/bull'; @@ -114,6 +118,11 @@ export class DataGatheringProcessor { to: new Date() }); + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + const data: Prisma.MarketDataUpdateInput[] = []; let lastMarketPrice: number; @@ -131,12 +140,14 @@ export class DataGatheringProcessor { ) ) { if ( - historicalData[symbol]?.[format(currentDate, DATE_FORMAT)] - ?.marketPrice + historicalData[assetProfileIdentifier]?.[ + format(currentDate, DATE_FORMAT) + ]?.marketPrice ) { lastMarketPrice = - historicalData[symbol]?.[format(currentDate, DATE_FORMAT)] - ?.marketPrice; + historicalData[assetProfileIdentifier]?.[ + format(currentDate, DATE_FORMAT) + ]?.marketPrice; } if (lastMarketPrice) { diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index aa6c952ef..ff65ebc83 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -131,7 +131,9 @@ export class DataGatheringService { }); const marketPrice = - historicalData[symbol][format(date, DATE_FORMAT)].marketPrice; + historicalData[getAssetProfileIdentifier({ dataSource, symbol })][ + format(date, DATE_FORMAT) + ].marketPrice; if (marketPrice) { return await this.prismaService.marketData.upsert({ @@ -176,7 +178,9 @@ export class DataGatheringService { assetProfileIdentifiers ); - for (const [symbol, assetProfile] of Object.entries(assetProfiles)) { + for (const assetProfile of Object.values(assetProfiles)) { + const { symbol } = assetProfile; + const symbolProfile = symbolProfiles.find( ({ symbol: symbolProfileSymbol }) => { return symbolProfileSymbol === symbol;