From 0b747bab4b1f9c914b45177c96c21b0a8457e234 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 20 Jun 2026 20:35:20 +0200 Subject: [PATCH] Fix missing asset profiles and historical data --- apps/api/src/app/admin/admin.service.ts | 12 +++- .../ghostfolio/ghostfolio.service.ts | 9 ++- .../endpoints/watchlist/watchlist.service.ts | 7 +- .../src/app/portfolio/portfolio.service.ts | 7 +- apps/api/src/app/symbol/symbol.service.ts | 12 +++- .../data-provider/data-provider.service.ts | 69 ++++++++++++------- .../exchange-rate-data.service.ts | 18 ++++- .../data-gathering.processor.ts | 21 ++++-- .../data-gathering/data-gathering.service.ts | 8 ++- 9 files changed, 120 insertions(+), 43 deletions(-) diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts index 8c0611c41..0fb7f713e 100644 --- a/apps/api/src/app/admin/admin.service.ts +++ b/apps/api/src/app/admin/admin.service.ts @@ -11,7 +11,10 @@ import { PROPERTY_IS_READ_ONLY_MODE, PROPERTY_IS_USER_SIGNUP_ENABLED } from '@ghostfolio/common/config'; -import { getCurrencyFromSymbol } from '@ghostfolio/common/helper'; +import { + getAssetProfileIdentifier, + getCurrencyFromSymbol +} from '@ghostfolio/common/helper'; import { AdminData, AdminUserResponse, @@ -68,14 +71,17 @@ export class AdminService { { dataSource, symbol } ]); - if (!assetProfiles[symbol]?.currency) { + const assetProfile = + assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (!assetProfile?.currency) { throw new BadRequestException( `Asset profile not found for ${symbol} (${dataSource})` ); } return this.symbolProfileService.add( - assetProfiles[symbol] as Prisma.SymbolProfileCreateInput + assetProfile as Prisma.SymbolProfileCreateInput ); } catch (error) { if ( diff --git a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts index b84ca881f..caff86ae0 100644 --- a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts +++ b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts @@ -16,6 +16,7 @@ import { DERIVED_CURRENCIES } from '@ghostfolio/common/config'; import { PROPERTY_DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER_MAX_REQUESTS } from '@ghostfolio/common/config'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { DataProviderGhostfolioAssetProfileResponse, DataProviderHistoricalResponse, @@ -60,7 +61,13 @@ export class GhostfolioService { } ]) .then(async (assetProfiles) => { - const assetProfile = assetProfiles[symbol]; + const assetProfile = + assetProfiles[ + getAssetProfileIdentifier({ + symbol, + dataSource: dataProviderService.getName() + }) + ]; const dataSourceOrigin = DataSource.GHOSTFOLIO; if (assetProfile) { diff --git a/apps/api/src/app/endpoints/watchlist/watchlist.service.ts b/apps/api/src/app/endpoints/watchlist/watchlist.service.ts index 791cc6c69..1d98b3beb 100644 --- a/apps/api/src/app/endpoints/watchlist/watchlist.service.ts +++ b/apps/api/src/app/endpoints/watchlist/watchlist.service.ts @@ -41,14 +41,17 @@ export class WatchlistService { { dataSource, symbol } ]); - if (!assetProfiles[symbol]?.currency) { + const assetProfile = + assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (!assetProfile?.currency) { throw new BadRequestException( `Asset profile not found for ${symbol} (${dataSource})` ); } await this.symbolProfileService.add( - assetProfiles[symbol] as Prisma.SymbolProfileCreateInput + assetProfile as Prisma.SymbolProfileCreateInput ); } diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index c15353521..db79dc25c 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -890,10 +890,13 @@ export class PortfolioService { marketPrice ); - if (historicalData[symbol]) { + const historicalDataItems = + historicalData[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (historicalDataItems) { let j = -1; for (const [date, { marketPrice }] of Object.entries( - historicalData[symbol] + historicalDataItems )) { while ( j + 1 < transactionPoints.length && diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index 732fcb23d..39d279de1 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -94,12 +94,17 @@ export class SymbolService { date = new Date(), symbol }: DataGatheringItem): Promise { + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + let historicalData: { - [symbol: string]: { + [assetProfileIdentifier: string]: { [date: string]: DataProviderHistoricalResponse; }; } = { - [symbol]: {} + [assetProfileIdentifier]: {} }; try { @@ -112,7 +117,8 @@ export class SymbolService { return { marketPrice: - historicalData?.[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice + historicalData?.[assetProfileIdentifier]?.[format(date, DATE_FORMAT)] + ?.marketPrice }; } diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index fb9efff44..bc528c2ca 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -86,12 +86,11 @@ export class DataProviderService implements OnModuleInit { return false; } - // TODO: Change symbol in response to assetProfileIdentifier public async getAssetProfiles(items: AssetProfileIdentifier[]): Promise<{ - [symbol: string]: Partial; + [assetProfileIdentifier: string]: Partial; }> { const response: { - [symbol: string]: Partial; + [assetProfileIdentifier: string]: Partial; } = {}; const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => { @@ -117,7 +116,12 @@ export class DataProviderService implements OnModuleInit { promises.push( promise.then((assetProfile) => { if (isCurrency(assetProfile?.currency)) { - response[symbol] = assetProfile; + response[ + getAssetProfileIdentifier({ + symbol, + dataSource: DataSource[dataSource] + }) + ] = { ...assetProfile, symbol }; } }) ); @@ -335,17 +339,20 @@ export class DataProviderService implements OnModuleInit { }); } - // TODO: Change symbol in response to assetProfileIdentifier public async getHistorical( aItems: AssetProfileIdentifier[], aGranularity: Granularity = 'month', from: Date, to: Date ): Promise<{ - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; }> { let response: { - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; } = {}; if (isEmpty(aItems) || !isValid(from) || !isValid(to)) { @@ -385,13 +392,20 @@ export class DataProviderService implements OnModuleInit { ORDER BY date;`; response = marketDataByGranularity.reduce((r, marketData) => { - const { date, marketPrice, symbol } = marketData; + const { dataSource, date, marketPrice, symbol } = marketData; - if (!r[symbol]) { - r[symbol] = {}; + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + + if (!r[assetProfileIdentifier]) { + r[assetProfileIdentifier] = {}; } - r[symbol][format(new Date(date), DATE_FORMAT)] = { marketPrice }; + r[assetProfileIdentifier][format(new Date(date), DATE_FORMAT)] = { + marketPrice + }; return r; }, {}); @@ -402,7 +416,6 @@ export class DataProviderService implements OnModuleInit { } } - // TODO: Change symbol in response to assetProfileIdentifier public async getHistoricalRaw({ assetProfileIdentifiers, from, @@ -412,7 +425,9 @@ export class DataProviderService implements OnModuleInit { from: Date; to: Date; }): Promise<{ - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; }> { for (const { currency, rootCurrency } of DERIVED_CURRENCIES) { if ( @@ -445,11 +460,14 @@ export class DataProviderService implements OnModuleInit { ); const result: { - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; } = {}; const promises: Promise<{ data: { [date: string]: DataProviderHistoricalResponse }; + dataSource: DataSource; symbol: string; }>[] = []; for (const { dataSource, symbol } of assetProfileIdentifiers) { @@ -467,6 +485,7 @@ export class DataProviderService implements OnModuleInit { promises.push( Promise.resolve({ data, + dataSource, symbol }) ); @@ -480,7 +499,7 @@ export class DataProviderService implements OnModuleInit { requestTimeout: ms('30 seconds') }) .then((data) => { - return { symbol, data: data?.[symbol] }; + return { dataSource, symbol, data: data?.[symbol] }; }) ); } @@ -490,22 +509,26 @@ export class DataProviderService implements OnModuleInit { try { const allData = await Promise.all(promises); - for (const { data, symbol } of allData) { + for (const { data, dataSource, symbol } of allData) { const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => { return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol; }); if (currency) { // Add derived currency - result[`${DEFAULT_CURRENCY}${currency.currency}`] = - this.transformHistoricalData({ - allData, - currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`, - factor: currency.factor - }); + result[ + getAssetProfileIdentifier({ + dataSource, + symbol: `${DEFAULT_CURRENCY}${currency.currency}` + }) + ] = this.transformHistoricalData({ + allData, + currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`, + factor: currency.factor + }); } - result[symbol] = data; + result[getAssetProfileIdentifier({ dataSource, symbol })] = data; } } catch (error) { this.logger.error(error); diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts index d8a08c1c4..3b48ce292 100644 --- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts @@ -15,6 +15,7 @@ import { getYesterday, resetHours } from '@ghostfolio/common/helper'; +import { DataProviderHistoricalResponse } from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { @@ -163,7 +164,7 @@ export class ExchangeRateDataService { } public async loadCurrencies() { - const result = await this.dataProviderService.getHistorical( + const historicalData = await this.dataProviderService.getHistorical( this.currencyPairs, 'day', getYesterday(), @@ -177,8 +178,21 @@ export class ExchangeRateDataService { requestTimeout: ms('30 seconds') }); + const result: { + [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + } = {}; + for (const { dataSource, symbol } of this.currencyPairs) { - const quote = quotes[getAssetProfileIdentifier({ dataSource, symbol })]; + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + + if (historicalData[assetProfileIdentifier]) { + result[symbol] = historicalData[assetProfileIdentifier]; + } + + const quote = quotes[assetProfileIdentifier]; if (isNumber(quote?.marketPrice)) { result[symbol] = { diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts index ee5cb838a..8b7e3489f 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts @@ -10,7 +10,11 @@ import { GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME } from '@ghostfolio/common/config'; -import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; +import { + DATE_FORMAT, + getAssetProfileIdentifier, + getStartOfUtcDate +} from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { Process, Processor } from '@nestjs/bull'; @@ -114,6 +118,11 @@ export class DataGatheringProcessor { to: new Date() }); + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + const data: Prisma.MarketDataUpdateInput[] = []; let lastMarketPrice: number; @@ -131,12 +140,14 @@ export class DataGatheringProcessor { ) ) { if ( - historicalData[symbol]?.[format(currentDate, DATE_FORMAT)] - ?.marketPrice + historicalData[assetProfileIdentifier]?.[ + format(currentDate, DATE_FORMAT) + ]?.marketPrice ) { lastMarketPrice = - historicalData[symbol]?.[format(currentDate, DATE_FORMAT)] - ?.marketPrice; + historicalData[assetProfileIdentifier]?.[ + format(currentDate, DATE_FORMAT) + ]?.marketPrice; } if (lastMarketPrice) { diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index 789dea49b..d9d7cf3f8 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -135,7 +135,9 @@ export class DataGatheringService { }); const marketPrice = - historicalData[symbol][format(date, DATE_FORMAT)].marketPrice; + historicalData[getAssetProfileIdentifier({ dataSource, symbol })][ + format(date, DATE_FORMAT) + ].marketPrice; if (marketPrice) { return await this.prismaService.marketData.upsert({ @@ -180,7 +182,9 @@ export class DataGatheringService { assetProfileIdentifiers ); - for (const [symbol, assetProfile] of Object.entries(assetProfiles)) { + for (const assetProfile of Object.values(assetProfiles)) { + const { symbol } = assetProfile; + const symbolProfile = symbolProfiles.find( ({ symbol: symbolProfileSymbol }) => { return symbolProfileSymbol === symbol;