Browse Source

Bugfix/missing asset profiles and historical data for symbols across data sources (#7089)

* Fix missing asset profiles and historical data

* Update changelog
pull/7096/head
Thomas Kaul 1 week ago
committed by GitHub
parent
commit
af9889f0ff
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 4
      CHANGELOG.md
  2. 12
      apps/api/src/app/admin/admin.service.ts
  3. 9
      apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts
  4. 7
      apps/api/src/app/endpoints/watchlist/watchlist.service.ts
  5. 7
      apps/api/src/app/portfolio/portfolio.service.ts
  6. 12
      apps/api/src/app/symbol/symbol.service.ts
  7. 71
      apps/api/src/services/data-provider/data-provider.service.ts
  8. 18
      apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
  9. 21
      apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
  10. 8
      apps/api/src/services/queues/data-gathering/data-gathering.service.ts

4
CHANGELOG.md

@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved the language localization for German (`de`) - Improved the language localization for German (`de`)
### Fixed
- Fixed an issue in the data provider service where asset profiles and historical data could be missing for symbols that exist in multiple data sources by keying the responses by the asset profile identifier
## 3.13.0 - 2026-06-20 ## 3.13.0 - 2026-06-20
### Added ### Added

12
apps/api/src/app/admin/admin.service.ts

@ -11,7 +11,10 @@ import {
PROPERTY_IS_READ_ONLY_MODE, PROPERTY_IS_READ_ONLY_MODE,
PROPERTY_IS_USER_SIGNUP_ENABLED PROPERTY_IS_USER_SIGNUP_ENABLED
} from '@ghostfolio/common/config'; } from '@ghostfolio/common/config';
import { getCurrencyFromSymbol } from '@ghostfolio/common/helper'; import {
getAssetProfileIdentifier,
getCurrencyFromSymbol
} from '@ghostfolio/common/helper';
import { import {
AdminData, AdminData,
AdminUserResponse, AdminUserResponse,
@ -68,14 +71,17 @@ export class AdminService {
{ dataSource, symbol } { dataSource, symbol }
]); ]);
if (!assetProfiles[symbol]?.currency) { const assetProfile =
assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })];
if (!assetProfile?.currency) {
throw new BadRequestException( throw new BadRequestException(
`Asset profile not found for ${symbol} (${dataSource})` `Asset profile not found for ${symbol} (${dataSource})`
); );
} }
return this.symbolProfileService.add( return this.symbolProfileService.add(
assetProfiles[symbol] as Prisma.SymbolProfileCreateInput assetProfile as Prisma.SymbolProfileCreateInput
); );
} catch (error) { } catch (error) {
if ( if (

9
apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts

@ -16,6 +16,7 @@ import {
DERIVED_CURRENCIES DERIVED_CURRENCIES
} from '@ghostfolio/common/config'; } from '@ghostfolio/common/config';
import { PROPERTY_DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER_MAX_REQUESTS } from '@ghostfolio/common/config'; import { PROPERTY_DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER_MAX_REQUESTS } from '@ghostfolio/common/config';
import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import { import {
DataProviderGhostfolioAssetProfileResponse, DataProviderGhostfolioAssetProfileResponse,
DataProviderHistoricalResponse, DataProviderHistoricalResponse,
@ -60,7 +61,13 @@ export class GhostfolioService {
} }
]) ])
.then(async (assetProfiles) => { .then(async (assetProfiles) => {
const assetProfile = assetProfiles[symbol]; const assetProfile =
assetProfiles[
getAssetProfileIdentifier({
symbol,
dataSource: dataProviderService.getName()
})
];
const dataSourceOrigin = DataSource.GHOSTFOLIO; const dataSourceOrigin = DataSource.GHOSTFOLIO;
if (assetProfile) { if (assetProfile) {

7
apps/api/src/app/endpoints/watchlist/watchlist.service.ts

@ -40,14 +40,17 @@ export class WatchlistService {
{ dataSource, symbol } { dataSource, symbol }
]); ]);
if (!assetProfiles[symbol]?.currency) { const assetProfile =
assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })];
if (!assetProfile?.currency) {
throw new BadRequestException( throw new BadRequestException(
`Asset profile not found for ${symbol} (${dataSource})` `Asset profile not found for ${symbol} (${dataSource})`
); );
} }
await this.symbolProfileService.add( await this.symbolProfileService.add(
assetProfiles[symbol] as Prisma.SymbolProfileCreateInput assetProfile as Prisma.SymbolProfileCreateInput
); );
} }

7
apps/api/src/app/portfolio/portfolio.service.ts

@ -889,10 +889,13 @@ export class PortfolioService {
marketPrice marketPrice
); );
if (historicalData[symbol]) { const historicalDataItems =
historicalData[getAssetProfileIdentifier({ dataSource, symbol })];
if (historicalDataItems) {
let j = -1; let j = -1;
for (const [date, { marketPrice }] of Object.entries( for (const [date, { marketPrice }] of Object.entries(
historicalData[symbol] historicalDataItems
)) { )) {
while ( while (
j + 1 < transactionPoints.length && j + 1 < transactionPoints.length &&

12
apps/api/src/app/symbol/symbol.service.ts

@ -94,12 +94,17 @@ export class SymbolService {
date = new Date(), date = new Date(),
symbol symbol
}: DataGatheringItem): Promise<DataProviderHistoricalResponse> { }: DataGatheringItem): Promise<DataProviderHistoricalResponse> {
const assetProfileIdentifier = getAssetProfileIdentifier({
dataSource,
symbol
});
let historicalData: { let historicalData: {
[symbol: string]: { [assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse; [date: string]: DataProviderHistoricalResponse;
}; };
} = { } = {
[symbol]: {} [assetProfileIdentifier]: {}
}; };
try { try {
@ -112,7 +117,8 @@ export class SymbolService {
return { return {
marketPrice: marketPrice:
historicalData?.[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice historicalData?.[assetProfileIdentifier]?.[format(date, DATE_FORMAT)]
?.marketPrice
}; };
} }

71
apps/api/src/services/data-provider/data-provider.service.ts

@ -86,12 +86,11 @@ export class DataProviderService implements OnModuleInit {
return false; return false;
} }
// TODO: Change symbol in response to assetProfileIdentifier
public async getAssetProfiles(items: AssetProfileIdentifier[]): Promise<{ public async getAssetProfiles(items: AssetProfileIdentifier[]): Promise<{
[symbol: string]: Partial<SymbolProfile>; [assetProfileIdentifier: string]: Partial<SymbolProfile>;
}> { }> {
const response: { const response: {
[symbol: string]: Partial<SymbolProfile>; [assetProfileIdentifier: string]: Partial<SymbolProfile>;
} = {}; } = {};
const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => { const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => {
@ -117,7 +116,12 @@ export class DataProviderService implements OnModuleInit {
promises.push( promises.push(
promise.then((assetProfile) => { promise.then((assetProfile) => {
if (isCurrency(assetProfile?.currency)) { if (isCurrency(assetProfile?.currency)) {
response[symbol] = assetProfile; response[
getAssetProfileIdentifier({
symbol,
dataSource: DataSource[dataSource]
})
] = { ...assetProfile, symbol };
} }
}) })
); );
@ -283,7 +287,7 @@ export class DataProviderService implements OnModuleInit {
symbol symbol
} }
]) ])
)?.[symbol]; )?.[assetProfileIdentifier];
} catch {} } catch {}
if (!assetProfile?.name) { if (!assetProfile?.name) {
@ -333,17 +337,20 @@ export class DataProviderService implements OnModuleInit {
}); });
} }
// TODO: Change symbol in response to assetProfileIdentifier
public async getHistorical( public async getHistorical(
aItems: AssetProfileIdentifier[], aItems: AssetProfileIdentifier[],
aGranularity: Granularity = 'month', aGranularity: Granularity = 'month',
from: Date, from: Date,
to: Date to: Date
): Promise<{ ): Promise<{
[symbol: string]: { [date: string]: DataProviderHistoricalResponse }; [assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse;
};
}> { }> {
let response: { let response: {
[symbol: string]: { [date: string]: DataProviderHistoricalResponse }; [assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse;
};
} = {}; } = {};
if (isEmpty(aItems) || !isValid(from) || !isValid(to)) { if (isEmpty(aItems) || !isValid(from) || !isValid(to)) {
@ -383,13 +390,20 @@ export class DataProviderService implements OnModuleInit {
ORDER BY date;`; ORDER BY date;`;
response = marketDataByGranularity.reduce((r, marketData) => { response = marketDataByGranularity.reduce((r, marketData) => {
const { date, marketPrice, symbol } = marketData; const { dataSource, date, marketPrice, symbol } = marketData;
if (!r[symbol]) { const assetProfileIdentifier = getAssetProfileIdentifier({
r[symbol] = {}; dataSource,
symbol
});
if (!r[assetProfileIdentifier]) {
r[assetProfileIdentifier] = {};
} }
r[symbol][format(new Date(date), DATE_FORMAT)] = { marketPrice }; r[assetProfileIdentifier][format(new Date(date), DATE_FORMAT)] = {
marketPrice
};
return r; return r;
}, {}); }, {});
@ -400,7 +414,6 @@ export class DataProviderService implements OnModuleInit {
} }
} }
// TODO: Change symbol in response to assetProfileIdentifier
public async getHistoricalRaw({ public async getHistoricalRaw({
assetProfileIdentifiers, assetProfileIdentifiers,
from, from,
@ -410,7 +423,9 @@ export class DataProviderService implements OnModuleInit {
from: Date; from: Date;
to: Date; to: Date;
}): Promise<{ }): Promise<{
[symbol: string]: { [date: string]: DataProviderHistoricalResponse }; [assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse;
};
}> { }> {
for (const { currency, rootCurrency } of DERIVED_CURRENCIES) { for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
if ( if (
@ -443,11 +458,14 @@ export class DataProviderService implements OnModuleInit {
); );
const result: { const result: {
[symbol: string]: { [date: string]: DataProviderHistoricalResponse }; [assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse;
};
} = {}; } = {};
const promises: Promise<{ const promises: Promise<{
data: { [date: string]: DataProviderHistoricalResponse }; data: { [date: string]: DataProviderHistoricalResponse };
dataSource: DataSource;
symbol: string; symbol: string;
}>[] = []; }>[] = [];
for (const { dataSource, symbol } of assetProfileIdentifiers) { for (const { dataSource, symbol } of assetProfileIdentifiers) {
@ -465,6 +483,7 @@ export class DataProviderService implements OnModuleInit {
promises.push( promises.push(
Promise.resolve({ Promise.resolve({
data, data,
dataSource,
symbol symbol
}) })
); );
@ -478,7 +497,7 @@ export class DataProviderService implements OnModuleInit {
requestTimeout: ms('30 seconds') requestTimeout: ms('30 seconds')
}) })
.then((data) => { .then((data) => {
return { symbol, data: data?.[symbol] }; return { dataSource, symbol, data: data?.[symbol] };
}) })
); );
} }
@ -488,22 +507,26 @@ export class DataProviderService implements OnModuleInit {
try { try {
const allData = await Promise.all(promises); const allData = await Promise.all(promises);
for (const { data, symbol } of allData) { for (const { data, dataSource, symbol } of allData) {
const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => { const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => {
return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol; return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol;
}); });
if (currency) { if (currency) {
// Add derived currency // Add derived currency
result[`${DEFAULT_CURRENCY}${currency.currency}`] = result[
this.transformHistoricalData({ getAssetProfileIdentifier({
allData, dataSource,
currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`, symbol: `${DEFAULT_CURRENCY}${currency.currency}`
factor: currency.factor })
}); ] = this.transformHistoricalData({
allData,
currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`,
factor: currency.factor
});
} }
result[symbol] = data; result[getAssetProfileIdentifier({ dataSource, symbol })] = data;
} }
} catch (error) { } catch (error) {
this.logger.error(error); this.logger.error(error);

18
apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts

@ -15,6 +15,7 @@ import {
getYesterday, getYesterday,
resetHours resetHours
} from '@ghostfolio/common/helper'; } from '@ghostfolio/common/helper';
import { DataProviderHistoricalResponse } from '@ghostfolio/common/interfaces';
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { import {
@ -163,7 +164,7 @@ export class ExchangeRateDataService {
} }
public async loadCurrencies() { public async loadCurrencies() {
const result = await this.dataProviderService.getHistorical( const historicalData = await this.dataProviderService.getHistorical(
this.currencyPairs, this.currencyPairs,
'day', 'day',
getYesterday(), getYesterday(),
@ -177,8 +178,21 @@ export class ExchangeRateDataService {
requestTimeout: ms('30 seconds') requestTimeout: ms('30 seconds')
}); });
const result: {
[symbol: string]: { [date: string]: DataProviderHistoricalResponse };
} = {};
for (const { dataSource, symbol } of this.currencyPairs) { for (const { dataSource, symbol } of this.currencyPairs) {
const quote = quotes[getAssetProfileIdentifier({ dataSource, symbol })]; const assetProfileIdentifier = getAssetProfileIdentifier({
dataSource,
symbol
});
if (historicalData[assetProfileIdentifier]) {
result[symbol] = historicalData[assetProfileIdentifier];
}
const quote = quotes[assetProfileIdentifier];
if (isNumber(quote?.marketPrice)) { if (isNumber(quote?.marketPrice)) {
result[symbol] = { result[symbol] = {

21
apps/api/src/services/queues/data-gathering/data-gathering.processor.ts

@ -10,7 +10,11 @@ import {
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
} from '@ghostfolio/common/config'; } from '@ghostfolio/common/config';
import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; import {
DATE_FORMAT,
getAssetProfileIdentifier,
getStartOfUtcDate
} from '@ghostfolio/common/helper';
import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
import { Process, Processor } from '@nestjs/bull'; import { Process, Processor } from '@nestjs/bull';
@ -114,6 +118,11 @@ export class DataGatheringProcessor {
to: new Date() to: new Date()
}); });
const assetProfileIdentifier = getAssetProfileIdentifier({
dataSource,
symbol
});
const data: Prisma.MarketDataUpdateInput[] = []; const data: Prisma.MarketDataUpdateInput[] = [];
let lastMarketPrice: number; let lastMarketPrice: number;
@ -131,12 +140,14 @@ export class DataGatheringProcessor {
) )
) { ) {
if ( if (
historicalData[symbol]?.[format(currentDate, DATE_FORMAT)] historicalData[assetProfileIdentifier]?.[
?.marketPrice format(currentDate, DATE_FORMAT)
]?.marketPrice
) { ) {
lastMarketPrice = lastMarketPrice =
historicalData[symbol]?.[format(currentDate, DATE_FORMAT)] historicalData[assetProfileIdentifier]?.[
?.marketPrice; format(currentDate, DATE_FORMAT)
]?.marketPrice;
} }
if (lastMarketPrice) { if (lastMarketPrice) {

8
apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@ -131,7 +131,9 @@ export class DataGatheringService {
}); });
const marketPrice = const marketPrice =
historicalData[symbol][format(date, DATE_FORMAT)].marketPrice; historicalData[getAssetProfileIdentifier({ dataSource, symbol })][
format(date, DATE_FORMAT)
].marketPrice;
if (marketPrice) { if (marketPrice) {
return await this.prismaService.marketData.upsert({ return await this.prismaService.marketData.upsert({
@ -176,7 +178,9 @@ export class DataGatheringService {
assetProfileIdentifiers assetProfileIdentifiers
); );
for (const [symbol, assetProfile] of Object.entries(assetProfiles)) { for (const assetProfile of Object.values(assetProfiles)) {
const { symbol } = assetProfile;
const symbolProfile = symbolProfiles.find( const symbolProfile = symbolProfiles.find(
({ symbol: symbolProfileSymbol }) => { ({ symbol: symbolProfileSymbol }) => {
return symbolProfileSymbol === symbol; return symbolProfileSymbol === symbol;

Loading…
Cancel
Save