Browse Source

Bugfix/missing asset profiles and historical data for symbols across data sources (#7089)

* Fix missing asset profiles and historical data

* Update changelog
pull/7096/head
Thomas Kaul 7 days ago
committed by GitHub
parent
commit
af9889f0ff
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 4
      CHANGELOG.md
  2. 12
      apps/api/src/app/admin/admin.service.ts
  3. 9
      apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts
  4. 7
      apps/api/src/app/endpoints/watchlist/watchlist.service.ts
  5. 7
      apps/api/src/app/portfolio/portfolio.service.ts
  6. 12
      apps/api/src/app/symbol/symbol.service.ts
  7. 71
      apps/api/src/services/data-provider/data-provider.service.ts
  8. 18
      apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
  9. 21
      apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
  10. 8
      apps/api/src/services/queues/data-gathering/data-gathering.service.ts

4
CHANGELOG.md

@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved the language localization for German (`de`)
### Fixed
- Fixed an issue in the data provider service where asset profiles and historical data could be missing for symbols that exist in multiple data sources by keying the responses by the asset profile identifier
## 3.13.0 - 2026-06-20
### Added

12
apps/api/src/app/admin/admin.service.ts

@ -11,7 +11,10 @@ import {
PROPERTY_IS_READ_ONLY_MODE,
PROPERTY_IS_USER_SIGNUP_ENABLED
} from '@ghostfolio/common/config';
import { getCurrencyFromSymbol } from '@ghostfolio/common/helper';
import {
getAssetProfileIdentifier,
getCurrencyFromSymbol
} from '@ghostfolio/common/helper';
import {
AdminData,
AdminUserResponse,
@ -68,14 +71,17 @@ export class AdminService {
{ dataSource, symbol }
]);
if (!assetProfiles[symbol]?.currency) {
const assetProfile =
assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })];
if (!assetProfile?.currency) {
throw new BadRequestException(
`Asset profile not found for ${symbol} (${dataSource})`
);
}
return this.symbolProfileService.add(
assetProfiles[symbol] as Prisma.SymbolProfileCreateInput
assetProfile as Prisma.SymbolProfileCreateInput
);
} catch (error) {
if (

9
apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts

@ -16,6 +16,7 @@ import {
DERIVED_CURRENCIES
} from '@ghostfolio/common/config';
import { PROPERTY_DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER_MAX_REQUESTS } from '@ghostfolio/common/config';
import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import {
DataProviderGhostfolioAssetProfileResponse,
DataProviderHistoricalResponse,
@ -60,7 +61,13 @@ export class GhostfolioService {
}
])
.then(async (assetProfiles) => {
const assetProfile = assetProfiles[symbol];
const assetProfile =
assetProfiles[
getAssetProfileIdentifier({
symbol,
dataSource: dataProviderService.getName()
})
];
const dataSourceOrigin = DataSource.GHOSTFOLIO;
if (assetProfile) {

7
apps/api/src/app/endpoints/watchlist/watchlist.service.ts

@ -40,14 +40,17 @@ export class WatchlistService {
{ dataSource, symbol }
]);
if (!assetProfiles[symbol]?.currency) {
const assetProfile =
assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })];
if (!assetProfile?.currency) {
throw new BadRequestException(
`Asset profile not found for ${symbol} (${dataSource})`
);
}
await this.symbolProfileService.add(
assetProfiles[symbol] as Prisma.SymbolProfileCreateInput
assetProfile as Prisma.SymbolProfileCreateInput
);
}

7
apps/api/src/app/portfolio/portfolio.service.ts

@ -889,10 +889,13 @@ export class PortfolioService {
marketPrice
);
if (historicalData[symbol]) {
const historicalDataItems =
historicalData[getAssetProfileIdentifier({ dataSource, symbol })];
if (historicalDataItems) {
let j = -1;
for (const [date, { marketPrice }] of Object.entries(
historicalData[symbol]
historicalDataItems
)) {
while (
j + 1 < transactionPoints.length &&

12
apps/api/src/app/symbol/symbol.service.ts

@ -94,12 +94,17 @@ export class SymbolService {
date = new Date(),
symbol
}: DataGatheringItem): Promise<DataProviderHistoricalResponse> {
const assetProfileIdentifier = getAssetProfileIdentifier({
dataSource,
symbol
});
let historicalData: {
[symbol: string]: {
[assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse;
};
} = {
[symbol]: {}
[assetProfileIdentifier]: {}
};
try {
@ -112,7 +117,8 @@ export class SymbolService {
return {
marketPrice:
historicalData?.[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice
historicalData?.[assetProfileIdentifier]?.[format(date, DATE_FORMAT)]
?.marketPrice
};
}

71
apps/api/src/services/data-provider/data-provider.service.ts

@ -86,12 +86,11 @@ export class DataProviderService implements OnModuleInit {
return false;
}
// TODO: Change symbol in response to assetProfileIdentifier
public async getAssetProfiles(items: AssetProfileIdentifier[]): Promise<{
[symbol: string]: Partial<SymbolProfile>;
[assetProfileIdentifier: string]: Partial<SymbolProfile>;
}> {
const response: {
[symbol: string]: Partial<SymbolProfile>;
[assetProfileIdentifier: string]: Partial<SymbolProfile>;
} = {};
const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => {
@ -117,7 +116,12 @@ export class DataProviderService implements OnModuleInit {
promises.push(
promise.then((assetProfile) => {
if (isCurrency(assetProfile?.currency)) {
response[symbol] = assetProfile;
response[
getAssetProfileIdentifier({
symbol,
dataSource: DataSource[dataSource]
})
] = { ...assetProfile, symbol };
}
})
);
@ -283,7 +287,7 @@ export class DataProviderService implements OnModuleInit {
symbol
}
])
)?.[symbol];
)?.[assetProfileIdentifier];
} catch {}
if (!assetProfile?.name) {
@ -333,17 +337,20 @@ export class DataProviderService implements OnModuleInit {
});
}
// TODO: Change symbol in response to assetProfileIdentifier
public async getHistorical(
aItems: AssetProfileIdentifier[],
aGranularity: Granularity = 'month',
from: Date,
to: Date
): Promise<{
[symbol: string]: { [date: string]: DataProviderHistoricalResponse };
[assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse;
};
}> {
let response: {
[symbol: string]: { [date: string]: DataProviderHistoricalResponse };
[assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse;
};
} = {};
if (isEmpty(aItems) || !isValid(from) || !isValid(to)) {
@ -383,13 +390,20 @@ export class DataProviderService implements OnModuleInit {
ORDER BY date;`;
response = marketDataByGranularity.reduce((r, marketData) => {
const { date, marketPrice, symbol } = marketData;
const { dataSource, date, marketPrice, symbol } = marketData;
if (!r[symbol]) {
r[symbol] = {};
const assetProfileIdentifier = getAssetProfileIdentifier({
dataSource,
symbol
});
if (!r[assetProfileIdentifier]) {
r[assetProfileIdentifier] = {};
}
r[symbol][format(new Date(date), DATE_FORMAT)] = { marketPrice };
r[assetProfileIdentifier][format(new Date(date), DATE_FORMAT)] = {
marketPrice
};
return r;
}, {});
@ -400,7 +414,6 @@ export class DataProviderService implements OnModuleInit {
}
}
// TODO: Change symbol in response to assetProfileIdentifier
public async getHistoricalRaw({
assetProfileIdentifiers,
from,
@ -410,7 +423,9 @@ export class DataProviderService implements OnModuleInit {
from: Date;
to: Date;
}): Promise<{
[symbol: string]: { [date: string]: DataProviderHistoricalResponse };
[assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse;
};
}> {
for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
if (
@ -443,11 +458,14 @@ export class DataProviderService implements OnModuleInit {
);
const result: {
[symbol: string]: { [date: string]: DataProviderHistoricalResponse };
[assetProfileIdentifier: string]: {
[date: string]: DataProviderHistoricalResponse;
};
} = {};
const promises: Promise<{
data: { [date: string]: DataProviderHistoricalResponse };
dataSource: DataSource;
symbol: string;
}>[] = [];
for (const { dataSource, symbol } of assetProfileIdentifiers) {
@ -465,6 +483,7 @@ export class DataProviderService implements OnModuleInit {
promises.push(
Promise.resolve({
data,
dataSource,
symbol
})
);
@ -478,7 +497,7 @@ export class DataProviderService implements OnModuleInit {
requestTimeout: ms('30 seconds')
})
.then((data) => {
return { symbol, data: data?.[symbol] };
return { dataSource, symbol, data: data?.[symbol] };
})
);
}
@ -488,22 +507,26 @@ export class DataProviderService implements OnModuleInit {
try {
const allData = await Promise.all(promises);
for (const { data, symbol } of allData) {
for (const { data, dataSource, symbol } of allData) {
const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => {
return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol;
});
if (currency) {
// Add derived currency
result[`${DEFAULT_CURRENCY}${currency.currency}`] =
this.transformHistoricalData({
allData,
currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`,
factor: currency.factor
});
result[
getAssetProfileIdentifier({
dataSource,
symbol: `${DEFAULT_CURRENCY}${currency.currency}`
})
] = this.transformHistoricalData({
allData,
currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`,
factor: currency.factor
});
}
result[symbol] = data;
result[getAssetProfileIdentifier({ dataSource, symbol })] = data;
}
} catch (error) {
this.logger.error(error);

18
apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts

@ -15,6 +15,7 @@ import {
getYesterday,
resetHours
} from '@ghostfolio/common/helper';
import { DataProviderHistoricalResponse } from '@ghostfolio/common/interfaces';
import { Injectable, Logger } from '@nestjs/common';
import {
@ -163,7 +164,7 @@ export class ExchangeRateDataService {
}
public async loadCurrencies() {
const result = await this.dataProviderService.getHistorical(
const historicalData = await this.dataProviderService.getHistorical(
this.currencyPairs,
'day',
getYesterday(),
@ -177,8 +178,21 @@ export class ExchangeRateDataService {
requestTimeout: ms('30 seconds')
});
const result: {
[symbol: string]: { [date: string]: DataProviderHistoricalResponse };
} = {};
for (const { dataSource, symbol } of this.currencyPairs) {
const quote = quotes[getAssetProfileIdentifier({ dataSource, symbol })];
const assetProfileIdentifier = getAssetProfileIdentifier({
dataSource,
symbol
});
if (historicalData[assetProfileIdentifier]) {
result[symbol] = historicalData[assetProfileIdentifier];
}
const quote = quotes[assetProfileIdentifier];
if (isNumber(quote?.marketPrice)) {
result[symbol] = {

21
apps/api/src/services/queues/data-gathering/data-gathering.processor.ts

@ -10,7 +10,11 @@ import {
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
} from '@ghostfolio/common/config';
import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
import {
DATE_FORMAT,
getAssetProfileIdentifier,
getStartOfUtcDate
} from '@ghostfolio/common/helper';
import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
import { Process, Processor } from '@nestjs/bull';
@ -114,6 +118,11 @@ export class DataGatheringProcessor {
to: new Date()
});
const assetProfileIdentifier = getAssetProfileIdentifier({
dataSource,
symbol
});
const data: Prisma.MarketDataUpdateInput[] = [];
let lastMarketPrice: number;
@ -131,12 +140,14 @@ export class DataGatheringProcessor {
)
) {
if (
historicalData[symbol]?.[format(currentDate, DATE_FORMAT)]
?.marketPrice
historicalData[assetProfileIdentifier]?.[
format(currentDate, DATE_FORMAT)
]?.marketPrice
) {
lastMarketPrice =
historicalData[symbol]?.[format(currentDate, DATE_FORMAT)]
?.marketPrice;
historicalData[assetProfileIdentifier]?.[
format(currentDate, DATE_FORMAT)
]?.marketPrice;
}
if (lastMarketPrice) {

8
apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@ -131,7 +131,9 @@ export class DataGatheringService {
});
const marketPrice =
historicalData[symbol][format(date, DATE_FORMAT)].marketPrice;
historicalData[getAssetProfileIdentifier({ dataSource, symbol })][
format(date, DATE_FORMAT)
].marketPrice;
if (marketPrice) {
return await this.prismaService.marketData.upsert({
@ -176,7 +178,9 @@ export class DataGatheringService {
assetProfileIdentifiers
);
for (const [symbol, assetProfile] of Object.entries(assetProfiles)) {
for (const assetProfile of Object.values(assetProfiles)) {
const { symbol } = assetProfile;
const symbolProfile = symbolProfiles.find(
({ symbol: symbolProfileSymbol }) => {
return symbolProfileSymbol === symbol;

Loading…
Cancel
Save