topaLE
/
ghostfolio
mirror of https://github.com/ghostfolio/ghostfolio
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fix: handle null exchange rates and optimize backfill loop to prevent event loop stalls 

- Replace throw/catch pattern with direct if/else checks to avoid V8 stack trace overhead
- Add setImmediate yield every 500 iterations to unblock Bull heartbeat
- Downgrade missing rate logs from error to debug level
- Add dates array support for targeted queries instead of full date ranges
pull/6912/head
Andrea Bugeja 5 days ago
parent
ddb05e3f4e
commit
8643fb3995
2 changed files with 69 additions and 60 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 41
      apps/api/src/app/portfolio/current-rate.service.ts
  2. 88
      apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts

41
apps/api/src/app/portfolio/current-rate.service.ts
View File

@ -24,8 +24,6 @@ import { GetValuesParams } from './interfaces/get-values-params.interface';
@Injectable()
export class CurrentRateService {
private static readonly MARKET_DATA_PAGE_SIZE = 50000;
public constructor(
private readonly activitiesService: ActivitiesService,
private readonly dataProviderService: DataProviderService,
@ -84,32 +82,27 @@ export class CurrentRateService {
return { dataSource, symbol };
});
const marketDataCount = await this.marketDataService.getRangeCount({
assetProfileIdentifiers,
dateQuery
});
for (
let i = 0;
i < marketDataCount;
i += CurrentRateService.MARKET_DATA_PAGE_SIZE
) {
// Use page size to limit the number of records fetched at once
const data = await this.marketDataService.getRange({
assetProfileIdentifiers,
dateQuery,
skip: i,
take: CurrentRateService.MARKET_DATA_PAGE_SIZE
});
values.push(
...data.map(({ dataSource, date, marketPrice, symbol }) => ({
// Fetch each asset profile individually to use the composite index efficiently
// Process in batches of 10 to avoid overwhelming the database
const batchSize = 10;
for (let i = 0; i < assetProfileIdentifiers.length; i += batchSize) {
const batch = assetProfileIdentifiers.slice(i, i + batchSize);
const promises = batch.map(async (assetProfile) => {
const data = await this.marketDataService.getRange({
assetProfileIdentifiers: [assetProfile],
dateQuery
});
return data.map(({ dataSource, date, marketPrice, symbol }) => ({
dataSource,
date,
marketPrice,
symbol
}))
);
}));
});
const results = await Promise.all(promises);
for (const result of results) {
values.push(...result);
}
}
const response: GetValuesObject = {

88
apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
View File

@ -53,16 +53,18 @@ export class ExchangeRateDataService {
@LogPerformance
public async getExchangeRatesByCurrency({
currencies,
dates,
endDate = new Date(),
startDate,
targetCurrency
}: {
currencies: string[];
dates?: Date[];
endDate?: Date;
startDate: Date;
startDate?: Date;
targetCurrency: string;
}): Promise<ExchangeRatesByCurrency> {
if (!startDate) {
if (!startDate && !dates?.length) {
return {};
}
@ -73,6 +75,8 @@ export class ExchangeRateDataService {
for (const currency of currencies) {
exchangeRatesByCurrency[`${currency}${targetCurrency}`] =
await this.getExchangeRates({
dates,
endDate,
startDate,
currencyFrom: currency,
currencyTo: targetCurrency
@ -90,11 +94,14 @@ export class ExchangeRateDataService {
lastDateString
] ?? 1;
const loopStartDate =
startDate || dates?.reduce((min, d) => (d < min ? d : min), dates[0]);
// Start from the most recent date and fill in missing exchange rates
// using the latest available rate
for (
let date = endDate;
!isBefore(date, startDate);
loopStartDate && !isBefore(date, loopStartDate);
date = subDays(resetHours(date), 1)
) {
const dateString = format(date, DATE_FORMAT);
@ -110,7 +117,7 @@ export class ExchangeRateDataService {
previousExchangeRate;
if (currency === DEFAULT_CURRENCY && isBefore(date, new Date())) {
Logger.error(
Logger.debug(
`No exchange rate has been found for ${currency}${targetCurrency} at ${dateString}`,
'ExchangeRateDataService'
);
@ -355,19 +362,22 @@ export class ExchangeRateDataService {
private async getExchangeRates({
currencyFrom,
currencyTo,
dates,
endDate = new Date(),
startDate
}: {
currencyFrom: string;
currencyTo: string;
dates?: Date[];
endDate?: Date;
startDate: Date;
startDate?: Date;
}) {
const dates = eachDayOfInterval({ end: endDate, start: startDate });
const datesToProcess =
dates ?? eachDayOfInterval({ end: endDate, start: startDate });
const factors: { [dateString: string]: number } = {};
if (currencyFrom === currencyTo) {
for (const date of dates) {
for (const date of datesToProcess) {
factors[format(date, DATE_FORMAT)] = 1;
}
@ -378,7 +388,7 @@ export class ExchangeRateDataService {
this.derivedCurrencyFactors[`${currencyFrom}${currencyTo}`];
if (derivedCurrencyFactor) {
for (const date of dates) {
for (const date of datesToProcess) {
factors[format(date, DATE_FORMAT)] = derivedCurrencyFactor;
}
@ -395,7 +405,8 @@ export class ExchangeRateDataService {
symbol
}
],
dateQuery: { gte: startDate, lt: endDate }
dateQuery:
dates?.length > 0 ? { in: dates } : { gte: startDate, lt: endDate }
});
if (marketData?.length > 0) {
@ -414,7 +425,7 @@ export class ExchangeRateDataService {
try {
if (currencyFrom === DEFAULT_CURRENCY) {
for (const date of dates) {
for (const date of datesToProcess) {
marketPriceBaseCurrencyFromCurrency[format(date, DATE_FORMAT)] = 1;
}
} else {
@ -425,7 +436,10 @@ export class ExchangeRateDataService {
symbol: `${DEFAULT_CURRENCY}${currencyFrom}`
}
],
dateQuery: { gte: startDate, lt: endDate }
dateQuery:
dates?.length > 0
? { in: dates }
: { gte: startDate, lt: endDate }
});
for (const { date, marketPrice } of marketData) {
@ -437,7 +451,7 @@ export class ExchangeRateDataService {
try {
if (currencyTo === DEFAULT_CURRENCY) {
for (const date of dates) {
for (const date of datesToProcess) {
marketPriceBaseCurrencyToCurrency[format(date, DATE_FORMAT)] = 1;
}
} else {
@ -448,10 +462,10 @@ export class ExchangeRateDataService {
symbol: `${DEFAULT_CURRENCY}${currencyTo}`
}
],
dateQuery: {
gte: startDate,
lt: endDate
}
dateQuery:
dates?.length > 0
? { in: dates }
: { gte: startDate, lt: endDate }
});
for (const { date, marketPrice } of marketData) {
@ -461,30 +475,32 @@ export class ExchangeRateDataService {
}
} catch {}
for (const date of dates) {
try {
const factor =
(1 /
marketPriceBaseCurrencyFromCurrency[format(date, DATE_FORMAT)]) *
marketPriceBaseCurrencyToCurrency[format(date, DATE_FORMAT)];
for (let i = 0; i < datesToProcess.length; i++) {
if (i % 500 === 0 && process.env.NODE_ENV !== 'test') {
await new Promise((resolve) => setImmediate(resolve));
}
if (isNaN(factor)) {
throw new Error('Exchange rate is not a number');
} else {
factors[format(date, DATE_FORMAT)] = factor;
}
} catch {
let errorMessage = `No exchange rate has been found for ${currencyFrom}${currencyTo} at ${format(
date,
DATE_FORMAT
)}. Please complement market data for ${DEFAULT_CURRENCY}${currencyFrom}`;
if (DEFAULT_CURRENCY !== currencyTo) {
errorMessage = `${errorMessage} and ${DEFAULT_CURRENCY}${currencyTo}`;
const date = datesToProcess[i];
const dateString = format(date, DATE_FORMAT);
const priceFrom = marketPriceBaseCurrencyFromCurrency[dateString];
const priceTo = marketPriceBaseCurrencyToCurrency[dateString];
if (priceFrom && priceTo) {
const factor = (1 / priceFrom) * priceTo;
if (!isNaN(factor) && isFinite(factor)) {
factors[dateString] = factor;
continue;
}
}
Logger.error(`${errorMessage}.`, 'ExchangeRateDataService');
let errorMessage = `No exchange rate has been found for ${currencyFrom}${currencyTo} at ${dateString}. Please complement market data for ${DEFAULT_CURRENCY}${currencyFrom}`;
if (DEFAULT_CURRENCY !== currencyTo) {
errorMessage = `${errorMessage} and ${DEFAULT_CURRENCY}${currencyTo}`;
}
Logger.debug(`${errorMessage}.`, 'ExchangeRateDataService');
}
}

Write Preview
Loading…
Cancel
Save