From 2b5d2701bd00acb4fff83e1ebf2f5b19b42dfeb9 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:06:01 +0200 Subject: [PATCH] Task/improve exchange rate and market data gathering robustness (#7119) * Improve exchange rate robustness * Improve market data gathering robustness * Update changelog --- CHANGELOG.md | 2 + .../src/app/activities/activities.service.ts | 8 +- apps/api/src/app/import/import.service.ts | 15 ++-- .../src/app/portfolio/portfolio.service.ts | 8 +- .../market-data/market-data.service.ts | 74 ++++++++++--------- libs/common/src/lib/config.ts | 1 + 6 files changed, 58 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f175878fc..c8d6a7244 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue where symbols with special characters caused API request failures by URL encoding the symbol - Fixed the disabled state of the delete action in the asset profiles actions menu of the historical market data table in the admin control panel - Fixed the persistence of an empty `locale` string in the scraper configuration +- Fixed a transaction timeout that prevented gathering historical market data for symbols with a long history +- Fixed an exception in various portfolio endpoints when historical exchange rate data is missing ## 3.14.0 - 2026-06-22 diff --git a/apps/api/src/app/activities/activities.service.ts b/apps/api/src/app/activities/activities.service.ts index f57507e3d..44ab67e45 100644 --- a/apps/api/src/app/activities/activities.service.ts +++ b/apps/api/src/app/activities/activities.service.ts @@ -746,10 +746,10 @@ export class ActivitiesService { const value = new Big(order.quantity).mul(order.unitPrice).toNumber(); const [ - feeInAssetProfileCurrency, - feeInBaseCurrency, - unitPriceInAssetProfileCurrency, - valueInBaseCurrency + feeInAssetProfileCurrency = 0, + feeInBaseCurrency = 0, + unitPriceInAssetProfileCurrency = 0, + valueInBaseCurrency = 0 ] = await Promise.all([ this.exchangeRateDataService.toCurrencyAtDate( order.fee, diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index 2ecc4d3a5..ba704d5ad 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -592,18 +592,19 @@ export class ImportService { const value = new Big(quantity).mul(unitPrice).toNumber(); - const valueInBaseCurrency = this.exchangeRateDataService.toCurrencyAtDate( - value, - currency ?? assetProfile.currency, - userCurrency, - date - ); + const valueInBaseCurrency = + (await this.exchangeRateDataService.toCurrencyAtDate( + value, + currency ?? assetProfile.currency, + userCurrency, + date + )) ?? 0; activities.push({ ...order, error, value, - valueInBaseCurrency: await valueInBaseCurrency, + valueInBaseCurrency, // @ts-ignore SymbolProfile: assetProfile }); diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index 418e60401..77cbb5b49 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -205,21 +205,21 @@ export class PortfolioService { switch (type) { case ActivityType.DIVIDEND: dividendInBaseCurrency += - await this.exchangeRateDataService.toCurrencyAtDate( + (await this.exchangeRateDataService.toCurrencyAtDate( new Big(quantity).mul(unitPrice).toNumber(), currency ?? SymbolProfile.currency, userCurrency, date - ); + )) ?? 0; break; case ActivityType.INTEREST: interestInBaseCurrency += - await this.exchangeRateDataService.toCurrencyAtDate( + (await this.exchangeRateDataService.toCurrencyAtDate( unitPrice, currency ?? SymbolProfile.currency, userCurrency, date - ); + )) ?? 0; break; } diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 27c741055..086434724 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -1,6 +1,7 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.interface'; import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; +import { DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_TIMEOUT } from '@ghostfolio/common/config'; import { UpdateMarketDataDto } from '@ghostfolio/common/dtos'; import { resetHours } from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; @@ -155,49 +156,52 @@ export class MarketDataService { dataSource, symbol }: AssetProfileIdentifier & { data: Prisma.MarketDataUpdateInput[] }) { - await this.prismaService.$transaction(async (prisma) => { - if (data.length > 0) { - let minTime = Infinity; - let maxTime = -Infinity; + await this.prismaService.$transaction( + async (prisma) => { + if (data.length > 0) { + let minTime = Infinity; + let maxTime = -Infinity; - for (const { date } of data) { - const time = (date as Date).getTime(); + for (const { date } of data) { + const time = (date as Date).getTime(); - if (time < minTime) { - minTime = time; - } + if (time < minTime) { + minTime = time; + } - if (time > maxTime) { - maxTime = time; + if (time > maxTime) { + maxTime = time; + } } - } - const minDate = new Date(minTime); - const maxDate = new Date(maxTime); + const minDate = new Date(minTime); + const maxDate = new Date(maxTime); - await prisma.marketData.deleteMany({ - where: { - dataSource, - symbol, - date: { - gte: minDate, - lte: maxDate + await prisma.marketData.deleteMany({ + where: { + dataSource, + symbol, + date: { + gte: minDate, + lte: maxDate + } } - } - }); + }); - await prisma.marketData.createMany({ - data: data.map(({ date, marketPrice, state }) => ({ - dataSource, - symbol, - date: date as Date, - marketPrice: marketPrice as number, - state: state as MarketDataState - })), - skipDuplicates: true - }); - } - }); + await prisma.marketData.createMany({ + data: data.map(({ date, marketPrice, state }) => ({ + dataSource, + symbol, + date: date as Date, + marketPrice: marketPrice as number, + state: state as MarketDataState + })), + skipDuplicates: true + }); + } + }, + { timeout: DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_TIMEOUT } + ); } public async updateAssetProfileIdentifier( diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index e6d717c7b..f96a934e7 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -90,6 +90,7 @@ export const DEFAULT_PAGE_SIZE = 50; export const DEFAULT_PORT = 3333; export const DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY = 1; export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY = 1; +export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_TIMEOUT = 60000; export const DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY = 1; export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY = 1; export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000;