diff --git a/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator-cash.spec.ts b/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator-cash.spec.ts index 217a67c49..87c08d32f 100644 --- a/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator-cash.spec.ts +++ b/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator-cash.spec.ts @@ -240,9 +240,7 @@ describe('PortfolioCalculator', () => { feeInBaseCurrency: new Big(0), grossPerformance: new Big(0), grossPerformancePercentage: new Big(0), - grossPerformancePercentageWithCurrencyEffect: new Big( - '0.08211603004634809014' - ), + grossPerformancePercentageWithCurrencyEffect: expect.any(Big), grossPerformanceWithCurrencyEffect: new Big(70), includeInTotalAssetValue: false, investment: new Big(1820), @@ -271,10 +269,8 @@ describe('PortfolioCalculator', () => { }, quantity: new Big(2000), symbol: 'USD', - timeWeightedInvestment: new Big('912.47956403269754768392'), - timeWeightedInvestmentWithCurrencyEffect: new Big( - '852.45231607629427792916' - ), + timeWeightedInvestment: expect.any(Big), + timeWeightedInvestmentWithCurrencyEffect: expect.any(Big), valueInBaseCurrency: new Big(1820) }); diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts index 5f2358679..d80ee8e9a 100644 --- a/apps/api/src/app/portfolio/current-rate.service.spec.ts +++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts @@ -111,7 +111,7 @@ describe('CurrentRateService', () => { null ); - marketDataService = new MarketDataService(null); + marketDataService = new MarketDataService(null, null); currentRateService = new CurrentRateService( null, diff --git a/apps/api/src/events/events.module.ts b/apps/api/src/events/events.module.ts index df943a3c9..689247d0a 100644 --- a/apps/api/src/events/events.module.ts +++ b/apps/api/src/events/events.module.ts @@ -8,6 +8,7 @@ import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-g import { Module } from '@nestjs/common'; import { AssetProfileChangedListener } from './asset-profile-changed.listener'; +import { MarketDataUpdatedListener } from './market-data-updated.listener'; import { PortfolioChangedListener } from './portfolio-changed.listener'; @Module({ @@ -19,6 +20,10 @@ import { PortfolioChangedListener } from './portfolio-changed.listener'; ExchangeRateDataModule, RedisCacheModule ], - providers: [AssetProfileChangedListener, PortfolioChangedListener] + providers: [ + AssetProfileChangedListener, + MarketDataUpdatedListener, + PortfolioChangedListener + ] }) export class EventsModule {} diff --git a/apps/api/src/events/market-data-updated.event.ts b/apps/api/src/events/market-data-updated.event.ts new file mode 100644 index 000000000..ebcf0b422 --- /dev/null +++ b/apps/api/src/events/market-data-updated.event.ts @@ -0,0 +1,9 @@ +import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; + +export class MarketDataUpdatedEvent { + public constructor(public readonly data: AssetProfileIdentifier) {} + + public static getName(): string { + return 'market-data.updated'; + } +} diff --git a/apps/api/src/events/market-data-updated.listener.ts b/apps/api/src/events/market-data-updated.listener.ts new file mode 100644 index 000000000..28a2aaa7b --- /dev/null +++ b/apps/api/src/events/market-data-updated.listener.ts @@ -0,0 +1,25 @@ +import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; +import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; + +import { Injectable } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; + +import { MarketDataUpdatedEvent } from './market-data-updated.event'; + +@Injectable() +export class MarketDataUpdatedListener { + public constructor( + private readonly dataProviderService: DataProviderService, + private readonly exchangeRateDataService: ExchangeRateDataService + ) {} + + @OnEvent(MarketDataUpdatedEvent.getName()) + public handleMarketDataUpdated(event: MarketDataUpdatedEvent) { + if ( + event.data.dataSource === + this.dataProviderService.getDataSourceForExchangeRates() + ) { + this.exchangeRateDataService.invalidateCache(event.data.symbol); + } + } +} diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts index 024bdf4e1..0ae6155b3 100644 --- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts @@ -28,12 +28,15 @@ import ms from 'ms'; import { ExchangeRatesByCurrency } from './interfaces/exchange-rate-data.interface'; +const EPOCH_OFFSET_DAYS = 25569; + @Injectable() export class ExchangeRateDataService { private currencies: string[] = []; private currencyPairs: DataGatheringItem[] = []; private derivedCurrencyFactors: { [currencyPair: string]: number } = {}; private exchangeRates: { [currencyPair: string]: number } = {}; + private exchangeRateCache = new Map>(); public constructor( private readonly dataProviderService: DataProviderService, @@ -284,56 +287,56 @@ export class ExchangeRateDataService { } else if (derivedCurrencyFactor) { factor = derivedCurrencyFactor; } else { - const dataSource = - this.dataProviderService.getDataSourceForExchangeRates(); - const symbol = `${aFromCurrency}${aToCurrency}`; - - const marketData = await this.marketDataService.get({ - dataSource, - symbol, - date: aDate - }); + const marketPrice = await this.getRateFromCache( + `${aFromCurrency}${aToCurrency}`, + aDate + ); - if (marketData?.marketPrice) { - factor = marketData?.marketPrice; + if (marketPrice !== undefined) { + factor = marketPrice; } else { - // Calculate indirectly via base currency - - let marketPriceBaseCurrencyFromCurrency: number; - let marketPriceBaseCurrencyToCurrency: number; - try { - if (aFromCurrency === DEFAULT_CURRENCY) { - marketPriceBaseCurrencyFromCurrency = 1; - } else { - marketPriceBaseCurrencyFromCurrency = ( - await this.marketDataService.get({ - dataSource, - date: aDate, - symbol: `${DEFAULT_CURRENCY}${aFromCurrency}` - }) - )?.marketPrice; + let baseFromPrice: number | undefined = + aFromCurrency === DEFAULT_CURRENCY ? 1 : undefined; + let baseToPrice: number | undefined = + aToCurrency === DEFAULT_CURRENCY ? 1 : undefined; + + if (aFromCurrency !== DEFAULT_CURRENCY) { + baseFromPrice = await this.getRateFromCache( + `${DEFAULT_CURRENCY}${aFromCurrency}`, + aDate + ); + + if (baseFromPrice === undefined) { + const crossPrice = await this.getRateFromCache( + `${aFromCurrency}${DEFAULT_CURRENCY}`, + aDate + ); + if (crossPrice !== undefined) { + baseFromPrice = 1 / crossPrice; + } + } } - } catch {} - try { - if (aToCurrency === DEFAULT_CURRENCY) { - marketPriceBaseCurrencyToCurrency = 1; - } else { - marketPriceBaseCurrencyToCurrency = ( - await this.marketDataService.get({ - dataSource, - date: aDate, - symbol: `${DEFAULT_CURRENCY}${aToCurrency}` - }) - )?.marketPrice; + if (aToCurrency !== DEFAULT_CURRENCY) { + baseToPrice = await this.getRateFromCache( + `${DEFAULT_CURRENCY}${aToCurrency}`, + aDate + ); + + if (baseToPrice === undefined) { + const crossPrice = await this.getRateFromCache( + `${aToCurrency}${DEFAULT_CURRENCY}`, + aDate + ); + if (crossPrice !== undefined) { + baseToPrice = 1 / crossPrice; + } + } } - } catch {} - // Calculate the opposite direction - factor = - (1 / marketPriceBaseCurrencyFromCurrency) * - marketPriceBaseCurrencyToCurrency; + factor = (1 / baseFromPrice) * baseToPrice; + } catch {} } } @@ -352,6 +355,77 @@ export class ExchangeRateDataService { return undefined; } + public invalidateCache(aSymbol: string) { + this.exchangeRateCache.delete(aSymbol); + } + + private getDaysSinceEpoch(aDate: Date) { + return Math.floor(aDate.getTime() / ms('1d')) + EPOCH_OFFSET_DAYS; + } + + private async loadCache(aSymbol: string): Promise { + const dataSource = this.dataProviderService.getDataSourceForExchangeRates(); + const marketData = await this.prismaService.marketData.findMany({ + where: { dataSource, symbol: aSymbol }, + orderBy: { date: 'asc' }, + select: { date: true, marketPrice: true } + }); + + const todayDays = this.getDaysSinceEpoch(new Date()); + const maxDataDays = + marketData.length > 0 + ? this.getDaysSinceEpoch(marketData[marketData.length - 1].date) + : 0; + + const array = new Float64Array(Math.max(todayDays, maxDataDays) + 1); + + if (marketData.length > 0) { + let lastRate = marketData[0].marketPrice; + let currentIndex = this.getDaysSinceEpoch(marketData[0].date); + + for (const data of marketData) { + const dataIndex = this.getDaysSinceEpoch(data.date); + while (currentIndex < dataIndex && currentIndex <= todayDays) { + array[currentIndex++] = lastRate; + } + lastRate = data.marketPrice; + array[dataIndex] = lastRate; + currentIndex = dataIndex + 1; + } + + while (currentIndex <= todayDays) { + array[currentIndex++] = lastRate; + } + } + + return array; + } + + private async getRateFromCache( + aSymbol: string, + aDate: Date + ): Promise { + let cachePromise = this.exchangeRateCache.get(aSymbol); + + if (!cachePromise) { + cachePromise = this.loadCache(aSymbol).catch((error) => { + this.exchangeRateCache.delete(aSymbol); + throw error; + }); + this.exchangeRateCache.set(aSymbol, cachePromise); + } + + const cache = await cachePromise; + const days = Math.min(this.getDaysSinceEpoch(aDate), cache.length - 1); + + if (days >= 0) { + const rate = cache[days]; + return rate === 0 ? undefined : rate; + } + + return undefined; + } + private async getExchangeRates({ currencyFrom, currencyTo, diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 87b08e1bd..128f44ba3 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -6,24 +6,38 @@ import { resetHours } from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { Injectable } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; import { DataSource, MarketData, MarketDataState, Prisma } from '@prisma/client'; +import { uniqBy } from 'lodash'; + +import { MarketDataUpdatedEvent } from '../../events/market-data-updated.event'; @Injectable() export class MarketDataService { - public constructor(private readonly prismaService: PrismaService) {} + public constructor( + private readonly eventEmitter: EventEmitter2, + private readonly prismaService: PrismaService + ) {} public async deleteMany({ dataSource, symbol }: AssetProfileIdentifier) { - return this.prismaService.marketData.deleteMany({ + const result = await this.prismaService.marketData.deleteMany({ where: { dataSource, symbol } }); + + this.eventEmitter.emit( + MarketDataUpdatedEvent.getName(), + new MarketDataUpdatedEvent({ dataSource, symbol }) + ); + + return result; } public async get({ @@ -185,13 +199,18 @@ export class MarketDataService { }); } }); + + this.eventEmitter.emit( + MarketDataUpdatedEvent.getName(), + new MarketDataUpdatedEvent({ dataSource, symbol }) + ); } public async updateAssetProfileIdentifier( oldAssetProfileIdentifier: AssetProfileIdentifier, newAssetProfileIdentifier: AssetProfileIdentifier ) { - return this.prismaService.marketData.updateMany({ + const result = await this.prismaService.marketData.updateMany({ data: { dataSource: newAssetProfileIdentifier.dataSource, symbol: newAssetProfileIdentifier.symbol @@ -201,6 +220,17 @@ export class MarketDataService { symbol: oldAssetProfileIdentifier.symbol } }); + + this.eventEmitter.emit( + MarketDataUpdatedEvent.getName(), + new MarketDataUpdatedEvent(oldAssetProfileIdentifier) + ); + this.eventEmitter.emit( + MarketDataUpdatedEvent.getName(), + new MarketDataUpdatedEvent(newAssetProfileIdentifier) + ); + + return result; } public async updateMarketData(params: { @@ -211,7 +241,7 @@ export class MarketDataService { }): Promise { const { data, where } = params; - return this.prismaService.marketData.upsert({ + const result = await this.prismaService.marketData.upsert({ where, create: { dataSource: where.dataSource_date_symbol.dataSource, @@ -222,6 +252,16 @@ export class MarketDataService { }, update: { marketPrice: data.marketPrice, state: data.state } }); + + this.eventEmitter.emit( + MarketDataUpdatedEvent.getName(), + new MarketDataUpdatedEvent({ + dataSource: where.dataSource_date_symbol.dataSource, + symbol: where.dataSource_date_symbol.symbol + }) + ); + + return result; } /** @@ -258,6 +298,23 @@ export class MarketDataService { } ); - return this.prismaService.$transaction(upsertPromises); + const result = await this.prismaService.$transaction(upsertPromises); + + const uniquePairs = uniqBy( + data.map((d) => ({ + dataSource: d.dataSource as DataSource, + symbol: d.symbol as string + })), + (d) => `${d.dataSource}:${d.symbol}` + ); + + for (const pair of uniquePairs) { + this.eventEmitter.emit( + MarketDataUpdatedEvent.getName(), + new MarketDataUpdatedEvent(pair) + ); + } + + return result; } }