From 1d9ec794bca9576180fec48b123fdf450da5c7ec Mon Sep 17 00:00:00 2001 From: Reto Kaul Date: Sat, 13 Apr 2024 14:52:26 +0200 Subject: [PATCH] Integrate chart calculation into snapshot calculation --- apps/api/src/app/order/order.service.ts | 3 + .../calculator/portfolio-calculator.ts | 230 +++++++++++++++++- .../calculator/twr/portfolio-calculator.ts | 6 + .../portfolio-snapshot.interface.ts | 7 +- .../src/app/portfolio/portfolio.controller.ts | 12 + .../src/app/portfolio/portfolio.service.ts | 85 ++++++- 6 files changed, 329 insertions(+), 14 deletions(-) diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 35bfa1bcf..7e3e69acf 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -223,6 +223,7 @@ export class OrderService { userId: string; withExcludedAccounts?: boolean; }): Promise { + console.time('------ OrderService.getOrders'); let orderBy: Prisma.Enumerable = [ { date: 'asc' } ]; @@ -382,6 +383,8 @@ export class OrderService { }; }); + console.timeEnd('------ OrderService.getOrders'); + return { activities, count }; } diff --git a/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts index 1d2eadfbf..ad043f657 100644 --- a/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts @@ -69,6 +69,8 @@ export abstract class PortfolioCalculator { dateRange: DateRange; exchangeRateDataService: ExchangeRateDataService; }) { + console.time('--- PortfolioCalculator.constructor - 1'); + this.currency = currency; this.currentRateService = currentRateService; this.exchangeRateDataService = exchangeRateDataService; @@ -95,9 +97,16 @@ export abstract class PortfolioCalculator { this.endDate = endDate; this.startDate = startDate; + console.timeEnd('--- PortfolioCalculator.constructor - 1'); + console.time('--- PortfolioCalculator.constructor - 2'); + this.computeTransactionPoints(); + console.timeEnd('--- PortfolioCalculator.constructor - 2'); + + console.time('--- PortfolioCalculator.constructor - 3'); this.snapshotPromise = this.initialize(); + console.timeEnd('--- PortfolioCalculator.constructor - 3'); } protected abstract calculateOverallPerformance( @@ -126,6 +135,7 @@ export abstract class PortfolioCalculator { if (!transactionPoints.length) { return { + chartData: [], currentValueInBaseCurrency: new Big(0), grossPerformance: new Big(0), grossPerformancePercentage: new Big(0), @@ -247,6 +257,26 @@ export abstract class PortfolioCalculator { const endDateString = format(endDate, DATE_FORMAT); + const chartStartDate = this.getStartDate(); + const daysInMarket = differenceInDays(endDate, chartStartDate) + 1; + + const step = true /*withDataDecimation*/ + ? Math.round(daysInMarket / Math.min(daysInMarket, MAX_CHART_ITEMS)) + : 1; + + let chartDates = eachDayOfInterval( + { start: chartStartDate, end }, + { step } + ).map((date) => { + return resetHours(date); + }); + + const includesEndDate = isSameDay(last(chartDates), end); + + if (!includesEndDate) { + chartDates.push(resetHours(end)); + } + if (firstIndex > 0) { firstIndex--; } @@ -256,6 +286,34 @@ export abstract class PortfolioCalculator { const errors: ResponseError['errors'] = []; + const accumulatedValuesByDate: { + [date: string]: { + investmentValueWithCurrencyEffect: Big; + totalCurrentValue: Big; + totalCurrentValueWithCurrencyEffect: Big; + totalInvestmentValue: Big; + totalInvestmentValueWithCurrencyEffect: Big; + totalNetPerformanceValue: Big; + totalNetPerformanceValueWithCurrencyEffect: Big; + totalTimeWeightedInvestmentValue: Big; + totalTimeWeightedInvestmentValueWithCurrencyEffect: Big; + }; + } = {}; + + const valuesBySymbol: { + [symbol: string]: { + currentValues: { [date: string]: Big }; + currentValuesWithCurrencyEffect: { [date: string]: Big }; + investmentValuesAccumulated: { [date: string]: Big }; + investmentValuesAccumulatedWithCurrencyEffect: { [date: string]: Big }; + investmentValuesWithCurrencyEffect: { [date: string]: Big }; + netPerformanceValues: { [date: string]: Big }; + netPerformanceValuesWithCurrencyEffect: { [date: string]: Big }; + timeWeightedInvestmentValues: { [date: string]: Big }; + timeWeightedInvestmentValuesWithCurrencyEffect: { [date: string]: Big }; + }; + } = {}; + for (const item of lastTransactionPoint.items) { const marketPriceInBaseCurrency = ( marketSymbolMap[endDateString]?.[item.symbol] ?? item.averagePrice @@ -266,16 +324,25 @@ export abstract class PortfolioCalculator { ); const { + currentValues, + currentValuesWithCurrencyEffect, grossPerformance, grossPerformancePercentage, grossPerformancePercentageWithCurrencyEffect, grossPerformanceWithCurrencyEffect, hasErrors, + investmentValuesAccumulated, + investmentValuesAccumulatedWithCurrencyEffect, + investmentValuesWithCurrencyEffect, netPerformance, netPerformancePercentage, netPerformancePercentageWithCurrencyEffect, + netPerformanceValues, + netPerformanceValuesWithCurrencyEffect, netPerformanceWithCurrencyEffect, timeWeightedInvestment, + timeWeightedInvestmentValues, + timeWeightedInvestmentValuesWithCurrencyEffect, timeWeightedInvestmentWithCurrencyEffect, totalDividend, totalDividendInBaseCurrency, @@ -287,15 +354,29 @@ export abstract class PortfolioCalculator { } = this.getSymbolMetrics({ marketSymbolMap, start, + step, dataSource: item.dataSource, end: endDate, exchangeRates: exchangeRatesByCurrency[`${item.currency}${this.currency}`], + isChartMode: true, symbol: item.symbol }); hasAnySymbolMetricsErrors = hasAnySymbolMetricsErrors || hasErrors; + valuesBySymbol[item.symbol] = { + currentValues, + currentValuesWithCurrencyEffect, + investmentValuesAccumulated, + investmentValuesAccumulatedWithCurrencyEffect, + investmentValuesWithCurrencyEffect, + netPerformanceValues, + netPerformanceValuesWithCurrencyEffect, + timeWeightedInvestmentValues, + timeWeightedInvestmentValuesWithCurrencyEffect + }; + positions.push({ dividend: totalDividend, dividendInBaseCurrency: totalDividendInBaseCurrency, @@ -363,10 +444,143 @@ export abstract class PortfolioCalculator { } } + for (const currentDate of chartDates) { + const dateString = format(currentDate, DATE_FORMAT); + + for (const symbol of Object.keys(valuesBySymbol)) { + const symbolValues = valuesBySymbol[symbol]; + + const currentValue = + symbolValues.currentValues?.[dateString] ?? new Big(0); + + const currentValueWithCurrencyEffect = + symbolValues.currentValuesWithCurrencyEffect?.[dateString] ?? + new Big(0); + + const investmentValueAccumulated = + symbolValues.investmentValuesAccumulated?.[dateString] ?? new Big(0); + + const investmentValueAccumulatedWithCurrencyEffect = + symbolValues.investmentValuesAccumulatedWithCurrencyEffect?.[ + dateString + ] ?? new Big(0); + + const investmentValueWithCurrencyEffect = + symbolValues.investmentValuesWithCurrencyEffect?.[dateString] ?? + new Big(0); + + const netPerformanceValue = + symbolValues.netPerformanceValues?.[dateString] ?? new Big(0); + + const netPerformanceValueWithCurrencyEffect = + symbolValues.netPerformanceValuesWithCurrencyEffect?.[dateString] ?? + new Big(0); + + const timeWeightedInvestmentValue = + symbolValues.timeWeightedInvestmentValues?.[dateString] ?? new Big(0); + + const timeWeightedInvestmentValueWithCurrencyEffect = + symbolValues.timeWeightedInvestmentValuesWithCurrencyEffect?.[ + dateString + ] ?? new Big(0); + + accumulatedValuesByDate[dateString] = { + investmentValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.investmentValueWithCurrencyEffect ?? new Big(0) + ).add(investmentValueWithCurrencyEffect), + totalCurrentValue: ( + accumulatedValuesByDate[dateString]?.totalCurrentValue ?? new Big(0) + ).add(currentValue), + totalCurrentValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.totalCurrentValueWithCurrencyEffect ?? new Big(0) + ).add(currentValueWithCurrencyEffect), + totalInvestmentValue: ( + accumulatedValuesByDate[dateString]?.totalInvestmentValue ?? + new Big(0) + ).add(investmentValueAccumulated), + totalInvestmentValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.totalInvestmentValueWithCurrencyEffect ?? new Big(0) + ).add(investmentValueAccumulatedWithCurrencyEffect), + totalNetPerformanceValue: ( + accumulatedValuesByDate[dateString]?.totalNetPerformanceValue ?? + new Big(0) + ).add(netPerformanceValue), + totalNetPerformanceValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.totalNetPerformanceValueWithCurrencyEffect ?? new Big(0) + ).add(netPerformanceValueWithCurrencyEffect), + totalTimeWeightedInvestmentValue: ( + accumulatedValuesByDate[dateString] + ?.totalTimeWeightedInvestmentValue ?? new Big(0) + ).add(timeWeightedInvestmentValue), + totalTimeWeightedInvestmentValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.totalTimeWeightedInvestmentValueWithCurrencyEffect ?? new Big(0) + ).add(timeWeightedInvestmentValueWithCurrencyEffect) + }; + } + } + + const chartData: HistoricalDataItem[] = Object.entries( + accumulatedValuesByDate + ).map(([date, values]) => { + const { + investmentValueWithCurrencyEffect, + totalCurrentValue, + totalCurrentValueWithCurrencyEffect, + totalInvestmentValue, + totalInvestmentValueWithCurrencyEffect, + totalNetPerformanceValue, + totalNetPerformanceValueWithCurrencyEffect, + totalTimeWeightedInvestmentValue, + totalTimeWeightedInvestmentValueWithCurrencyEffect + } = values; + + console.log( + 'Chart: totalTimeWeightedInvestmentValue', + totalTimeWeightedInvestmentValue.toFixed() + ); + + const netPerformanceInPercentage = totalTimeWeightedInvestmentValue.eq(0) + ? 0 + : totalNetPerformanceValue + .div(totalTimeWeightedInvestmentValue) + .mul(100) + .toNumber(); + + const netPerformanceInPercentageWithCurrencyEffect = + totalTimeWeightedInvestmentValueWithCurrencyEffect.eq(0) + ? 0 + : totalNetPerformanceValueWithCurrencyEffect + .div(totalTimeWeightedInvestmentValueWithCurrencyEffect) + .mul(100) + .toNumber(); + + return { + date, + netPerformanceInPercentage, + netPerformanceInPercentageWithCurrencyEffect, + investmentValueWithCurrencyEffect: + investmentValueWithCurrencyEffect.toNumber(), + netPerformance: totalNetPerformanceValue.toNumber(), + netPerformanceWithCurrencyEffect: + totalNetPerformanceValueWithCurrencyEffect.toNumber(), + totalInvestment: totalInvestmentValue.toNumber(), + totalInvestmentValueWithCurrencyEffect: + totalInvestmentValueWithCurrencyEffect.toNumber(), + value: totalCurrentValue.toNumber(), + valueWithCurrencyEffect: totalCurrentValueWithCurrencyEffect.toNumber() + }; + }); + const overall = this.calculateOverallPerformance(positions); return { ...overall, + chartData, errors, positions, totalInterestWithCurrencyEffect, @@ -383,6 +597,8 @@ export abstract class PortfolioCalculator { dateRange?: DateRange; withDataDecimation?: boolean; }): Promise { + console.time('-------- PortfolioCalculator.getChart'); + if (this.getTransactionPoints().length === 0) { return []; } @@ -394,11 +610,15 @@ export abstract class PortfolioCalculator { ? Math.round(daysInMarket / Math.min(daysInMarket, MAX_CHART_ITEMS)) : 1; - return this.getChartData({ + const chartData = await this.getChartData({ step, end: endDate, start: startDate }); + + console.timeEnd('-------- PortfolioCalculator.getChart'); + + return chartData; } public async getChartData({ @@ -637,6 +857,11 @@ export abstract class PortfolioCalculator { totalTimeWeightedInvestmentValueWithCurrencyEffect } = values; + console.log( + 'Chart: totalTimeWeightedInvestmentValue', + totalTimeWeightedInvestmentValue.toFixed() + ); + const netPerformanceInPercentage = totalTimeWeightedInvestmentValue.eq(0) ? 0 : totalNetPerformanceValue @@ -743,8 +968,11 @@ export abstract class PortfolioCalculator { } public async getSnapshot() { + console.time('getSnapshot'); await this.snapshotPromise; + console.timeEnd('getSnapshot'); + return this.snapshot; } diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts index 7dcef89cb..c60f9a9f8 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts @@ -102,6 +102,11 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { } } + console.log( + 'Overall: totalTimeWeightedInvestmentValue', + totalTimeWeightedInvestment.toFixed() + ); + return { currentValueInBaseCurrency, grossPerformance, @@ -114,6 +119,7 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { totalInterestWithCurrencyEffect, totalInvestment, totalInvestmentWithCurrencyEffect, + chartData: [], netPerformancePercentage: totalTimeWeightedInvestment.eq(0) ? new Big(0) : netPerformance.div(totalTimeWeightedInvestment), diff --git a/apps/api/src/app/portfolio/interfaces/portfolio-snapshot.interface.ts b/apps/api/src/app/portfolio/interfaces/portfolio-snapshot.interface.ts index d89734987..0b5559981 100644 --- a/apps/api/src/app/portfolio/interfaces/portfolio-snapshot.interface.ts +++ b/apps/api/src/app/portfolio/interfaces/portfolio-snapshot.interface.ts @@ -1,8 +1,13 @@ -import { ResponseError, TimelinePosition } from '@ghostfolio/common/interfaces'; +import { + HistoricalDataItem, + ResponseError, + TimelinePosition +} from '@ghostfolio/common/interfaces'; import { Big } from 'big.js'; export interface PortfolioSnapshot extends ResponseError { + chartData: HistoricalDataItem[]; currentValueInBaseCurrency: Big; grossPerformance: Big; grossPerformanceWithCurrencyEffect: Big; diff --git a/apps/api/src/app/portfolio/portfolio.controller.ts b/apps/api/src/app/portfolio/portfolio.controller.ts index 56c0a231c..a887411ce 100644 --- a/apps/api/src/app/portfolio/portfolio.controller.ts +++ b/apps/api/src/app/portfolio/portfolio.controller.ts @@ -80,6 +80,8 @@ export class PortfolioController { @Query('tags') filterByTags?: string, @Query('withMarkets') withMarketsParam = 'false' ): Promise { + console.time('TOTAL'); + const withMarkets = withMarketsParam === 'true'; let hasDetails = true; @@ -100,6 +102,8 @@ export class PortfolioController { filterByTags }); + console.time('- PortfolioController.getDetails - 1'); + const { accounts, hasErrors, holdings, platforms, summary } = await this.portfolioService.getDetails({ dateRange, @@ -110,6 +114,10 @@ export class PortfolioController { withSummary: true }); + console.timeEnd('- PortfolioController.getDetails - 1'); + + console.time('- PortfolioController.getDetails - 2'); + if (hasErrors || hasNotDefinedValuesInObject(holdings)) { hasError = true; } @@ -202,6 +210,10 @@ export class PortfolioController { }; } + console.timeEnd('- PortfolioController.getDetails - 2'); + + console.timeEnd('TOTAL'); + return { accounts, hasError, diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index 95a68eaae..04208d500 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -334,6 +334,8 @@ export class PortfolioService { withMarkets?: boolean; withSummary?: boolean; }): Promise { + console.time('-- PortfolioService.getDetails - 1'); + userId = await this.getUserId(impersonationId, userId); const user = await this.userService.user({ id: userId }); const userCurrency = this.getUserCurrency(user); @@ -356,9 +358,16 @@ export class PortfolioService { currency: userCurrency }); + console.timeEnd('-- PortfolioService.getDetails - 1'); + + console.time('-- PortfolioService.getDetails - 2'); + const { currentValueInBaseCurrency, hasErrors, positions } = await portfolioCalculator.getSnapshot(); + console.timeEnd('-- PortfolioService.getDetails - 2'); + console.time('-- PortfolioService.getDetails - 3'); + const cashDetails = await this.accountService.getCashDetails({ filters, userId, @@ -407,6 +416,9 @@ export class PortfolioService { }; }); + console.timeEnd('-- PortfolioService.getDetails - 3'); + console.time('-- PortfolioService.getDetails - 4'); + const [dataProviderResponses, symbolProfiles] = await Promise.all([ this.dataProviderService.getQuotes({ user, items: dataGatheringItems }), this.symbolProfileService.getSymbolProfiles(dataGatheringItems) @@ -422,6 +434,9 @@ export class PortfolioService { portfolioItemsNow[position.symbol] = position; } + console.timeEnd('-- PortfolioService.getDetails - 4'); + console.time('-- PortfolioService.getDetails - 5'); + for (const { currency, dividend, @@ -562,6 +577,10 @@ export class PortfolioService { }; } + console.timeEnd('-- PortfolioService.getDetails - 5'); + + console.time('-- PortfolioService.getDetails - 6'); + let summary: PortfolioSummary; if (withSummary) { @@ -580,6 +599,8 @@ export class PortfolioService { }); } + console.timeEnd('-- PortfolioService.getDetails - 6'); + return { accounts, hasErrors, @@ -1022,15 +1043,20 @@ export class PortfolioService { dateRange = 'max', filters, impersonationId, + portfolioCalculator, userId, withExcludedAccounts = false }: { dateRange?: DateRange; filters?: Filter[]; impersonationId: string; + portfolioCalculator?: PortfolioCalculator; userId: string; withExcludedAccounts?: boolean; }): Promise { + // OPTIMIZE (1.34s) + console.time('------ PortfolioService.getPerformance'); + userId = await this.getUserId(impersonationId, userId); const user = await this.userService.user({ id: userId }); const userCurrency = this.getUserCurrency(user); @@ -1062,6 +1088,8 @@ export class PortfolioService { const { endDate, startDate } = getInterval(dateRange); + console.time('------- PortfolioService.getPerformance - 2'); + const { activities } = await this.orderService.getOrders({ endDate, filters, @@ -1070,6 +1098,9 @@ export class PortfolioService { withExcludedAccounts }); + console.timeEnd('------- PortfolioService.getPerformance - 2'); + console.time('------- PortfolioService.getPerformance - 3'); + if (accountBalanceItems?.length <= 0 && activities?.length <= 0) { return { chart: [], @@ -1091,14 +1122,17 @@ export class PortfolioService { }; } - const portfolioCalculator = this.calculatorFactory.createCalculator({ - activities, - dateRange, - calculationType: PerformanceCalculationType.TWR, - currency: userCurrency - }); + portfolioCalculator = + portfolioCalculator ?? + this.calculatorFactory.createCalculator({ + activities, + dateRange, + calculationType: PerformanceCalculationType.TWR, + currency: userCurrency + }); const { + chartData, currentValueInBaseCurrency, errors, grossPerformance, @@ -1113,6 +1147,9 @@ export class PortfolioService { totalInvestment } = await portfolioCalculator.getSnapshot(); + console.timeEnd('------- PortfolioService.getPerformance - 3'); + console.time('------- PortfolioService.getPerformance - 4'); + let currentNetPerformance = netPerformance; let currentNetPerformancePercent = netPerformancePercentage; @@ -1123,11 +1160,10 @@ export class PortfolioService { let currentNetPerformanceWithCurrencyEffect = netPerformanceWithCurrencyEffect; - const items = await portfolioCalculator.getChart({ - dateRange - }); + console.timeEnd('------- PortfolioService.getPerformance - 4'); + console.time('------- PortfolioService.getPerformance - 5'); - const itemOfToday = items.find(({ date }) => { + const itemOfToday = chartData.find(({ date }) => { return date === format(new Date(), DATE_FORMAT); }); @@ -1162,19 +1198,25 @@ export class PortfolioService { }); } + console.timeEnd('------- PortfolioService.getPerformance - 5'); + console.time('------- PortfolioService.getPerformance - 6'); + const mergedHistoricalDataItems = this.mergeHistoricalDataItems( accountBalanceItems, - items + chartData ); const currentHistoricalDataItem = last(mergedHistoricalDataItems); const currentNetWorth = currentHistoricalDataItem?.netWorth ?? 0; + console.timeEnd('------- PortfolioService.getPerformance - 6'); + console.timeEnd('------ PortfolioService.getPerformance'); + return { errors, hasErrors, chart: mergedHistoricalDataItems, - firstOrderDate: parseDate(items[0]?.date), + firstOrderDate: parseDate(chartData[0]?.date), performance: { currentNetWorth, currentGrossPerformance: grossPerformance.toNumber(), @@ -1579,11 +1621,17 @@ export class PortfolioService { userCurrency: string; userId: string; }): Promise { + // OPTIMIZE (1.1 s) + console.time('---- PortfolioService.getSummary'); + userId = await this.getUserId(impersonationId, userId); const user = await this.userService.user({ id: userId }); + console.time('----- PortfolioService.getSummary - 1'); + const performanceInformation = await this.getPerformance({ impersonationId, + portfolioCalculator, userId }); @@ -1593,6 +1641,9 @@ export class PortfolioService { withExcludedAccounts: true }); + console.timeEnd('----- PortfolioService.getSummary - 1'); + console.time('----- PortfolioService.getSummary - 2'); + const excludedActivities: Activity[] = []; const nonExcludedActivities: Activity[] = []; @@ -1620,6 +1671,9 @@ export class PortfolioService { const interest = await portfolioCalculator.getInterestInBaseCurrency(); + console.timeEnd('----- PortfolioService.getSummary - 2'); + console.time('----- PortfolioService.getSummary - 3'); + const liabilities = await portfolioCalculator.getLiabilitiesInBaseCurrency(); @@ -1656,6 +1710,9 @@ export class PortfolioService { }) ); + console.timeEnd('----- PortfolioService.getSummary - 3'); + console.time('----- PortfolioService.getSummary - 4'); + const cashDetailsWithExcludedAccounts = await this.accountService.getCashDetails({ userId, @@ -1695,6 +1752,10 @@ export class PortfolioService { ) })?.toNumber(); + console.timeEnd('----- PortfolioService.getSummary - 4'); + + console.timeEnd('---- PortfolioService.getSummary'); + return { ...performanceInformation.performance, annualizedPerformancePercent,