From c5ec3c9b6c8f9d1e5f98152086c0ef9a4f3e297f Mon Sep 17 00:00:00 2001 From: Reto Kaul Date: Sat, 13 Apr 2024 14:52:26 +0200 Subject: [PATCH] Integrate chart calculation into snapshot calculation --- apps/api/src/app/order/order.service.ts | 3 + .../calculator/portfolio-calculator.ts | 233 +++++++++++++++++- .../calculator/twr/portfolio-calculator.ts | 6 + .../src/app/portfolio/portfolio.controller.ts | 12 + .../src/app/portfolio/portfolio.service.ts | 93 +++++-- .../src/lib/models/portfolio-snapshot.ts | 5 +- 6 files changed, 334 insertions(+), 18 deletions(-) diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index e976fc80d..e2c21c278 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -247,6 +247,7 @@ export class OrderService { userId: string; withExcludedAccounts?: boolean; }): Promise { + console.time('------ OrderService.getOrders'); let orderBy: Prisma.Enumerable = [ { date: 'asc' } ]; @@ -406,6 +407,8 @@ export class OrderService { }; }); + console.timeEnd('------ OrderService.getOrders'); + return { activities, count }; } diff --git a/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts index e021eb2d4..3f3644bf8 100644 --- a/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts @@ -90,6 +90,7 @@ export abstract class PortfolioCalculator { useCache: boolean; userId: string; }) { + console.time('--- PortfolioCalculator.constructor - 1'); this.accountBalanceItems = accountBalanceItems; this.configurationService = configurationService; this.currency = currency; @@ -138,9 +139,16 @@ export abstract class PortfolioCalculator { this.endDate = endDate; this.startDate = startDate; + console.timeEnd('--- PortfolioCalculator.constructor - 1'); + console.time('--- PortfolioCalculator.constructor - 2'); + this.computeTransactionPoints(); + console.timeEnd('--- PortfolioCalculator.constructor - 2'); + + console.time('--- PortfolioCalculator.constructor - 3'); this.snapshotPromise = this.initialize(); + console.timeEnd('--- PortfolioCalculator.constructor - 3'); } protected abstract calculateOverallPerformance( @@ -169,6 +177,7 @@ export abstract class PortfolioCalculator { if (!transactionPoints.length) { return { + chartData: [], currentValueInBaseCurrency: new Big(0), grossPerformance: new Big(0), grossPerformancePercentage: new Big(0), @@ -290,6 +299,26 @@ export abstract class PortfolioCalculator { const endDateString = format(endDate, DATE_FORMAT); + const chartStartDate = this.getStartDate(); + const daysInMarket = differenceInDays(endDate, chartStartDate) + 1; + + const step = true /*withDataDecimation*/ + ? Math.round(daysInMarket / Math.min(daysInMarket, MAX_CHART_ITEMS)) + : 1; + + let chartDates = eachDayOfInterval( + { start: chartStartDate, end }, + { step } + ).map((date) => { + return resetHours(date); + }); + + const includesEndDate = isSameDay(last(chartDates), end); + + if (!includesEndDate) { + chartDates.push(resetHours(end)); + } + if (firstIndex > 0) { firstIndex--; } @@ -299,6 +328,34 @@ export abstract class PortfolioCalculator { const errors: ResponseError['errors'] = []; + const accumulatedValuesByDate: { + [date: string]: { + investmentValueWithCurrencyEffect: Big; + totalCurrentValue: Big; + totalCurrentValueWithCurrencyEffect: Big; + totalInvestmentValue: Big; + totalInvestmentValueWithCurrencyEffect: Big; + totalNetPerformanceValue: Big; + totalNetPerformanceValueWithCurrencyEffect: Big; + totalTimeWeightedInvestmentValue: Big; + totalTimeWeightedInvestmentValueWithCurrencyEffect: Big; + }; + } = {}; + + const valuesBySymbol: { + [symbol: string]: { + currentValues: { [date: string]: Big }; + currentValuesWithCurrencyEffect: { [date: string]: Big }; + investmentValuesAccumulated: { [date: string]: Big }; + investmentValuesAccumulatedWithCurrencyEffect: { [date: string]: Big }; + investmentValuesWithCurrencyEffect: { [date: string]: Big }; + netPerformanceValues: { [date: string]: Big }; + netPerformanceValuesWithCurrencyEffect: { [date: string]: Big }; + timeWeightedInvestmentValues: { [date: string]: Big }; + timeWeightedInvestmentValuesWithCurrencyEffect: { [date: string]: Big }; + }; + } = {}; + for (const item of lastTransactionPoint.items) { const marketPriceInBaseCurrency = ( marketSymbolMap[endDateString]?.[item.symbol] ?? item.averagePrice @@ -309,16 +366,25 @@ export abstract class PortfolioCalculator { ); const { + currentValues, + currentValuesWithCurrencyEffect, grossPerformance, grossPerformancePercentage, grossPerformancePercentageWithCurrencyEffect, grossPerformanceWithCurrencyEffect, hasErrors, + investmentValuesAccumulated, + investmentValuesAccumulatedWithCurrencyEffect, + investmentValuesWithCurrencyEffect, netPerformance, netPerformancePercentage, netPerformancePercentageWithCurrencyEffect, + netPerformanceValues, + netPerformanceValuesWithCurrencyEffect, netPerformanceWithCurrencyEffect, timeWeightedInvestment, + timeWeightedInvestmentValues, + timeWeightedInvestmentValuesWithCurrencyEffect, timeWeightedInvestmentWithCurrencyEffect, totalDividend, totalDividendInBaseCurrency, @@ -330,15 +396,29 @@ export abstract class PortfolioCalculator { } = this.getSymbolMetrics({ marketSymbolMap, start, + step, dataSource: item.dataSource, end: endDate, exchangeRates: exchangeRatesByCurrency[`${item.currency}${this.currency}`], + isChartMode: true, symbol: item.symbol }); hasAnySymbolMetricsErrors = hasAnySymbolMetricsErrors || hasErrors; + valuesBySymbol[item.symbol] = { + currentValues, + currentValuesWithCurrencyEffect, + investmentValuesAccumulated, + investmentValuesAccumulatedWithCurrencyEffect, + investmentValuesWithCurrencyEffect, + netPerformanceValues, + netPerformanceValuesWithCurrencyEffect, + timeWeightedInvestmentValues, + timeWeightedInvestmentValuesWithCurrencyEffect + }; + positions.push({ dividend: totalDividend, dividendInBaseCurrency: totalDividendInBaseCurrency, @@ -406,10 +486,143 @@ export abstract class PortfolioCalculator { } } + for (const currentDate of chartDates) { + const dateString = format(currentDate, DATE_FORMAT); + + for (const symbol of Object.keys(valuesBySymbol)) { + const symbolValues = valuesBySymbol[symbol]; + + const currentValue = + symbolValues.currentValues?.[dateString] ?? new Big(0); + + const currentValueWithCurrencyEffect = + symbolValues.currentValuesWithCurrencyEffect?.[dateString] ?? + new Big(0); + + const investmentValueAccumulated = + symbolValues.investmentValuesAccumulated?.[dateString] ?? new Big(0); + + const investmentValueAccumulatedWithCurrencyEffect = + symbolValues.investmentValuesAccumulatedWithCurrencyEffect?.[ + dateString + ] ?? new Big(0); + + const investmentValueWithCurrencyEffect = + symbolValues.investmentValuesWithCurrencyEffect?.[dateString] ?? + new Big(0); + + const netPerformanceValue = + symbolValues.netPerformanceValues?.[dateString] ?? new Big(0); + + const netPerformanceValueWithCurrencyEffect = + symbolValues.netPerformanceValuesWithCurrencyEffect?.[dateString] ?? + new Big(0); + + const timeWeightedInvestmentValue = + symbolValues.timeWeightedInvestmentValues?.[dateString] ?? new Big(0); + + const timeWeightedInvestmentValueWithCurrencyEffect = + symbolValues.timeWeightedInvestmentValuesWithCurrencyEffect?.[ + dateString + ] ?? new Big(0); + + accumulatedValuesByDate[dateString] = { + investmentValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.investmentValueWithCurrencyEffect ?? new Big(0) + ).add(investmentValueWithCurrencyEffect), + totalCurrentValue: ( + accumulatedValuesByDate[dateString]?.totalCurrentValue ?? new Big(0) + ).add(currentValue), + totalCurrentValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.totalCurrentValueWithCurrencyEffect ?? new Big(0) + ).add(currentValueWithCurrencyEffect), + totalInvestmentValue: ( + accumulatedValuesByDate[dateString]?.totalInvestmentValue ?? + new Big(0) + ).add(investmentValueAccumulated), + totalInvestmentValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.totalInvestmentValueWithCurrencyEffect ?? new Big(0) + ).add(investmentValueAccumulatedWithCurrencyEffect), + totalNetPerformanceValue: ( + accumulatedValuesByDate[dateString]?.totalNetPerformanceValue ?? + new Big(0) + ).add(netPerformanceValue), + totalNetPerformanceValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.totalNetPerformanceValueWithCurrencyEffect ?? new Big(0) + ).add(netPerformanceValueWithCurrencyEffect), + totalTimeWeightedInvestmentValue: ( + accumulatedValuesByDate[dateString] + ?.totalTimeWeightedInvestmentValue ?? new Big(0) + ).add(timeWeightedInvestmentValue), + totalTimeWeightedInvestmentValueWithCurrencyEffect: ( + accumulatedValuesByDate[dateString] + ?.totalTimeWeightedInvestmentValueWithCurrencyEffect ?? new Big(0) + ).add(timeWeightedInvestmentValueWithCurrencyEffect) + }; + } + } + + const chartData: HistoricalDataItem[] = Object.entries( + accumulatedValuesByDate + ).map(([date, values]) => { + const { + investmentValueWithCurrencyEffect, + totalCurrentValue, + totalCurrentValueWithCurrencyEffect, + totalInvestmentValue, + totalInvestmentValueWithCurrencyEffect, + totalNetPerformanceValue, + totalNetPerformanceValueWithCurrencyEffect, + totalTimeWeightedInvestmentValue, + totalTimeWeightedInvestmentValueWithCurrencyEffect + } = values; + + console.log( + 'Chart: totalTimeWeightedInvestmentValue', + totalTimeWeightedInvestmentValue.toFixed() + ); + + const netPerformanceInPercentage = totalTimeWeightedInvestmentValue.eq(0) + ? 0 + : totalNetPerformanceValue + .div(totalTimeWeightedInvestmentValue) + .mul(100) + .toNumber(); + + const netPerformanceInPercentageWithCurrencyEffect = + totalTimeWeightedInvestmentValueWithCurrencyEffect.eq(0) + ? 0 + : totalNetPerformanceValueWithCurrencyEffect + .div(totalTimeWeightedInvestmentValueWithCurrencyEffect) + .mul(100) + .toNumber(); + + return { + date, + netPerformanceInPercentage, + netPerformanceInPercentageWithCurrencyEffect, + investmentValueWithCurrencyEffect: + investmentValueWithCurrencyEffect.toNumber(), + netPerformance: totalNetPerformanceValue.toNumber(), + netPerformanceWithCurrencyEffect: + totalNetPerformanceValueWithCurrencyEffect.toNumber(), + totalInvestment: totalInvestmentValue.toNumber(), + totalInvestmentValueWithCurrencyEffect: + totalInvestmentValueWithCurrencyEffect.toNumber(), + value: totalCurrentValue.toNumber(), + valueWithCurrencyEffect: totalCurrentValueWithCurrencyEffect.toNumber() + }; + }); + const overall = this.calculateOverallPerformance(positions); return { ...overall, + chartData, errors, positions, totalInterestWithCurrencyEffect, @@ -426,6 +639,12 @@ export abstract class PortfolioCalculator { dateRange?: DateRange; withDataDecimation?: boolean; }): Promise { + console.time('-------- PortfolioCalculator.getChart'); + + if (this.getTransactionPoints().length === 0) { + return []; + } + const { endDate, startDate } = getInterval(dateRange, this.getStartDate()); const daysInMarket = differenceInDays(endDate, startDate) + 1; @@ -433,11 +652,15 @@ export abstract class PortfolioCalculator { ? Math.round(daysInMarket / Math.min(daysInMarket, MAX_CHART_ITEMS)) : 1; - return this.getChartData({ + const chartData = await this.getChartData({ step, end: endDate, start: startDate }); + + console.timeEnd('-------- PortfolioCalculator.getChart'); + + return chartData; } public async getChartData({ @@ -737,6 +960,11 @@ export abstract class PortfolioCalculator { totalTimeWeightedInvestmentValueWithCurrencyEffect } = values; + console.log( + 'Chart: totalTimeWeightedInvestmentValue', + totalTimeWeightedInvestmentValue.toFixed() + ); + const netPerformanceInPercentage = totalTimeWeightedInvestmentValue.eq(0) ? 0 : totalNetPerformanceValue @@ -848,8 +1076,11 @@ export abstract class PortfolioCalculator { } public async getSnapshot() { + console.time('getSnapshot'); await this.snapshotPromise; + console.timeEnd('getSnapshot'); + return this.snapshot; } diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts index f8b62a940..476f9e620 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts @@ -98,6 +98,11 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { } } + console.log( + 'Overall: totalTimeWeightedInvestmentValue', + totalTimeWeightedInvestment.toFixed() + ); + return { currentValueInBaseCurrency, grossPerformance, @@ -110,6 +115,7 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { totalInterestWithCurrencyEffect, totalInvestment, totalInvestmentWithCurrencyEffect, + chartData: [], netPerformancePercentage: totalTimeWeightedInvestment.eq(0) ? new Big(0) : netPerformance.div(totalTimeWeightedInvestment), diff --git a/apps/api/src/app/portfolio/portfolio.controller.ts b/apps/api/src/app/portfolio/portfolio.controller.ts index 4a07cd65b..a2fd13894 100644 --- a/apps/api/src/app/portfolio/portfolio.controller.ts +++ b/apps/api/src/app/portfolio/portfolio.controller.ts @@ -81,6 +81,8 @@ export class PortfolioController { @Query('tags') filterByTags?: string, @Query('withMarkets') withMarketsParam = 'false' ): Promise { + console.time('TOTAL'); + const withMarkets = withMarketsParam === 'true'; let hasDetails = true; @@ -101,6 +103,8 @@ export class PortfolioController { filterByTags }); + console.time('- PortfolioController.getDetails - 1'); + const { accounts, hasErrors, holdings, platforms, summary } = await this.portfolioService.getDetails({ dateRange, @@ -111,6 +115,10 @@ export class PortfolioController { withSummary: true }); + console.timeEnd('- PortfolioController.getDetails - 1'); + + console.time('- PortfolioController.getDetails - 2'); + if (hasErrors || hasNotDefinedValuesInObject(holdings)) { hasError = true; } @@ -208,6 +216,10 @@ export class PortfolioController { }; } + console.timeEnd('- PortfolioController.getDetails - 2'); + + console.timeEnd('TOTAL'); + return { accounts, hasError, diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index a98887ca9..dab5c4d85 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -339,6 +339,8 @@ export class PortfolioService { withMarkets?: boolean; withSummary?: boolean; }): Promise { + console.time('-- PortfolioService.getDetails - 1'); + userId = await this.getUserId(impersonationId, userId); const user = await this.userService.user({ id: userId }); const userCurrency = this.getUserCurrency(user); @@ -365,9 +367,16 @@ export class PortfolioService { this.request.user?.Settings.settings.isExperimentalFeatures }); + console.timeEnd('-- PortfolioService.getDetails - 1'); + + console.time('-- PortfolioService.getDetails - 2'); + const { currentValueInBaseCurrency, hasErrors, positions } = await portfolioCalculator.getSnapshot(); + console.timeEnd('-- PortfolioService.getDetails - 2'); + console.time('-- PortfolioService.getDetails - 3'); + const cashDetails = await this.accountService.getCashDetails({ filters, userId, @@ -416,6 +425,9 @@ export class PortfolioService { }; }); + console.timeEnd('-- PortfolioService.getDetails - 3'); + console.time('-- PortfolioService.getDetails - 4'); + const [dataProviderResponses, symbolProfiles] = await Promise.all([ this.dataProviderService.getQuotes({ user, items: dataGatheringItems }), this.symbolProfileService.getSymbolProfiles(dataGatheringItems) @@ -431,6 +443,9 @@ export class PortfolioService { portfolioItemsNow[position.symbol] = position; } + console.timeEnd('-- PortfolioService.getDetails - 4'); + console.time('-- PortfolioService.getDetails - 5'); + for (const { currency, dividend, @@ -571,6 +586,10 @@ export class PortfolioService { }; } + console.timeEnd('-- PortfolioService.getDetails - 5'); + + console.time('-- PortfolioService.getDetails - 6'); + let summary: PortfolioSummary; if (withSummary) { @@ -589,6 +608,8 @@ export class PortfolioService { }); } + console.timeEnd('-- PortfolioService.getDetails - 6'); + return { accounts, hasErrors, @@ -1051,15 +1072,20 @@ export class PortfolioService { dateRange = 'max', filters, impersonationId, + portfolioCalculator, userId, withExcludedAccounts = false }: { dateRange?: DateRange; filters?: Filter[]; impersonationId: string; + portfolioCalculator?: PortfolioCalculator; userId: string; withExcludedAccounts?: boolean; }): Promise { + // OPTIMIZE (1.34s) + console.time('------ PortfolioService.getPerformance'); + userId = await this.getUserId(impersonationId, userId); const user = await this.userService.user({ id: userId }); const userCurrency = this.getUserCurrency(user); @@ -1096,6 +1122,8 @@ export class PortfolioService { const { endDate } = getInterval(dateRange); + console.time('------- PortfolioService.getPerformance - 2'); + const { activities } = await this.orderService.getOrders({ endDate, filters, @@ -1104,6 +1132,9 @@ export class PortfolioService { withExcludedAccounts }); + console.timeEnd('------- PortfolioService.getPerformance - 2'); + console.time('------- PortfolioService.getPerformance - 3'); + if (accountBalanceItems?.length <= 0 && activities?.length <= 0) { return { chart: [], @@ -1125,19 +1156,22 @@ export class PortfolioService { }; } - const portfolioCalculator = this.calculatorFactory.createCalculator({ - accountBalanceItems, - activities, - dateRange, - userId, - calculationType: PerformanceCalculationType.TWR, - currency: userCurrency, - hasFilters: filters?.length > 0, - isExperimentalFeatures: - this.request.user.Settings.settings.isExperimentalFeatures - }); + portfolioCalculator = + portfolioCalculator ?? + this.calculatorFactory.createCalculator({ + accountBalanceItems, + activities, + dateRange, + userId, + calculationType: PerformanceCalculationType.TWR, + currency: userCurrency, + hasFilters: filters?.length > 0, + isExperimentalFeatures: + this.request.user.Settings.settings.isExperimentalFeatures + }); const { + chartData, currentValueInBaseCurrency, errors, grossPerformance, @@ -1152,6 +1186,9 @@ export class PortfolioService { totalInvestment } = await portfolioCalculator.getSnapshot(); + console.timeEnd('------- PortfolioService.getPerformance - 3'); + console.time('------- PortfolioService.getPerformance - 4'); + let currentNetPerformance = netPerformance; let currentNetPerformancePercentage = netPerformancePercentage; @@ -1164,11 +1201,13 @@ export class PortfolioService { let currentNetWorth = 0; - const items = await portfolioCalculator.getChart({ + /*const items = await portfolioCalculator.getChart({ dateRange - }); + });*/ + console.timeEnd('------- PortfolioService.getPerformance - 4'); + console.time('------- PortfolioService.getPerformance - 5'); - const itemOfToday = items.find(({ date }) => { + const itemOfToday = chartData.find(({ date }) => { return date === format(new Date(), DATE_FORMAT); }); @@ -1190,11 +1229,15 @@ export class PortfolioService { currentNetWorth = itemOfToday.netWorth; } + console.timeEnd('------- PortfolioService.getPerformance - 5'); + + console.timeEnd('------ PortfolioService.getPerformance'); + return { errors, hasErrors, - chart: items, - firstOrderDate: parseDate(items[0]?.date), + chart: chartData, + firstOrderDate: parseDate(chartData[0]?.date), performance: { currentNetWorth, currentValueInBaseCurrency: currentValueInBaseCurrency.toNumber(), @@ -1603,15 +1646,23 @@ export class PortfolioService { userCurrency: string; userId: string; }): Promise { + // OPTIMIZE (1.1 s) + console.time('---- PortfolioService.getSummary'); + userId = await this.getUserId(impersonationId, userId); const user = await this.userService.user({ id: userId }); + console.time('----- PortfolioService.getSummary - 1'); + const { activities } = await this.orderService.getOrders({ userCurrency, userId, withExcludedAccounts: true }); + console.timeEnd('----- PortfolioService.getSummary - 1'); + console.time('----- PortfolioService.getSummary - 2'); + const excludedActivities: Activity[] = []; const nonExcludedActivities: Activity[] = []; @@ -1652,6 +1703,9 @@ export class PortfolioService { const interest = await portfolioCalculator.getInterestInBaseCurrency(); + console.timeEnd('----- PortfolioService.getSummary - 2'); + console.time('----- PortfolioService.getSummary - 3'); + const liabilities = await portfolioCalculator.getLiabilitiesInBaseCurrency(); @@ -1688,6 +1742,9 @@ export class PortfolioService { }) ); + console.timeEnd('----- PortfolioService.getSummary - 3'); + console.time('----- PortfolioService.getSummary - 4'); + const cashDetailsWithExcludedAccounts = await this.accountService.getCashDetails({ userId, @@ -1725,6 +1782,10 @@ export class PortfolioService { ) })?.toNumber(); + console.timeEnd('----- PortfolioService.getSummary - 4'); + + console.timeEnd('---- PortfolioService.getSummary'); + return { annualizedPerformancePercent, annualizedPerformancePercentWithCurrencyEffect, diff --git a/libs/common/src/lib/models/portfolio-snapshot.ts b/libs/common/src/lib/models/portfolio-snapshot.ts index 909f44f2a..868201951 100644 --- a/libs/common/src/lib/models/portfolio-snapshot.ts +++ b/libs/common/src/lib/models/portfolio-snapshot.ts @@ -1,14 +1,17 @@ import { transformToBig } from '@ghostfolio/common/class-transformer'; -import { UniqueAsset } from '@ghostfolio/common/interfaces'; +import { HistoricalDataItem, UniqueAsset } from '@ghostfolio/common/interfaces'; import { TimelinePosition } from '@ghostfolio/common/models'; import { Big } from 'big.js'; import { Transform, Type } from 'class-transformer'; export class PortfolioSnapshot { + chartData: HistoricalDataItem[]; + @Transform(transformToBig, { toClassOnly: true }) @Type(() => Big) currentValueInBaseCurrency: Big; + errors?: UniqueAsset[]; @Transform(transformToBig, { toClassOnly: true })