Integrate chart calculation into snapshot calculation

apps/api/src/app/order/order.service.ts

@@ -223,6 +223,7 @@ export class OrderService {
     userId: string;
     withExcludedAccounts?: boolean;
   }): Promise<Activities> {
+    console.time('------ OrderService.getOrders');
     let orderBy: Prisma.Enumerable<Prisma.OrderOrderByWithRelationInput> = [
       { date: 'asc' }
     ];
@@ -382,6 +383,8 @@ export class OrderService {
       };
     });
 
+    console.timeEnd('------ OrderService.getOrders');
+
     return { activities, count };
   }
 

apps/api/src/app/portfolio/calculator/portfolio-calculator.ts

@@ -69,6 +69,8 @@ export abstract class PortfolioCalculator {
     dateRange: DateRange;
     exchangeRateDataService: ExchangeRateDataService;
   }) {
+    console.time('--- PortfolioCalculator.constructor - 1');
+
     this.currency = currency;
     this.currentRateService = currentRateService;
     this.exchangeRateDataService = exchangeRateDataService;
@@ -95,9 +97,16 @@ export abstract class PortfolioCalculator {
     this.endDate = endDate;
     this.startDate = startDate;
 
+    console.timeEnd('--- PortfolioCalculator.constructor - 1');
+    console.time('--- PortfolioCalculator.constructor - 2');
+
     this.computeTransactionPoints();
 
+    console.timeEnd('--- PortfolioCalculator.constructor - 2');
+
+    console.time('--- PortfolioCalculator.constructor - 3');
     this.snapshotPromise = this.initialize();
+    console.timeEnd('--- PortfolioCalculator.constructor - 3');
   }
 
   protected abstract calculateOverallPerformance(
@@ -126,6 +135,7 @@ export abstract class PortfolioCalculator {
 
     if (!transactionPoints.length) {
       return {
+        chartData: [],
         currentValueInBaseCurrency: new Big(0),
         grossPerformance: new Big(0),
         grossPerformancePercentage: new Big(0),
@@ -247,6 +257,26 @@ export abstract class PortfolioCalculator {
 
     const endDateString = format(endDate, DATE_FORMAT);
 
+    const chartStartDate = this.getStartDate();
+    const daysInMarket = differenceInDays(endDate, chartStartDate) + 1;
+
+    const step = true /*withDataDecimation*/
+      ? Math.round(daysInMarket / Math.min(daysInMarket, MAX_CHART_ITEMS))
+      : 1;
+
+    let chartDates = eachDayOfInterval(
+      { start: chartStartDate, end },
+      { step }
+    ).map((date) => {
+      return resetHours(date);
+    });
+
+    const includesEndDate = isSameDay(last(chartDates), end);
+
+    if (!includesEndDate) {
+      chartDates.push(resetHours(end));
+    }
+
     if (firstIndex > 0) {
       firstIndex--;
     }
@@ -256,6 +286,34 @@ export abstract class PortfolioCalculator {
 
     const errors: ResponseError['errors'] = [];
 
+    const accumulatedValuesByDate: {
+      [date: string]: {
+        investmentValueWithCurrencyEffect: Big;
+        totalCurrentValue: Big;
+        totalCurrentValueWithCurrencyEffect: Big;
+        totalInvestmentValue: Big;
+        totalInvestmentValueWithCurrencyEffect: Big;
+        totalNetPerformanceValue: Big;
+        totalNetPerformanceValueWithCurrencyEffect: Big;
+        totalTimeWeightedInvestmentValue: Big;
+        totalTimeWeightedInvestmentValueWithCurrencyEffect: Big;
+      };
+    } = {};
+
+    const valuesBySymbol: {
+      [symbol: string]: {
+        currentValues: { [date: string]: Big };
+        currentValuesWithCurrencyEffect: { [date: string]: Big };
+        investmentValuesAccumulated: { [date: string]: Big };
+        investmentValuesAccumulatedWithCurrencyEffect: { [date: string]: Big };
+        investmentValuesWithCurrencyEffect: { [date: string]: Big };
+        netPerformanceValues: { [date: string]: Big };
+        netPerformanceValuesWithCurrencyEffect: { [date: string]: Big };
+        timeWeightedInvestmentValues: { [date: string]: Big };
+        timeWeightedInvestmentValuesWithCurrencyEffect: { [date: string]: Big };
+      };
+    } = {};
+
     for (const item of lastTransactionPoint.items) {
       const marketPriceInBaseCurrency = (
         marketSymbolMap[endDateString]?.[item.symbol] ?? item.averagePrice
@@ -266,16 +324,25 @@ export abstract class PortfolioCalculator {
       );
 
       const {
+        currentValues,
+        currentValuesWithCurrencyEffect,
         grossPerformance,
         grossPerformancePercentage,
         grossPerformancePercentageWithCurrencyEffect,
         grossPerformanceWithCurrencyEffect,
         hasErrors,
+        investmentValuesAccumulated,
+        investmentValuesAccumulatedWithCurrencyEffect,
+        investmentValuesWithCurrencyEffect,
         netPerformance,
         netPerformancePercentage,
         netPerformancePercentageWithCurrencyEffect,
+        netPerformanceValues,
+        netPerformanceValuesWithCurrencyEffect,
         netPerformanceWithCurrencyEffect,
         timeWeightedInvestment,
+        timeWeightedInvestmentValues,
+        timeWeightedInvestmentValuesWithCurrencyEffect,
         timeWeightedInvestmentWithCurrencyEffect,
         totalDividend,
         totalDividendInBaseCurrency,
@@ -287,15 +354,29 @@ export abstract class PortfolioCalculator {
       } = this.getSymbolMetrics({
         marketSymbolMap,
         start,
+        step,
         dataSource: item.dataSource,
         end: endDate,
         exchangeRates:
           exchangeRatesByCurrency[`${item.currency}${this.currency}`],
+        isChartMode: true,
         symbol: item.symbol
       });
 
       hasAnySymbolMetricsErrors = hasAnySymbolMetricsErrors || hasErrors;
 
+      valuesBySymbol[item.symbol] = {
+        currentValues,
+        currentValuesWithCurrencyEffect,
+        investmentValuesAccumulated,
+        investmentValuesAccumulatedWithCurrencyEffect,
+        investmentValuesWithCurrencyEffect,
+        netPerformanceValues,
+        netPerformanceValuesWithCurrencyEffect,
+        timeWeightedInvestmentValues,
+        timeWeightedInvestmentValuesWithCurrencyEffect
+      };
+
       positions.push({
         dividend: totalDividend,
         dividendInBaseCurrency: totalDividendInBaseCurrency,
@@ -363,10 +444,143 @@ export abstract class PortfolioCalculator {
       }
     }
 
+    for (const currentDate of chartDates) {
+      const dateString = format(currentDate, DATE_FORMAT);
+
+      for (const symbol of Object.keys(valuesBySymbol)) {
+        const symbolValues = valuesBySymbol[symbol];
+
+        const currentValue =
+          symbolValues.currentValues?.[dateString] ?? new Big(0);
+
+        const currentValueWithCurrencyEffect =
+          symbolValues.currentValuesWithCurrencyEffect?.[dateString] ??
+          new Big(0);
+
+        const investmentValueAccumulated =
+          symbolValues.investmentValuesAccumulated?.[dateString] ?? new Big(0);
+
+        const investmentValueAccumulatedWithCurrencyEffect =
+          symbolValues.investmentValuesAccumulatedWithCurrencyEffect?.[
+            dateString
+          ] ?? new Big(0);
+
+        const investmentValueWithCurrencyEffect =
+          symbolValues.investmentValuesWithCurrencyEffect?.[dateString] ??
+          new Big(0);
+
+        const netPerformanceValue =
+          symbolValues.netPerformanceValues?.[dateString] ?? new Big(0);
+
+        const netPerformanceValueWithCurrencyEffect =
+          symbolValues.netPerformanceValuesWithCurrencyEffect?.[dateString] ??
+          new Big(0);
+
+        const timeWeightedInvestmentValue =
+          symbolValues.timeWeightedInvestmentValues?.[dateString] ?? new Big(0);
+
+        const timeWeightedInvestmentValueWithCurrencyEffect =
+          symbolValues.timeWeightedInvestmentValuesWithCurrencyEffect?.[
+            dateString
+          ] ?? new Big(0);
+
+        accumulatedValuesByDate[dateString] = {
+          investmentValueWithCurrencyEffect: (
+            accumulatedValuesByDate[dateString]
+              ?.investmentValueWithCurrencyEffect ?? new Big(0)
+          ).add(investmentValueWithCurrencyEffect),
+          totalCurrentValue: (
+            accumulatedValuesByDate[dateString]?.totalCurrentValue ?? new Big(0)
+          ).add(currentValue),
+          totalCurrentValueWithCurrencyEffect: (
+            accumulatedValuesByDate[dateString]
+              ?.totalCurrentValueWithCurrencyEffect ?? new Big(0)
+          ).add(currentValueWithCurrencyEffect),
+          totalInvestmentValue: (
+            accumulatedValuesByDate[dateString]?.totalInvestmentValue ??
+            new Big(0)
+          ).add(investmentValueAccumulated),
+          totalInvestmentValueWithCurrencyEffect: (
+            accumulatedValuesByDate[dateString]
+              ?.totalInvestmentValueWithCurrencyEffect ?? new Big(0)
+          ).add(investmentValueAccumulatedWithCurrencyEffect),
+          totalNetPerformanceValue: (
+            accumulatedValuesByDate[dateString]?.totalNetPerformanceValue ??
+            new Big(0)
+          ).add(netPerformanceValue),
+          totalNetPerformanceValueWithCurrencyEffect: (
+            accumulatedValuesByDate[dateString]
+              ?.totalNetPerformanceValueWithCurrencyEffect ?? new Big(0)
+          ).add(netPerformanceValueWithCurrencyEffect),
+          totalTimeWeightedInvestmentValue: (
+            accumulatedValuesByDate[dateString]
+              ?.totalTimeWeightedInvestmentValue ?? new Big(0)
+          ).add(timeWeightedInvestmentValue),
+          totalTimeWeightedInvestmentValueWithCurrencyEffect: (
+            accumulatedValuesByDate[dateString]
+              ?.totalTimeWeightedInvestmentValueWithCurrencyEffect ?? new Big(0)
+          ).add(timeWeightedInvestmentValueWithCurrencyEffect)
+        };
+      }
+    }
+
+    const chartData: HistoricalDataItem[] = Object.entries(
+      accumulatedValuesByDate
+    ).map(([date, values]) => {
+      const {
+        investmentValueWithCurrencyEffect,
+        totalCurrentValue,
+        totalCurrentValueWithCurrencyEffect,
+        totalInvestmentValue,
+        totalInvestmentValueWithCurrencyEffect,
+        totalNetPerformanceValue,
+        totalNetPerformanceValueWithCurrencyEffect,
+        totalTimeWeightedInvestmentValue,
+        totalTimeWeightedInvestmentValueWithCurrencyEffect
+      } = values;
+
+      console.log(
+        'Chart: totalTimeWeightedInvestmentValue',
+        totalTimeWeightedInvestmentValue.toFixed()
+      );
+
+      const netPerformanceInPercentage = totalTimeWeightedInvestmentValue.eq(0)
+        ? 0
+        : totalNetPerformanceValue
+            .div(totalTimeWeightedInvestmentValue)
+            .mul(100)
+            .toNumber();
+
+      const netPerformanceInPercentageWithCurrencyEffect =
+        totalTimeWeightedInvestmentValueWithCurrencyEffect.eq(0)
+          ? 0
+          : totalNetPerformanceValueWithCurrencyEffect
+              .div(totalTimeWeightedInvestmentValueWithCurrencyEffect)
+              .mul(100)
+              .toNumber();
+
+      return {
+        date,
+        netPerformanceInPercentage,
+        netPerformanceInPercentageWithCurrencyEffect,
+        investmentValueWithCurrencyEffect:
+          investmentValueWithCurrencyEffect.toNumber(),
+        netPerformance: totalNetPerformanceValue.toNumber(),
+        netPerformanceWithCurrencyEffect:
+          totalNetPerformanceValueWithCurrencyEffect.toNumber(),
+        totalInvestment: totalInvestmentValue.toNumber(),
+        totalInvestmentValueWithCurrencyEffect:
+          totalInvestmentValueWithCurrencyEffect.toNumber(),
+        value: totalCurrentValue.toNumber(),
+        valueWithCurrencyEffect: totalCurrentValueWithCurrencyEffect.toNumber()
+      };
+    });
+
     const overall = this.calculateOverallPerformance(positions);
 
     return {
       ...overall,
+      chartData,
       errors,
       positions,
       totalInterestWithCurrencyEffect,
@@ -383,6 +597,8 @@ export abstract class PortfolioCalculator {
     dateRange?: DateRange;
     withDataDecimation?: boolean;
   }): Promise<HistoricalDataItem[]> {
+    console.time('-------- PortfolioCalculator.getChart');
+
     if (this.getTransactionPoints().length === 0) {
       return [];
     }
@@ -394,11 +610,15 @@ export abstract class PortfolioCalculator {
       ? Math.round(daysInMarket / Math.min(daysInMarket, MAX_CHART_ITEMS))
       : 1;
 
-    return this.getChartData({
+    const chartData = await this.getChartData({
       step,
       end: endDate,
       start: startDate
     });
+
+    console.timeEnd('-------- PortfolioCalculator.getChart');
+
+    return chartData;
   }
 
   public async getChartData({
@@ -637,6 +857,11 @@ export abstract class PortfolioCalculator {
         totalTimeWeightedInvestmentValueWithCurrencyEffect
       } = values;
 
+      console.log(
+        'Chart: totalTimeWeightedInvestmentValue',
+        totalTimeWeightedInvestmentValue.toFixed()
+      );
+
       const netPerformanceInPercentage = totalTimeWeightedInvestmentValue.eq(0)
         ? 0
         : totalNetPerformanceValue
@@ -743,8 +968,11 @@ export abstract class PortfolioCalculator {
   }
 
   public async getSnapshot() {
+    console.time('getSnapshot');
     await this.snapshotPromise;
 
+    console.timeEnd('getSnapshot');
+
     return this.snapshot;
   }
 

apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts

@@ -102,6 +102,11 @@ export class TWRPortfolioCalculator extends PortfolioCalculator {
       }
     }
 
+    console.log(
+      'Overall: totalTimeWeightedInvestmentValue',
+      totalTimeWeightedInvestment.toFixed()
+    );
+
     return {
       currentValueInBaseCurrency,
       grossPerformance,
@@ -114,6 +119,7 @@ export class TWRPortfolioCalculator extends PortfolioCalculator {
       totalInterestWithCurrencyEffect,
       totalInvestment,
       totalInvestmentWithCurrencyEffect,
+      chartData: [],
       netPerformancePercentage: totalTimeWeightedInvestment.eq(0)
         ? new Big(0)
         : netPerformance.div(totalTimeWeightedInvestment),

apps/api/src/app/portfolio/interfaces/portfolio-snapshot.interface.ts

@@ -1,8 +1,13 @@
-import { ResponseError, TimelinePosition } from '@ghostfolio/common/interfaces';
+import {
+  HistoricalDataItem,
+  ResponseError,
+  TimelinePosition
+} from '@ghostfolio/common/interfaces';
 
 import { Big } from 'big.js';
 
 export interface PortfolioSnapshot extends ResponseError {
+  chartData: HistoricalDataItem[];
   currentValueInBaseCurrency: Big;
   grossPerformance: Big;
   grossPerformanceWithCurrencyEffect: Big;

apps/api/src/app/portfolio/portfolio.controller.ts

@@ -80,6 +80,8 @@ export class PortfolioController {
     @Query('tags') filterByTags?: string,
     @Query('withMarkets') withMarketsParam = 'false'
   ): Promise<PortfolioDetails & { hasError: boolean }> {
+    console.time('TOTAL');
+
     const withMarkets = withMarketsParam === 'true';
 
     let hasDetails = true;
@@ -100,6 +102,8 @@ export class PortfolioController {
       filterByTags
     });
 
+    console.time('- PortfolioController.getDetails - 1');
+
     const { accounts, hasErrors, holdings, platforms, summary } =
       await this.portfolioService.getDetails({
         dateRange,
@@ -110,6 +114,10 @@ export class PortfolioController {
         withSummary: true
       });
 
+    console.timeEnd('- PortfolioController.getDetails - 1');
+
+    console.time('- PortfolioController.getDetails - 2');
+
     if (hasErrors || hasNotDefinedValuesInObject(holdings)) {
       hasError = true;
     }
@@ -202,6 +210,10 @@ export class PortfolioController {
       };
     }
 
+    console.timeEnd('- PortfolioController.getDetails - 2');
+
+    console.timeEnd('TOTAL');
+
     return {
       accounts,
       hasError,

apps/api/src/app/portfolio/portfolio.service.ts

@@ -334,6 +334,8 @@ export class PortfolioService {
     withMarkets?: boolean;
     withSummary?: boolean;
   }): Promise<PortfolioDetails & { hasErrors: boolean }> {
+    console.time('-- PortfolioService.getDetails - 1');
+
     userId = await this.getUserId(impersonationId, userId);
     const user = await this.userService.user({ id: userId });
     const userCurrency = this.getUserCurrency(user);
@@ -356,9 +358,16 @@ export class PortfolioService {
       currency: userCurrency
     });
 
+    console.timeEnd('-- PortfolioService.getDetails - 1');
+
+    console.time('-- PortfolioService.getDetails - 2');
+
     const { currentValueInBaseCurrency, hasErrors, positions } =
       await portfolioCalculator.getSnapshot();
 
+    console.timeEnd('-- PortfolioService.getDetails - 2');
+    console.time('-- PortfolioService.getDetails - 3');
+
     const cashDetails = await this.accountService.getCashDetails({
       filters,
       userId,
@@ -407,6 +416,9 @@ export class PortfolioService {
       };
     });
 
+    console.timeEnd('-- PortfolioService.getDetails - 3');
+    console.time('-- PortfolioService.getDetails - 4');
+
     const [dataProviderResponses, symbolProfiles] = await Promise.all([
       this.dataProviderService.getQuotes({ user, items: dataGatheringItems }),
       this.symbolProfileService.getSymbolProfiles(dataGatheringItems)
@@ -422,6 +434,9 @@ export class PortfolioService {
       portfolioItemsNow[position.symbol] = position;
     }
 
+    console.timeEnd('-- PortfolioService.getDetails - 4');
+    console.time('-- PortfolioService.getDetails - 5');
+
     for (const {
       currency,
       dividend,
@@ -562,6 +577,10 @@ export class PortfolioService {
       };
     }
 
+    console.timeEnd('-- PortfolioService.getDetails - 5');
+
+    console.time('-- PortfolioService.getDetails - 6');
+
     let summary: PortfolioSummary;
 
     if (withSummary) {
@@ -580,6 +599,8 @@ export class PortfolioService {
       });
     }
 
+    console.timeEnd('-- PortfolioService.getDetails - 6');
+
     return {
       accounts,
       hasErrors,
@@ -1022,15 +1043,20 @@ export class PortfolioService {
     dateRange = 'max',
     filters,
     impersonationId,
+    portfolioCalculator,
     userId,
     withExcludedAccounts = false
   }: {
     dateRange?: DateRange;
     filters?: Filter[];
     impersonationId: string;
+    portfolioCalculator?: PortfolioCalculator;
     userId: string;
     withExcludedAccounts?: boolean;
   }): Promise<PortfolioPerformanceResponse> {
+    // OPTIMIZE (1.34s)
+    console.time('------ PortfolioService.getPerformance');
+
     userId = await this.getUserId(impersonationId, userId);
     const user = await this.userService.user({ id: userId });
     const userCurrency = this.getUserCurrency(user);
@@ -1062,6 +1088,8 @@ export class PortfolioService {
 
     const { endDate, startDate } = getInterval(dateRange);
 
+    console.time('------- PortfolioService.getPerformance - 2');
+
     const { activities } = await this.orderService.getOrders({
       endDate,
       filters,
@@ -1070,6 +1098,9 @@ export class PortfolioService {
       withExcludedAccounts
     });
 
+    console.timeEnd('------- PortfolioService.getPerformance - 2');
+    console.time('------- PortfolioService.getPerformance - 3');
+
     if (accountBalanceItems?.length <= 0 && activities?.length <= 0) {
       return {
         chart: [],
@@ -1091,14 +1122,17 @@ export class PortfolioService {
       };
     }
 
-    const portfolioCalculator = this.calculatorFactory.createCalculator({
+    portfolioCalculator =
-      activities,
+      portfolioCalculator ??
-      dateRange,
+      this.calculatorFactory.createCalculator({
-      calculationType: PerformanceCalculationType.TWR,
+        activities,
-      currency: userCurrency
+        dateRange,
-    });
+        calculationType: PerformanceCalculationType.TWR,
+        currency: userCurrency
+      });
 
     const {
+      chartData,
       currentValueInBaseCurrency,
       errors,
       grossPerformance,
@@ -1113,6 +1147,9 @@ export class PortfolioService {
       totalInvestment
     } = await portfolioCalculator.getSnapshot();
 
+    console.timeEnd('------- PortfolioService.getPerformance - 3');
+    console.time('------- PortfolioService.getPerformance - 4');
+
     let currentNetPerformance = netPerformance;
 
     let currentNetPerformancePercent = netPerformancePercentage;
@@ -1123,11 +1160,10 @@ export class PortfolioService {
     let currentNetPerformanceWithCurrencyEffect =
       netPerformanceWithCurrencyEffect;
 
-    const items = await portfolioCalculator.getChart({
+    console.timeEnd('------- PortfolioService.getPerformance - 4');
-      dateRange
+    console.time('------- PortfolioService.getPerformance - 5');
-    });
 
-    const itemOfToday = items.find(({ date }) => {
+    const itemOfToday = chartData.find(({ date }) => {
       return date === format(new Date(), DATE_FORMAT);
     });
 
@@ -1162,19 +1198,25 @@ export class PortfolioService {
       });
     }
 
+    console.timeEnd('------- PortfolioService.getPerformance - 5');
+    console.time('------- PortfolioService.getPerformance - 6');
+
     const mergedHistoricalDataItems = this.mergeHistoricalDataItems(
       accountBalanceItems,
-      items
+      chartData
     );
 
     const currentHistoricalDataItem = last(mergedHistoricalDataItems);
     const currentNetWorth = currentHistoricalDataItem?.netWorth ?? 0;
 
+    console.timeEnd('------- PortfolioService.getPerformance - 6');
+    console.timeEnd('------ PortfolioService.getPerformance');
+
     return {
       errors,
       hasErrors,
       chart: mergedHistoricalDataItems,
-      firstOrderDate: parseDate(items[0]?.date),
+      firstOrderDate: parseDate(chartData[0]?.date),
       performance: {
         currentNetWorth,
         currentGrossPerformance: grossPerformance.toNumber(),
@@ -1579,11 +1621,17 @@ export class PortfolioService {
     userCurrency: string;
     userId: string;
   }): Promise<PortfolioSummary> {
+    // OPTIMIZE (1.1 s)
+    console.time('---- PortfolioService.getSummary');
+
     userId = await this.getUserId(impersonationId, userId);
     const user = await this.userService.user({ id: userId });
 
+    console.time('----- PortfolioService.getSummary - 1');
+
     const performanceInformation = await this.getPerformance({
       impersonationId,
+      portfolioCalculator,
       userId
     });
 
@@ -1593,6 +1641,9 @@ export class PortfolioService {
       withExcludedAccounts: true
     });
 
+    console.timeEnd('----- PortfolioService.getSummary - 1');
+    console.time('----- PortfolioService.getSummary - 2');
+
     const excludedActivities: Activity[] = [];
     const nonExcludedActivities: Activity[] = [];
 
@@ -1620,6 +1671,9 @@ export class PortfolioService {
 
     const interest = await portfolioCalculator.getInterestInBaseCurrency();
 
+    console.timeEnd('----- PortfolioService.getSummary - 2');
+    console.time('----- PortfolioService.getSummary - 3');
+
     const liabilities =
       await portfolioCalculator.getLiabilitiesInBaseCurrency();
 
@@ -1656,6 +1710,9 @@ export class PortfolioService {
       })
     );
 
+    console.timeEnd('----- PortfolioService.getSummary - 3');
+    console.time('----- PortfolioService.getSummary - 4');
+
     const cashDetailsWithExcludedAccounts =
       await this.accountService.getCashDetails({
         userId,
@@ -1695,6 +1752,10 @@ export class PortfolioService {
         )
       })?.toNumber();
 
+    console.timeEnd('----- PortfolioService.getSummary - 4');
+
+    console.timeEnd('---- PortfolioService.getSummary');
+
     return {
       ...performanceInformation.performance,
       annualizedPerformancePercent,