From 1713c4d972cf8d7914d30d0803cb0523252683a7 Mon Sep 17 00:00:00 2001 From: ceroma <678940+ceroma@users.noreply.github.com> Date: Tue, 1 Oct 2024 22:16:43 -0300 Subject: [PATCH] Feature/improve performance of portfolio snapshot computation - 3 This is similar to the previous diff. Here we can speed things up by using the dates in `chatDatesMap` instead of computing the whole interval every time. For reference, before this diff, the `while (isBefore(...))` loop was taking on average ~88ms per symbol for me (Raspberry Pi 4, 481 symbols in the DB). After this diff, it takes ~12ms per symbol, shaving off almost 40s of total computation time. Test Plan: Check that result of portfolio snapshot computation is the same before and after this diff: 1. Flush portfolio snapshot cache: ``` $ docker exec -it ghostfolio-redis-1 redis-cli --pass $REDIS_PASSWORD del "portfolio-snapshot-f9e4c63e-4b8e-46fc-b6ed-75ca0205f12b" ``` 2. Open Accounts to trigger snapshot calculation 3. Dump portfolio snapshot to a file: ``` $ docker exec -it ghostfolio-redis-1 redis-cli --pass $REDIS_PASSWORD --no-auth-warning get "portfolio-snapshot-f9e4c63e-4b8e-46fc-b6ed-75ca0205f12b" | python -c "import json, sys; json.dump(json.loads(json.loads(json.load(sys.stdin))), sys.stdout, indent=2, sort_keys=True)" > snapshot-before.json ``` 4. Apply this patch 5. Repeat steps 1-3 6. Compare results, check everything matches except for expiration time: ``` $ diff snapshot-before.json snapshot-after.json 2c2 < "expiration": 1727834561292, --- > "expiration": 1727836180162, ``` --- .../calculator/twr/portfolio-calculator.ts | 53 ++++++++++--------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts index f5e301cba..b26b6fac1 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts @@ -22,7 +22,7 @@ import { import { cloneDeep, first, last, sortBy } from 'lodash'; export class TWRPortfolioCalculator extends PortfolioCalculator { - private chartDatesDescending: string[]; + private chartDates: string[]; protected calculateOverallPerformance( positions: TimelinePosition[] @@ -228,11 +228,11 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { const dateOfFirstTransaction = new Date(first(orders).date); - const unitPriceAtStartDate = - marketSymbolMap[format(start, DATE_FORMAT)]?.[symbol]; + const endDateString = format(end, DATE_FORMAT); + const startDateString = format(start, DATE_FORMAT); - const unitPriceAtEndDate = - marketSymbolMap[format(end, DATE_FORMAT)]?.[symbol]; + const unitPriceAtStartDate = marketSymbolMap[startDateString]?.[symbol]; + const unitPriceAtEndDate = marketSymbolMap[endDateString]?.[symbol]; if ( !unitPriceAtEndDate || @@ -278,7 +278,7 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { // Add a synthetic order at the start and the end date orders.push({ - date: format(start, DATE_FORMAT), + date: startDateString, fee: new Big(0), feeInBaseCurrency: new Big(0), itemType: 'start', @@ -292,7 +292,7 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { }); orders.push({ - date: format(end, DATE_FORMAT), + date: endDateString, fee: new Big(0), feeInBaseCurrency: new Big(0), itemType: 'end', @@ -305,7 +305,6 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { unitPrice: unitPriceAtEndDate }); - let day = start; let lastUnitPrice: Big; const ordersByDate: { [date: string]: PortfolioOrderItem[] } = {}; @@ -315,15 +314,23 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { ordersByDate[order.date].push(order); } - while (isBefore(day, end)) { - const dateString = format(day, DATE_FORMAT); + if (!this.chartDates) { + this.chartDates = Object.keys(chartDateMap).sort(); + } + + for (const dateString of this.chartDates) { + if (dateString < startDateString) { + continue; + } else if (dateString > endDateString) { + break; + } if (ordersByDate[dateString]?.length > 0) { for (let order of ordersByDate[dateString]) { order.unitPriceFromMarketData = marketSymbolMap[dateString]?.[symbol] ?? lastUnitPrice; } - } else if (chartDateMap[dateString]) { + } else { orders.push({ date: dateString, fee: new Big(0), @@ -343,8 +350,6 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { const lastOrder = last(orders); lastUnitPrice = lastOrder.unitPriceFromMarketData ?? lastOrder.unitPrice; - - day = addDays(day, 1); } // Sort orders so that the start and end placeholder order are at the correct @@ -821,14 +826,14 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { startDate = start; } - const endDateString = format(endDate, DATE_FORMAT); - const startDateString = format(startDate, DATE_FORMAT); + const rangeEndDateString = format(endDate, DATE_FORMAT); + const rangeStartDateString = format(startDate, DATE_FORMAT); const currentValuesAtDateRangeStartWithCurrencyEffect = - currentValuesWithCurrencyEffect[startDateString] ?? new Big(0); + currentValuesWithCurrencyEffect[rangeStartDateString] ?? new Big(0); const investmentValuesAccumulatedAtStartDateWithCurrencyEffect = - investmentValuesAccumulatedWithCurrencyEffect[startDateString] ?? + investmentValuesAccumulatedWithCurrencyEffect[rangeStartDateString] ?? new Big(0); const grossPerformanceAtDateRangeStartWithCurrencyEffect = @@ -839,14 +844,12 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { let average = new Big(0); let dayCount = 0; - if (!this.chartDatesDescending) { - this.chartDatesDescending = Object.keys(chartDateMap).sort().reverse(); - } + for (let i = this.chartDates.length - 1; i >= 0; i -= 1) { + const date = this.chartDates[i]; - for (const date of this.chartDatesDescending) { - if (date > endDateString) { + if (date > rangeEndDateString) { continue; - } else if (date < startDateString) { + } else if (date < rangeStartDateString) { break; } @@ -869,13 +872,13 @@ export class TWRPortfolioCalculator extends PortfolioCalculator { } netPerformanceWithCurrencyEffectMap[dateRange] = - netPerformanceValuesWithCurrencyEffect[endDateString]?.minus( + netPerformanceValuesWithCurrencyEffect[rangeEndDateString]?.minus( // If the date range is 'max', take 0 as a start value. Otherwise, // the value of the end of the day of the start date is taken which // differs from the buying price. dateRange === 'max' ? new Big(0) - : (netPerformanceValuesWithCurrencyEffect[startDateString] ?? + : (netPerformanceValuesWithCurrencyEffect[rangeStartDateString] ?? new Big(0)) ) ?? new Big(0);