Merge pull request #88 from dandevaud/bugfix/fix-marketdata-update

Bugfix/fix marketdata update

apps/api/src/app/portfolio/calculator/constantPortfolioReturn/portfolio-calculator.ts

@@ -1,14 +1,25 @@
-import { getFactor } from '@ghostfolio/api/helper/portfolio.helper';
-import { DATE_FORMAT, parseDate, resetHours } from '@ghostfolio/common/helper';
+import { LogPerformance } from '@ghostfolio/api/aop/logging.interceptor';
 import {
-  HistoricalDataItem,
-  SymbolMetrics,
-  UniqueAsset
-} from '@ghostfolio/common/interfaces';
-import { PortfolioSnapshot, TimelinePosition } from '@ghostfolio/common/models';
+  getFactor,
+  getInterval
+} from '@ghostfolio/api/helper/portfolio.helper';
+import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
+import { MAX_CHART_ITEMS } from '@ghostfolio/common/config';
+import { DATE_FORMAT, parseDate, resetHours } from '@ghostfolio/common/helper';
+import { HistoricalDataItem } from '@ghostfolio/common/interfaces';
+import { DateRange } from '@ghostfolio/common/types';
 
 import { Big } from 'big.js';
-import { addDays, eachDayOfInterval, format } from 'date-fns';
+import {
+  addDays,
+  differenceInDays,
+  eachDayOfInterval,
+  format,
+  isAfter,
+  isBefore,
+  isEqual,
+  subDays
+} from 'date-fns';
 
 import { PortfolioOrder } from '../../interfaces/portfolio-order.interface';
 import { TWRPortfolioCalculator } from '../twr/portfolio-calculator';
@@ -17,44 +28,58 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
   private holdings: { [date: string]: { [symbol: string]: Big } } = {};
   private holdingCurrencies: { [symbol: string]: string } = {};
 
-  protected calculateOverallPerformance(
-    positions: TimelinePosition[]
-  ): PortfolioSnapshot {
-    return super.calculateOverallPerformance(positions);
-  }
-
-  protected getSymbolMetrics({
-    dataSource,
-    end,
-    exchangeRates,
-    isChartMode = false,
-    marketSymbolMap,
-    start,
-    step = 1,
-    symbol
+  @LogPerformance
+  public async getChart({
+    dateRange = 'max',
+    withDataDecimation = true,
+    withTimeWeightedReturn = false
   }: {
-    end: Date;
-    exchangeRates: { [dateString: string]: number };
-    isChartMode?: boolean;
-    marketSymbolMap: {
-      [date: string]: { [symbol: string]: Big };
-    };
-    start: Date;
-    step?: number;
-  } & UniqueAsset): SymbolMetrics {
-    return super.getSymbolMetrics({
-      dataSource,
-      end,
-      exchangeRates,
-      isChartMode,
-      marketSymbolMap,
-      start,
+    dateRange?: DateRange;
+    withDataDecimation?: boolean;
+    withTimeWeightedReturn?: boolean;
+  }): Promise<HistoricalDataItem[]> {
+    const { endDate, startDate } = getInterval(dateRange, this.getStartDate());
+
+    const daysInMarket = differenceInDays(endDate, startDate) + 1;
+    const step = withDataDecimation
+      ? Math.round(daysInMarket / Math.min(daysInMarket, MAX_CHART_ITEMS))
+      : 1;
+
+    let item = super.getChartData({
       step,
-      symbol
+      end: endDate,
+      start: startDate
     });
+
+    if (!withTimeWeightedReturn) {
+      return item;
+    } else {
+      let timeWeighted = await this.getTimeWeightedChartData({
+        step,
+        end: endDate,
+        start: startDate
+      });
+
+      return item.then((data) => {
+        return data.map((item) => {
+          let timeWeightedItem = timeWeighted.find(
+            (timeWeightedItem) => timeWeightedItem.date === item.date
+          );
+          if (timeWeightedItem) {
+            item.timeWeightedPerformance =
+              timeWeightedItem.netPerformanceInPercentage;
+            item.timeWeightedPerformanceWithCurrencyEffect =
+              timeWeightedItem.netPerformanceInPercentageWithCurrencyEffect;
+          }
+
+          return item;
+        });
+      });
+    }
   }
 
-  public override async getChartData({
+  @LogPerformance
+  private async getTimeWeightedChartData({
     end = new Date(Date.now()),
     start,
     step = 1
@@ -63,13 +88,18 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     start: Date;
     step?: number;
   }): Promise<HistoricalDataItem[]> {
-    const timelineHoldings = this.getHoldings(start, end);
+    let marketMapTask = this.computeMarketMap({ in: [start, end] });
+    const timelineHoldings = await this.getHoldings(start, end);
+
     const calculationDates = Object.keys(timelineHoldings)
       .filter((date) => {
         let parsed = parseDate(date);
-        parsed >= start && parsed <= end;
+        return (
+          isAfter(parsed, subDays(start, 1)) &&
+          isBefore(parsed, addDays(end, 1))
+        );
       })
-      .sort();
+      .sort((a, b) => parseDate(a).getTime() - parseDate(b).getTime());
     let data: HistoricalDataItem[] = [];
     const startString = format(start, DATE_FORMAT);
 
@@ -88,11 +118,13 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
       valueWithCurrencyEffect: 0
     });
 
+    await marketMapTask;
+
     let totalInvestment = Object.keys(timelineHoldings[startString]).reduce(
       (sum, holding) => {
         return sum.plus(
           timelineHoldings[startString][holding].mul(
-            this.marketMap[startString][holding]
+            this.marketMap[startString][holding] ?? new Big(0)
           )
         );
       },
@@ -149,6 +181,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     return data;
   }
 
+  @LogPerformance
   private async handleSingleHolding(
     previousDate: string,
     holding: string,
@@ -203,6 +236,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     };
   }
 
+  @LogPerformance
   private getCurrency(symbol: string) {
     if (!this.holdingCurrencies[symbol]) {
       this.holdingCurrencies[symbol] = this.activities.find(
@@ -213,11 +247,16 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     return this.holdingCurrencies[symbol];
   }
 
-  private getHoldings(start: Date, end: Date) {
+  @LogPerformance
+  private async getHoldings(start: Date, end: Date) {
     if (
       this.holdings &&
-      Object.keys(this.holdings).some((h) => parseDate(h) >= end) &&
-      Object.keys(this.holdings).some((h) => parseDate(h) <= start)
+      Object.keys(this.holdings).some((h) =>
+        isAfter(parseDate(h), subDays(end, 1))
+      ) &&
+      Object.keys(this.holdings).some((h) =>
+        isBefore(parseDate(h), addDays(start, 1))
+      )
     ) {
       return this.holdings;
     }
@@ -226,7 +265,8 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     return this.holdings;
   }
 
-  private computeHoldings(start: Date, end: Date) {
+  @LogPerformance
+  private async computeHoldings(start: Date, end: Date) {
     const investmentByDate = this.getInvestmentByDate();
     const transactionDates = Object.keys(investmentByDate).sort();
     let dates = eachDayOfInterval({ start, end }, { step: 1 })
@@ -258,6 +298,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     this.holdings = currentHoldings;
   }
 
+  @LogPerformance
   private calculateInitialHoldings(
     investmentByDate: { [date: string]: PortfolioOrder[] },
     start: Date,
@@ -288,6 +329,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     }
   }
 
+  @LogPerformance
   private getInvestmentByDate(): { [date: string]: PortfolioOrder[] } {
     return this.activities.reduce((groupedByDate, order) => {
       if (!groupedByDate[order.date]) {
@@ -299,4 +341,40 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
       return groupedByDate;
     }, {});
   }
+
+  @LogPerformance
+  private async computeMarketMap(dateQuery: { in: Date[] }) {
+    const dataGatheringItems: IDataGatheringItem[] = this.activities.map(
+      (activity) => {
+        return {
+          symbol: activity.SymbolProfile.symbol,
+          dataSource: activity.SymbolProfile.dataSource
+        };
+      }
+    );
+    const { values: marketSymbols } = await this.currentRateService.getValues({
+      dataGatheringItems,
+      dateQuery
+    });
+
+    const marketSymbolMap: {
+      [date: string]: { [symbol: string]: Big };
+    } = {};
+
+    for (const marketSymbol of marketSymbols) {
+      const date = format(marketSymbol.date, DATE_FORMAT);
+
+      if (!marketSymbolMap[date]) {
+        marketSymbolMap[date] = {};
+      }
+
+      if (marketSymbol.marketPrice) {
+        marketSymbolMap[date][marketSymbol.symbol] = new Big(
+          marketSymbol.marketPrice
+        );
+      }
+    }
+
+    this.marketMap = marketSymbolMap;
+  }
 }

apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts

@@ -64,7 +64,7 @@ export class PortfolioCalculatorFactory {
           redisCacheService: this.redisCacheService
         });
       case PerformanceCalculationType.TWR:
-        return new TWRPortfolioCalculator({
+        return new CPRPortfolioCalculator({
           accountBalanceItems,
           activities,
           currency,

apps/api/src/app/portfolio/calculator/portfolio-calculator.ts

@@ -54,7 +54,7 @@ export abstract class PortfolioCalculator {
 
   private configurationService: ConfigurationService;
   protected currency: string;
-  private currentRateService: CurrentRateService;
+  protected currentRateService: CurrentRateService;
   private dataProviderInfos: DataProviderInfo[];
   private dateRange: DateRange;
   private endDate: Date;
@@ -424,10 +424,12 @@ export abstract class PortfolioCalculator {
 
   public async getChart({
     dateRange = 'max',
-    withDataDecimation = true
+    withDataDecimation = true,
+    withTimeWeightedReturn = false
   }: {
     dateRange?: DateRange;
     withDataDecimation?: boolean;
+    withTimeWeightedReturn?: boolean;
   }): Promise<HistoricalDataItem[]> {
     const { endDate, startDate } = getInterval(dateRange, this.getStartDate());
 

apps/api/src/app/portfolio/portfolio.service.ts

@@ -1282,18 +1282,9 @@ export class PortfolioService {
     let currentNetWorth = 0;
 
     let items = await portfolioCalculator.getChart({
-      dateRange
-    });
-
-    items = await this.calculatedTimeWeightedPerformance(
-      calculateTimeWeightedPerformance,
-      activities,
       dateRange,
-      userId,
-      userCurrency,
-      filters,
-      items
-    );
+      withTimeWeightedReturn: calculateTimeWeightedPerformance
+    });
 
     const itemOfToday = items.find(({ date }) => {
       return date === format(new Date(), DATE_FORMAT);
@@ -1342,44 +1333,6 @@ export class PortfolioService {
     };
   }
 
-  private async calculatedTimeWeightedPerformance(
-    calculateTimeWeightedPerformance: boolean,
-    activities: Activity[],
-    dateRange: string,
-    userId: string,
-    userCurrency: string,
-    filters: Filter[],
-    items: HistoricalDataItem[]
-  ) {
-    if (calculateTimeWeightedPerformance) {
-      const portfolioCalculatorCPR = this.calculatorFactory.createCalculator({
-        activities,
-        dateRange,
-        userId,
-        calculationType: PerformanceCalculationType.CPR,
-        currency: userCurrency,
-        hasFilters: filters?.length > 0,
-        isExperimentalFeatures:
-          this.request.user.Settings.settings.isExperimentalFeatures
-      });
-      let timeWeightedInvestmentItems = await portfolioCalculatorCPR.getChart({
-        dateRange
-      });
-
-      items = items.map((item) => {
-        let matchingItem = timeWeightedInvestmentItems.find(
-          (timeWeightedInvestmentItem) =>
-            timeWeightedInvestmentItem.date === item.date
-        );
-        item.timeWeightedPerformance = matchingItem.netPerformanceInPercentage;
-        item.timeWeightedPerformanceWithCurrencyEffect =
-          matchingItem.netPerformanceInPercentageWithCurrencyEffect;
-        return item;
-      });
-    }
-    return items;
-  }
-
   @LogPerformance
   public async getReport(impersonationId: string): Promise<PortfolioReport> {
     const userId = await this.getUserId(impersonationId, this.request.user.id);

apps/api/src/services/data-gathering/data-gathering.service.ts

@@ -117,21 +117,16 @@ export class DataGatheringService {
         historicalData[symbol][format(date, DATE_FORMAT)].marketPrice;
 
       if (marketPrice) {
-        await this.lock.acquireAsync();
-        try {
-          return await this.prismaService.marketData.upsert({
-            create: {
-              dataSource,
-              date,
-              marketPrice,
-              symbol
-            },
-            update: { marketPrice },
-            where: { dataSource_date_symbol: { dataSource, date, symbol } }
-          });
-        } finally {
-          this.lock.release();
-        }
+        return await this.prismaService.marketData.upsert({
+          create: {
+            dataSource,
+            date,
+            marketPrice,
+            symbol
+          },
+          update: { marketPrice },
+          where: { dataSource_date_symbol: { dataSource, date, symbol } }
+        });
       }
     } catch (error) {
       Logger.error(error, 'DataGatheringService');

apps/api/src/services/market-data/market-data.service.ts

@@ -124,22 +124,17 @@ export class MarketDataService {
     where: Prisma.MarketDataWhereUniqueInput;
   }): Promise<MarketData> {
     const { data, where } = params;
-    await this.lock.acquireAsync();
-    try {
-      return this.prismaService.marketData.upsert({
-        where,
-        create: {
-          dataSource: where.dataSource_date_symbol.dataSource,
-          date: where.dataSource_date_symbol.date,
-          marketPrice: data.marketPrice,
-          state: data.state,
-          symbol: where.dataSource_date_symbol.symbol
-        },
-        update: { marketPrice: data.marketPrice, state: data.state }
-      });
-    } finally {
-      this.lock.release();
-    }
+    return this.prismaService.marketData.upsert({
+      where,
+      create: {
+        dataSource: where.dataSource_date_symbol.dataSource,
+        date: where.dataSource_date_symbol.date,
+        marketPrice: data.marketPrice,
+        state: data.state,
+        symbol: where.dataSource_date_symbol.symbol
+      },
+      update: { marketPrice: data.marketPrice, state: data.state }
+    });
   }
 
   /**
@@ -153,31 +148,26 @@ export class MarketDataService {
   }): Promise<MarketData[]> {
     const upsertPromises = data.map(
       async ({ dataSource, date, marketPrice, symbol, state }) => {
-        await this.lock.acquireAsync();
-        try {
-          return this.prismaService.marketData.upsert({
-            create: {
+        return this.prismaService.marketData.upsert({
+          create: {
+            dataSource: <DataSource>dataSource,
+            date: <Date>date,
+            marketPrice: <number>marketPrice,
+            state: <MarketDataState>state,
+            symbol: <string>symbol
+          },
+          update: {
+            marketPrice: <number>marketPrice,
+            state: <MarketDataState>state
+          },
+          where: {
+            dataSource_date_symbol: {
               dataSource: <DataSource>dataSource,
               date: <Date>date,
-              marketPrice: <number>marketPrice,
-              state: <MarketDataState>state,
               symbol: <string>symbol
-            },
-            update: {
-              marketPrice: <number>marketPrice,
-              state: <MarketDataState>state
-            },
-            where: {
-              dataSource_date_symbol: {
-                dataSource: <DataSource>dataSource,
-                date: <Date>date,
-                symbol: <string>symbol
-              }
             }
-          });
-        } finally {
-          this.lock.release();
-        }
+          }
+        });
       }
     );
     return await Promise.all(upsertPromises);

yarn.lock

@@ -13842,7 +13842,7 @@ lru-cache@6.0.0, lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
-lru-cache@^10.0.1, lru-cache@^10.2.0, lru-cache@^10.2.2:
+lru-cache@^10.0.1, lru-cache@^10.2.0:
   version "10.2.2"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.2.2.tgz#48206bc114c1252940c41b25b41af5b545aca878"
   integrity sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==
@@ -16803,11 +16803,38 @@ semver-dsl@^1.0.1:
   dependencies:
     semver "^5.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@7.3.2, semver@7.6.0, semver@7.x, semver@^5.3.0, semver@^5.6.0, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@~7.0.0:
+"semver@2 || 3 || 4 || 5", semver@^5.3.0, semver@^5.6.0:
+  version "5.7.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
+  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
+
+semver@7.6.0:
+  version "7.6.0"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d"
+  integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==
+  dependencies:
+    lru-cache "^6.0.0"
+
+semver@7.x, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2:
   version "7.3.2"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938"
   integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==
 
+semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0, semver@^6.3.1:
+  version "6.3.1"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
+  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
+
+semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0:
+  version "7.6.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.2.tgz#1e3b34759f896e8f14d6134732ce798aeb0c6e13"
+  integrity sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==
+
+semver@~7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e"
+  integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==
+
 send@0.18.0:
   version "0.18.0"
   resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be"
@@ -17564,9 +17591,6 @@ tar-stream@^2.1.4, tar-stream@~2.2.0:
     readable-stream "^3.1.1"
 
 tar@^6.1.11, tar@^6.1.2, tar@^6.2.0:
-  version "6.2.1"
-  resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a"
-  integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==
   version "6.2.1"
   resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a"
   integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==