Feature/improve handling of derived currencies (#2891)

* Improve handling of derived currencies * Update changelog
1 year ago · 5ba5b86d5f
6 changed files with 217 additions and 196 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ### Changed

+- Improved the handling of derived currencies
 - Improved the labels in the portfolio evolution chart and investment timeline on the analysis page
 - Improved the language localization for German (`de`)
 - Upgraded `prisma` from version `5.7.1` to `5.8.1`
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@ -9,14 +9,19 @@ import {
 import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
-import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';
+import {
+  DEFAULT_CURRENCY,
+  DERIVED_CURRENCIES,
+  PROPERTY_DATA_SOURCE_MAPPING
+} from '@ghostfolio/common/config';
 import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import type { Granularity, UserWithSettings } from '@ghostfolio/common/types';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
-import { format, isValid } from 'date-fns';
-import { groupBy, isEmpty, isNumber } from 'lodash';
+import Big from 'big.js';
+import { eachDayOfInterval, format, isValid } from 'date-fns';
+import { groupBy, isEmpty, isNumber, uniqWith } from 'lodash';
 import ms from 'ms';

@Injectable()
@ -205,6 +210,31 @@ export class DataProviderService {
  ): Promise<{
    [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
  }> {
+    let dataGatheringItems = aDataGatheringItems;
+
+    for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
+      if (
+        this.hasCurrency({
+          dataGatheringItems,
+          currency: `${DEFAULT_CURRENCY}${currency}`
+        })
+      ) {
+        // Skip derived currency
+        dataGatheringItems = dataGatheringItems.filter(({ symbol }) => {
+          return symbol !== `${DEFAULT_CURRENCY}${currency}`;
+        });
+        // Add root currency
+        dataGatheringItems.push({
+          dataSource: this.getDataSourceForExchangeRates(),
+          symbol: `${DEFAULT_CURRENCY}${rootCurrency}`
+        });
+      }
+    }
+
+    dataGatheringItems = uniqWith(dataGatheringItems, (obj1, obj2) => {
+      return obj1.dataSource === obj2.dataSource && obj1.symbol === obj2.symbol;
+    });
+
    const result: {
      [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
    } = {};
@ -213,25 +243,59 @@ export class DataProviderService {
      data: { [date: string]: IDataProviderHistoricalResponse };
      symbol: string;
    }>[] = [];
-    for (const { dataSource, symbol } of aDataGatheringItems) {
+    for (const { dataSource, symbol } of dataGatheringItems) {
      const dataProvider = this.getDataProvider(dataSource);
      if (dataProvider.canHandle(symbol)) {
-        promises.push(
-          dataProvider
-            .getHistorical({
-              from,
-              symbol,
-              to,
-              requestTimeout: ms('30 seconds')
+        if (symbol === `${DEFAULT_CURRENCY}USX`) {
+          const data: {
+            [date: string]: IDataProviderHistoricalResponse;
+          } = {};
+
+          for (const date of eachDayOfInterval({ end: to, start: from })) {
+            data[format(date, DATE_FORMAT)] = { marketPrice: 100 };
+          }
+
+          promises.push(
+            Promise.resolve({
+              data,
+              symbol
            })
-            .then((data) => ({ data: data?.[symbol], symbol }))
-        );
+          );
+        } else {
+          promises.push(
+            dataProvider
+              .getHistorical({
+                from,
+                symbol,
+                to,
+                requestTimeout: ms('30 seconds')
+              })
+              .then((data) => {
+                return { symbol, data: data?.[symbol] };
+              })
+          );
+        }
      }
    }

    try {
      const allData = await Promise.all(promises);
+
      for (const { data, symbol } of allData) {
+        const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => {
+          return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol;
+        });
+
+        if (currency) {
+          // Add derived currency
+          result[`${DEFAULT_CURRENCY}${currency.currency}`] =
+            this.transformHistoricalData({
+              allData,
+              currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`,
+              factor: currency.factor
+            });
+        }
+
        result[symbol] = data;
      }
    } catch (error) {
@ -257,6 +321,19 @@ export class DataProviderService {
    } = {};
    const startTimeTotal = performance.now();

+    if (
+      items.some(({ symbol }) => {
+        return symbol === `${DEFAULT_CURRENCY}USX`;
+      })
+    ) {
+      response[`${DEFAULT_CURRENCY}USX`] = {
+        currency: 'USX',
+        dataSource: this.getDataSourceForExchangeRates(),
+        marketPrice: 100,
+        marketState: 'open'
+      };
+    }
+
    // Get items from cache
    const itemsToFetch: UniqueAsset[] = [];

@ -326,19 +403,56 @@ export class DataProviderService {

        promises.push(
          promise.then(async (result) => {
-            for (const [symbol, dataProviderResponse] of Object.entries(
-              result
-            )) {
+            for (let [symbol, dataProviderResponse] of Object.entries(result)) {
+              if (
+                [
+                  ...DERIVED_CURRENCIES.map(({ currency }) => {
+                    return `${DEFAULT_CURRENCY}${currency}`;
+                  }),
+                  `${DEFAULT_CURRENCY}USX`
+                ].includes(symbol)
+              ) {
+                continue;
+              }
+
              response[symbol] = dataProviderResponse;

              this.redisCacheService.set(
                this.redisCacheService.getQuoteKey({
-                  dataSource: DataSource[dataSource],
-                  symbol
+                  symbol,
+                  dataSource: DataSource[dataSource]
                }),
-                JSON.stringify(dataProviderResponse),
+                JSON.stringify(response[symbol]),
                this.configurationService.get('CACHE_QUOTES_TTL')
              );
+
+              for (const {
+                currency,
+                factor,
+                rootCurrency
+              } of DERIVED_CURRENCIES) {
+                if (symbol === `${DEFAULT_CURRENCY}${rootCurrency}`) {
+                  response[`${DEFAULT_CURRENCY}${currency}`] = {
+                    ...dataProviderResponse,
+                    currency,
+                    marketPrice: new Big(
+                      result[`${DEFAULT_CURRENCY}${rootCurrency}`].marketPrice
+                    )
+                      .mul(factor)
+                      .toNumber(),
+                    marketState: 'open'
+                  };
+
+                  this.redisCacheService.set(
+                    this.redisCacheService.getQuoteKey({
+                      dataSource: DataSource[dataSource],
+                      symbol: `${DEFAULT_CURRENCY}${currency}`
+                    }),
+                    JSON.stringify(response[`${DEFAULT_CURRENCY}${currency}`]),
+                    this.configurationService.get('CACHE_QUOTES_TTL')
+                  );
+                }
+              }
            }

            Logger.debug(
@ -472,6 +586,21 @@ export class DataProviderService {
    throw new Error('No data provider has been found.');
  }

+  private hasCurrency({
+    currency,
+    dataGatheringItems
+  }: {
+    currency: string;
+    dataGatheringItems: UniqueAsset[];
+  }) {
+    return dataGatheringItems.some(({ dataSource, symbol }) => {
+      return (
+        dataSource === this.getDataSourceForExchangeRates() &&
+        symbol === currency
+      );
+    });
+  }
+
  private isPremiumDataSource(aDataSource: DataSource) {
    const premiumDataSources: DataSource[] = [
      DataSource.EOD_HISTORICAL_DATA,
@ -479,4 +608,35 @@ export class DataProviderService {
    ];
    return premiumDataSources.includes(aDataSource);
  }
+
+  private transformHistoricalData({
+    allData,
+    currency,
+    factor
+  }: {
+    allData: {
+      data: {
+        [date: string]: IDataProviderHistoricalResponse;
+      };
+      symbol: string;
+    }[];
+    currency: string;
+    factor: number;
+  }) {
+    const rootData = allData.find(({ symbol }) => {
+      return symbol === currency;
+    })?.data;
+
+    const data: {
+      [date: string]: IDataProviderHistoricalResponse;
+    } = {};
+
+    for (const date in rootData) {
+      data[date] = {
+        marketPrice: new Big(factor).mul(rootData[date].marketPrice).toNumber()
+      };
+    }
+
+    return data;
+  }
 }
--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
@ -20,7 +20,6 @@ import {
  DataSource,
  SymbolProfile
 } from '@prisma/client';
-import Big from 'big.js';
 import { format, isToday } from 'date-fns';
 import got from 'got';

@ -93,10 +92,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
      return response.reduce(
        (result, historicalItem, index, array) => {
          result[this.convertFromEodSymbol(symbol)][historicalItem.date] = {
-            marketPrice: this.getConvertedValue({
-              symbol: symbol,
-              value: historicalItem.close
-            })
+            marketPrice: historicalItem.close
          };

          return result;
@ -196,48 +192,6 @@ export class EodHistoricalDataService implements DataProviderInterface {
        {}
      );

-      if (response[`${DEFAULT_CURRENCY}GBP`]) {
-        response[`${DEFAULT_CURRENCY}GBp`] = {
-          ...response[`${DEFAULT_CURRENCY}GBP`],
-          currency: 'GBp',
-          marketPrice: this.getConvertedValue({
-            symbol: `${DEFAULT_CURRENCY}GBp`,
-            value: response[`${DEFAULT_CURRENCY}GBP`].marketPrice
-          })
-        };
-      }
-
-      if (response[`${DEFAULT_CURRENCY}ILS`]) {
-        response[`${DEFAULT_CURRENCY}ILA`] = {
-          ...response[`${DEFAULT_CURRENCY}ILS`],
-          currency: 'ILA',
-          marketPrice: this.getConvertedValue({
-            symbol: `${DEFAULT_CURRENCY}ILA`,
-            value: response[`${DEFAULT_CURRENCY}ILS`].marketPrice
-          })
-        };
-      }
-
-      if (response[`${DEFAULT_CURRENCY}USX`]) {
-        response[`${DEFAULT_CURRENCY}USX`] = {
-          currency: 'USX',
-          dataSource: this.getName(),
-          marketPrice: new Big(1).mul(100).toNumber(),
-          marketState: 'open'
-        };
-      }
-
-      if (response[`${DEFAULT_CURRENCY}ZAR`]) {
-        response[`${DEFAULT_CURRENCY}ZAc`] = {
-          ...response[`${DEFAULT_CURRENCY}ZAR`],
-          currency: 'ZAc',
-          marketPrice: this.getConvertedValue({
-            symbol: `${DEFAULT_CURRENCY}ZAc`,
-            value: response[`${DEFAULT_CURRENCY}ZAR`].marketPrice
-          })
-        };
-      }
-
      return response;
    } catch (error) {
      let message = error;
@ -337,27 +291,6 @@ export class EodHistoricalDataService implements DataProviderInterface {
    return aSymbol;
  }

-  private getConvertedValue({
-    symbol,
-    value
-  }: {
-    symbol: string;
-    value: number;
-  }) {
-    if (symbol === `${DEFAULT_CURRENCY}GBp`) {
-      // Convert GPB to GBp (pence)
-      return new Big(value).mul(100).toNumber();
-    } else if (symbol === `${DEFAULT_CURRENCY}ILA`) {
-      // Convert ILS to ILA
-      return new Big(value).mul(100).toNumber();
-    } else if (symbol === `${DEFAULT_CURRENCY}ZAc`) {
-      // Convert ZAR to ZAc
-      return new Big(value).mul(100).toNumber();
-    }
-
-    return value;
-  }
-
  private async getSearchResult(aQuery: string): Promise<
    (LookupItem & {
      assetClass: AssetClass;
--- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
+++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
@ -16,7 +16,6 @@ import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Injectable, Logger } from '@nestjs/common';
 import { DataSource, SymbolProfile } from '@prisma/client';
-import Big from 'big.js';
 import { addDays, format, isSameDay } from 'date-fns';
 import yahooFinance from 'yahoo-finance2';
 import { Quote } from 'yahoo-finance2/dist/esm/src/modules/quote';
@ -77,10 +76,7 @@ export class YahooFinanceService implements DataProviderInterface {

      for (const historicalItem of historicalResult) {
        response[format(historicalItem.date, DATE_FORMAT)] = {
-          marketPrice: this.getConvertedValue({
-            symbol,
-            value: historicalItem.dividends
-          })
+          marketPrice: historicalItem.dividends
        };
      }

@ -129,10 +125,7 @@ export class YahooFinanceService implements DataProviderInterface {

      for (const historicalItem of historicalResult) {
        response[symbol][format(historicalItem.date, DATE_FORMAT)] = {
-          marketPrice: this.getConvertedValue({
-            symbol: symbol,
-            value: historicalItem.close
-          })
+          marketPrice: historicalItem.close
        };
      }

@ -204,57 +197,6 @@ export class YahooFinanceService implements DataProviderInterface {
              : 'closed',
          marketPrice: quote.regularMarketPrice || 0
        };
-
-        if (
-          symbol === `${DEFAULT_CURRENCY}GBP` &&
-          yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}GBp=X`)
-        ) {
-          // Convert GPB to GBp (pence)
-          response[`${DEFAULT_CURRENCY}GBp`] = {
-            ...response[symbol],
-            currency: 'GBp',
-            marketPrice: this.getConvertedValue({
-              symbol: `${DEFAULT_CURRENCY}GBp`,
-              value: response[symbol].marketPrice
-            })
-          };
-        } else if (
-          symbol === `${DEFAULT_CURRENCY}ILS` &&
-          yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ILA=X`)
-        ) {
-          // Convert ILS to ILA
-          response[`${DEFAULT_CURRENCY}ILA`] = {
-            ...response[symbol],
-            currency: 'ILA',
-            marketPrice: this.getConvertedValue({
-              symbol: `${DEFAULT_CURRENCY}ILA`,
-              value: response[symbol].marketPrice
-            })
-          };
-        } else if (
-          symbol === `${DEFAULT_CURRENCY}ZAR` &&
-          yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ZAc=X`)
-        ) {
-          // Convert ZAR to ZAc (cents)
-          response[`${DEFAULT_CURRENCY}ZAc`] = {
-            ...response[symbol],
-            currency: 'ZAc',
-            marketPrice: this.getConvertedValue({
-              symbol: `${DEFAULT_CURRENCY}ZAc`,
-              value: response[symbol].marketPrice
-            })
-          };
-        }
-      }
-
-      if (yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}USX=X`)) {
-        // Convert USD to USX (cent)
-        response[`${DEFAULT_CURRENCY}USX`] = {
-          currency: 'USX',
-          dataSource: this.getName(),
-          marketPrice: new Big(1).mul(100).toNumber(),
-          marketState: 'open'
-        };
      }

      return response;
@ -357,27 +299,6 @@ export class YahooFinanceService implements DataProviderInterface {
    return { items };
  }

-  private getConvertedValue({
-    symbol,
-    value
-  }: {
-    symbol: string;
-    value: number;
-  }) {
-    if (symbol === `${DEFAULT_CURRENCY}GBp`) {
-      // Convert GPB to GBp (pence)
-      return new Big(value).mul(100).toNumber();
-    } else if (symbol === `${DEFAULT_CURRENCY}ILA`) {
-      // Convert ILS to ILA
-      return new Big(value).mul(100).toNumber();
-    } else if (symbol === `${DEFAULT_CURRENCY}ZAc`) {
-      // Convert ZAR to ZAc (cents)
-      return new Big(value).mul(100).toNumber();
-    }
-
-    return value;
-  }
-
  private async getQuotesWithQuoteSummary(aYahooFinanceSymbols: string[]) {
    const quoteSummaryPromises = aYahooFinanceSymbols.map((symbol) => {
      return yahooFinance.quoteSummary(symbol).catch(() => {
--- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
+++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
@ -5,6 +5,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import {
  DEFAULT_CURRENCY,
+  DERIVED_CURRENCIES,
  PROPERTY_CURRENCIES
 } from '@ghostfolio/common/config';
 import {
@ -168,30 +169,6 @@ export class ExchangeRateDataService {
      const [currency1, currency2] = symbol.match(/.{1,3}/g);
      const [date] = Object.keys(result[symbol]);

-      // Add derived currencies
-      if (currency2 === 'GBP') {
-        resultExtended[`${currency1}GBp`] = {
-          [date]: {
-            marketPrice:
-              result[`${currency1}${currency2}`][date].marketPrice * 100
-          }
-        };
-      } else if (currency2 === 'ILS') {
-        resultExtended[`${currency1}ILA`] = {
-          [date]: {
-            marketPrice:
-              result[`${currency1}${currency2}`][date].marketPrice * 100
-          }
-        };
-      } else if (currency2 === 'ZAR') {
-        resultExtended[`${currency1}ZAc`] = {
-          [date]: {
-            marketPrice:
-              result[`${currency1}${currency2}`][date].marketPrice * 100
-          }
-        };
-      }
-
      // Calculate the opposite direction
      resultExtended[`${currency2}${currency1}`] = {
        [date]: {
@ -486,8 +463,8 @@ export class ExchangeRateDataService {
          }
        }
      })
-    ).forEach((account) => {
-      currencies.push(account.currency);
+    ).forEach(({ currency }) => {
+      currencies.push(currency);
    });

    (
@ -496,8 +473,8 @@ export class ExchangeRateDataService {
        orderBy: [{ currency: 'asc' }],
        select: { currency: true }
      })
-    ).forEach((symbolProfile) => {
-      currencies.push(symbolProfile.currency);
+    ).forEach(({ currency }) => {
+      currencies.push(currency);
    });

    const customCurrencies = (await this.propertyService.getByKey(
@ -508,6 +485,16 @@ export class ExchangeRateDataService {
      currencies = currencies.concat(customCurrencies);
    }

+    // Add derived currencies
+    currencies.push('USX');
+
+    for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
+      if (currencies.includes(currency) || currencies.includes(rootCurrency)) {
+        currencies.push(currency);
+        currencies.push(rootCurrency);
+      }
+    }
+
    return uniq(currencies).filter(Boolean).sort();
  }

--- a/libs/common/src/lib/config.ts
+++ b/libs/common/src/lib/config.ts
@ -41,6 +41,25 @@ export const DEFAULT_LANGUAGE_CODE = 'en';
 export const DEFAULT_PAGE_SIZE = 50;
 export const DEFAULT_ROOT_URL = 'http://localhost:4200';

+// USX is handled separately
+export const DERIVED_CURRENCIES = [
+  {
+    currency: 'GBp',
+    factor: 100,
+    rootCurrency: 'GBP'
+  },
+  {
+    currency: 'ILA',
+    factor: 100,
+    rootCurrency: 'ILS'
+  },
+  {
+    currency: 'ZAc',
+    factor: 100,
+    rootCurrency: 'ZAR'
+  }
+];
+
 export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180';

 export const GATHER_ASSET_PROFILE_PROCESS = 'GATHER_ASSET_PROFILE';