Feature/improve handling of derived currencies (#2891)

* Improve handling of derived currencies

* Update changelog

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Improved the handling of derived currencies
 - Improved the labels in the portfolio evolution chart and investment timeline on the analysis page
 - Improved the language localization for German (`de`)
 - Upgraded `prisma` from version `5.7.1` to `5.8.1`

apps/api/src/services/data-provider/data-provider.service.ts

@@ -9,14 +9,19 @@ import {
 import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
-import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';
+import {
+  DEFAULT_CURRENCY,
+  DERIVED_CURRENCIES,
+  PROPERTY_DATA_SOURCE_MAPPING
+} from '@ghostfolio/common/config';
 import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import type { Granularity, UserWithSettings } from '@ghostfolio/common/types';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
-import { format, isValid } from 'date-fns';
+import Big from 'big.js';
-import { groupBy, isEmpty, isNumber } from 'lodash';
+import { eachDayOfInterval, format, isValid } from 'date-fns';
+import { groupBy, isEmpty, isNumber, uniqWith } from 'lodash';
 import ms from 'ms';
 
 @Injectable()
@@ -205,6 +210,31 @@ export class DataProviderService {
   ): Promise<{
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }> {
+    let dataGatheringItems = aDataGatheringItems;
+
+    for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
+      if (
+        this.hasCurrency({
+          dataGatheringItems,
+          currency: `${DEFAULT_CURRENCY}${currency}`
+        })
+      ) {
+        // Skip derived currency
+        dataGatheringItems = dataGatheringItems.filter(({ symbol }) => {
+          return symbol !== `${DEFAULT_CURRENCY}${currency}`;
+        });
+        // Add root currency
+        dataGatheringItems.push({
+          dataSource: this.getDataSourceForExchangeRates(),
+          symbol: `${DEFAULT_CURRENCY}${rootCurrency}`
+        });
+      }
+    }
+
+    dataGatheringItems = uniqWith(dataGatheringItems, (obj1, obj2) => {
+      return obj1.dataSource === obj2.dataSource && obj1.symbol === obj2.symbol;
+    });
+
     const result: {
       [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
     } = {};
@@ -213,25 +243,59 @@ export class DataProviderService {
       data: { [date: string]: IDataProviderHistoricalResponse };
       symbol: string;
     }>[] = [];
-    for (const { dataSource, symbol } of aDataGatheringItems) {
+    for (const { dataSource, symbol } of dataGatheringItems) {
       const dataProvider = this.getDataProvider(dataSource);
       if (dataProvider.canHandle(symbol)) {
-        promises.push(
+        if (symbol === `${DEFAULT_CURRENCY}USX`) {
-          dataProvider
+          const data: {
-            .getHistorical({
+            [date: string]: IDataProviderHistoricalResponse;
-              from,
+          } = {};
-              symbol,
+
-              to,
+          for (const date of eachDayOfInterval({ end: to, start: from })) {
-              requestTimeout: ms('30 seconds')
+            data[format(date, DATE_FORMAT)] = { marketPrice: 100 };
+          }
+
+          promises.push(
+            Promise.resolve({
+              data,
+              symbol
             })
-            .then((data) => ({ data: data?.[symbol], symbol }))
+          );
-        );
+        } else {
+          promises.push(
+            dataProvider
+              .getHistorical({
+                from,
+                symbol,
+                to,
+                requestTimeout: ms('30 seconds')
+              })
+              .then((data) => {
+                return { symbol, data: data?.[symbol] };
+              })
+          );
+        }
       }
     }
 
     try {
       const allData = await Promise.all(promises);
+
       for (const { data, symbol } of allData) {
+        const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => {
+          return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol;
+        });
+
+        if (currency) {
+          // Add derived currency
+          result[`${DEFAULT_CURRENCY}${currency.currency}`] =
+            this.transformHistoricalData({
+              allData,
+              currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`,
+              factor: currency.factor
+            });
+        }
+
         result[symbol] = data;
       }
     } catch (error) {
@@ -257,6 +321,19 @@ export class DataProviderService {
     } = {};
     const startTimeTotal = performance.now();
 
+    if (
+      items.some(({ symbol }) => {
+        return symbol === `${DEFAULT_CURRENCY}USX`;
+      })
+    ) {
+      response[`${DEFAULT_CURRENCY}USX`] = {
+        currency: 'USX',
+        dataSource: this.getDataSourceForExchangeRates(),
+        marketPrice: 100,
+        marketState: 'open'
+      };
+    }
+
     // Get items from cache
     const itemsToFetch: UniqueAsset[] = [];
 
@@ -326,19 +403,56 @@ export class DataProviderService {
 
         promises.push(
           promise.then(async (result) => {
-            for (const [symbol, dataProviderResponse] of Object.entries(
+            for (let [symbol, dataProviderResponse] of Object.entries(result)) {
-              result
+              if (
-            )) {
+                [
+                  ...DERIVED_CURRENCIES.map(({ currency }) => {
+                    return `${DEFAULT_CURRENCY}${currency}`;
+                  }),
+                  `${DEFAULT_CURRENCY}USX`
+                ].includes(symbol)
+              ) {
+                continue;
+              }
+
               response[symbol] = dataProviderResponse;
 
               this.redisCacheService.set(
                 this.redisCacheService.getQuoteKey({
-                  dataSource: DataSource[dataSource],
+                  symbol,
-                  symbol
+                  dataSource: DataSource[dataSource]
                 }),
-                JSON.stringify(dataProviderResponse),
+                JSON.stringify(response[symbol]),
                 this.configurationService.get('CACHE_QUOTES_TTL')
               );
+
+              for (const {
+                currency,
+                factor,
+                rootCurrency
+              } of DERIVED_CURRENCIES) {
+                if (symbol === `${DEFAULT_CURRENCY}${rootCurrency}`) {
+                  response[`${DEFAULT_CURRENCY}${currency}`] = {
+                    ...dataProviderResponse,
+                    currency,
+                    marketPrice: new Big(
+                      result[`${DEFAULT_CURRENCY}${rootCurrency}`].marketPrice
+                    )
+                      .mul(factor)
+                      .toNumber(),
+                    marketState: 'open'
+                  };
+
+                  this.redisCacheService.set(
+                    this.redisCacheService.getQuoteKey({
+                      dataSource: DataSource[dataSource],
+                      symbol: `${DEFAULT_CURRENCY}${currency}`
+                    }),
+                    JSON.stringify(response[`${DEFAULT_CURRENCY}${currency}`]),
+                    this.configurationService.get('CACHE_QUOTES_TTL')
+                  );
+                }
+              }
             }
 
             Logger.debug(
@@ -472,6 +586,21 @@ export class DataProviderService {
     throw new Error('No data provider has been found.');
   }
 
+  private hasCurrency({
+    currency,
+    dataGatheringItems
+  }: {
+    currency: string;
+    dataGatheringItems: UniqueAsset[];
+  }) {
+    return dataGatheringItems.some(({ dataSource, symbol }) => {
+      return (
+        dataSource === this.getDataSourceForExchangeRates() &&
+        symbol === currency
+      );
+    });
+  }
+
   private isPremiumDataSource(aDataSource: DataSource) {
     const premiumDataSources: DataSource[] = [
       DataSource.EOD_HISTORICAL_DATA,
@@ -479,4 +608,35 @@ export class DataProviderService {
     ];
     return premiumDataSources.includes(aDataSource);
   }
+
+  private transformHistoricalData({
+    allData,
+    currency,
+    factor
+  }: {
+    allData: {
+      data: {
+        [date: string]: IDataProviderHistoricalResponse;
+      };
+      symbol: string;
+    }[];
+    currency: string;
+    factor: number;
+  }) {
+    const rootData = allData.find(({ symbol }) => {
+      return symbol === currency;
+    })?.data;
+
+    const data: {
+      [date: string]: IDataProviderHistoricalResponse;
+    } = {};
+
+    for (const date in rootData) {
+      data[date] = {
+        marketPrice: new Big(factor).mul(rootData[date].marketPrice).toNumber()
+      };
+    }
+
+    return data;
+  }
 }

apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts

@@ -20,7 +20,6 @@ import {
   DataSource,
   SymbolProfile
 } from '@prisma/client';
-import Big from 'big.js';
 import { format, isToday } from 'date-fns';
 import got from 'got';
 
@@ -93,10 +92,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
       return response.reduce(
         (result, historicalItem, index, array) => {
           result[this.convertFromEodSymbol(symbol)][historicalItem.date] = {
-            marketPrice: this.getConvertedValue({
+            marketPrice: historicalItem.close
-              symbol: symbol,
-              value: historicalItem.close
-            })
           };
 
           return result;
@@ -196,48 +192,6 @@ export class EodHistoricalDataService implements DataProviderInterface {
         {}
       );
 
-      if (response[`${DEFAULT_CURRENCY}GBP`]) {
-        response[`${DEFAULT_CURRENCY}GBp`] = {
-          ...response[`${DEFAULT_CURRENCY}GBP`],
-          currency: 'GBp',
-          marketPrice: this.getConvertedValue({
-            symbol: `${DEFAULT_CURRENCY}GBp`,
-            value: response[`${DEFAULT_CURRENCY}GBP`].marketPrice
-          })
-        };
-      }
-
-      if (response[`${DEFAULT_CURRENCY}ILS`]) {
-        response[`${DEFAULT_CURRENCY}ILA`] = {
-          ...response[`${DEFAULT_CURRENCY}ILS`],
-          currency: 'ILA',
-          marketPrice: this.getConvertedValue({
-            symbol: `${DEFAULT_CURRENCY}ILA`,
-            value: response[`${DEFAULT_CURRENCY}ILS`].marketPrice
-          })
-        };
-      }
-
-      if (response[`${DEFAULT_CURRENCY}USX`]) {
-        response[`${DEFAULT_CURRENCY}USX`] = {
-          currency: 'USX',
-          dataSource: this.getName(),
-          marketPrice: new Big(1).mul(100).toNumber(),
-          marketState: 'open'
-        };
-      }
-
-      if (response[`${DEFAULT_CURRENCY}ZAR`]) {
-        response[`${DEFAULT_CURRENCY}ZAc`] = {
-          ...response[`${DEFAULT_CURRENCY}ZAR`],
-          currency: 'ZAc',
-          marketPrice: this.getConvertedValue({
-            symbol: `${DEFAULT_CURRENCY}ZAc`,
-            value: response[`${DEFAULT_CURRENCY}ZAR`].marketPrice
-          })
-        };
-      }
-
       return response;
     } catch (error) {
       let message = error;
@@ -337,27 +291,6 @@ export class EodHistoricalDataService implements DataProviderInterface {
     return aSymbol;
   }
 
-  private getConvertedValue({
-    symbol,
-    value
-  }: {
-    symbol: string;
-    value: number;
-  }) {
-    if (symbol === `${DEFAULT_CURRENCY}GBp`) {
-      // Convert GPB to GBp (pence)
-      return new Big(value).mul(100).toNumber();
-    } else if (symbol === `${DEFAULT_CURRENCY}ILA`) {
-      // Convert ILS to ILA
-      return new Big(value).mul(100).toNumber();
-    } else if (symbol === `${DEFAULT_CURRENCY}ZAc`) {
-      // Convert ZAR to ZAc
-      return new Big(value).mul(100).toNumber();
-    }
-
-    return value;
-  }
-
   private async getSearchResult(aQuery: string): Promise<
     (LookupItem & {
       assetClass: AssetClass;

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@@ -16,7 +16,6 @@ import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Injectable, Logger } from '@nestjs/common';
 import { DataSource, SymbolProfile } from '@prisma/client';
-import Big from 'big.js';
 import { addDays, format, isSameDay } from 'date-fns';
 import yahooFinance from 'yahoo-finance2';
 import { Quote } from 'yahoo-finance2/dist/esm/src/modules/quote';
@@ -77,10 +76,7 @@ export class YahooFinanceService implements DataProviderInterface {
 
       for (const historicalItem of historicalResult) {
         response[format(historicalItem.date, DATE_FORMAT)] = {
-          marketPrice: this.getConvertedValue({
+          marketPrice: historicalItem.dividends
-            symbol,
-            value: historicalItem.dividends
-          })
         };
       }
 
@@ -129,10 +125,7 @@ export class YahooFinanceService implements DataProviderInterface {
 
       for (const historicalItem of historicalResult) {
         response[symbol][format(historicalItem.date, DATE_FORMAT)] = {
-          marketPrice: this.getConvertedValue({
+          marketPrice: historicalItem.close
-            symbol: symbol,
-            value: historicalItem.close
-          })
         };
       }
 
@@ -204,57 +197,6 @@ export class YahooFinanceService implements DataProviderInterface {
               : 'closed',
           marketPrice: quote.regularMarketPrice || 0
         };
-
-        if (
-          symbol === `${DEFAULT_CURRENCY}GBP` &&
-          yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}GBp=X`)
-        ) {
-          // Convert GPB to GBp (pence)
-          response[`${DEFAULT_CURRENCY}GBp`] = {
-            ...response[symbol],
-            currency: 'GBp',
-            marketPrice: this.getConvertedValue({
-              symbol: `${DEFAULT_CURRENCY}GBp`,
-              value: response[symbol].marketPrice
-            })
-          };
-        } else if (
-          symbol === `${DEFAULT_CURRENCY}ILS` &&
-          yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ILA=X`)
-        ) {
-          // Convert ILS to ILA
-          response[`${DEFAULT_CURRENCY}ILA`] = {
-            ...response[symbol],
-            currency: 'ILA',
-            marketPrice: this.getConvertedValue({
-              symbol: `${DEFAULT_CURRENCY}ILA`,
-              value: response[symbol].marketPrice
-            })
-          };
-        } else if (
-          symbol === `${DEFAULT_CURRENCY}ZAR` &&
-          yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ZAc=X`)
-        ) {
-          // Convert ZAR to ZAc (cents)
-          response[`${DEFAULT_CURRENCY}ZAc`] = {
-            ...response[symbol],
-            currency: 'ZAc',
-            marketPrice: this.getConvertedValue({
-              symbol: `${DEFAULT_CURRENCY}ZAc`,
-              value: response[symbol].marketPrice
-            })
-          };
-        }
-      }
-
-      if (yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}USX=X`)) {
-        // Convert USD to USX (cent)
-        response[`${DEFAULT_CURRENCY}USX`] = {
-          currency: 'USX',
-          dataSource: this.getName(),
-          marketPrice: new Big(1).mul(100).toNumber(),
-          marketState: 'open'
-        };
       }
 
       return response;
@@ -357,27 +299,6 @@ export class YahooFinanceService implements DataProviderInterface {
     return { items };
   }
 
-  private getConvertedValue({
-    symbol,
-    value
-  }: {
-    symbol: string;
-    value: number;
-  }) {
-    if (symbol === `${DEFAULT_CURRENCY}GBp`) {
-      // Convert GPB to GBp (pence)
-      return new Big(value).mul(100).toNumber();
-    } else if (symbol === `${DEFAULT_CURRENCY}ILA`) {
-      // Convert ILS to ILA
-      return new Big(value).mul(100).toNumber();
-    } else if (symbol === `${DEFAULT_CURRENCY}ZAc`) {
-      // Convert ZAR to ZAc (cents)
-      return new Big(value).mul(100).toNumber();
-    }
-
-    return value;
-  }
-
   private async getQuotesWithQuoteSummary(aYahooFinanceSymbols: string[]) {
     const quoteSummaryPromises = aYahooFinanceSymbols.map((symbol) => {
       return yahooFinance.quoteSummary(symbol).catch(() => {

apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts

@@ -5,6 +5,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import {
   DEFAULT_CURRENCY,
+  DERIVED_CURRENCIES,
   PROPERTY_CURRENCIES
 } from '@ghostfolio/common/config';
 import {
@@ -168,30 +169,6 @@ export class ExchangeRateDataService {
       const [currency1, currency2] = symbol.match(/.{1,3}/g);
       const [date] = Object.keys(result[symbol]);
 
-      // Add derived currencies
-      if (currency2 === 'GBP') {
-        resultExtended[`${currency1}GBp`] = {
-          [date]: {
-            marketPrice:
-              result[`${currency1}${currency2}`][date].marketPrice * 100
-          }
-        };
-      } else if (currency2 === 'ILS') {
-        resultExtended[`${currency1}ILA`] = {
-          [date]: {
-            marketPrice:
-              result[`${currency1}${currency2}`][date].marketPrice * 100
-          }
-        };
-      } else if (currency2 === 'ZAR') {
-        resultExtended[`${currency1}ZAc`] = {
-          [date]: {
-            marketPrice:
-              result[`${currency1}${currency2}`][date].marketPrice * 100
-          }
-        };
-      }
-
       // Calculate the opposite direction
       resultExtended[`${currency2}${currency1}`] = {
         [date]: {
@@ -486,8 +463,8 @@ export class ExchangeRateDataService {
           }
         }
       })
-    ).forEach((account) => {
+    ).forEach(({ currency }) => {
-      currencies.push(account.currency);
+      currencies.push(currency);
     });
 
     (
@@ -496,8 +473,8 @@ export class ExchangeRateDataService {
         orderBy: [{ currency: 'asc' }],
         select: { currency: true }
       })
-    ).forEach((symbolProfile) => {
+    ).forEach(({ currency }) => {
-      currencies.push(symbolProfile.currency);
+      currencies.push(currency);
     });
 
     const customCurrencies = (await this.propertyService.getByKey(
@@ -508,6 +485,16 @@ export class ExchangeRateDataService {
       currencies = currencies.concat(customCurrencies);
     }
 
+    // Add derived currencies
+    currencies.push('USX');
+
+    for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
+      if (currencies.includes(currency) || currencies.includes(rootCurrency)) {
+        currencies.push(currency);
+        currencies.push(rootCurrency);
+      }
+    }
+
     return uniq(currencies).filter(Boolean).sort();
   }
 

libs/common/src/lib/config.ts

@@ -41,6 +41,25 @@ export const DEFAULT_LANGUAGE_CODE = 'en';
 export const DEFAULT_PAGE_SIZE = 50;
 export const DEFAULT_ROOT_URL = 'http://localhost:4200';
 
+// USX is handled separately
+export const DERIVED_CURRENCIES = [
+  {
+    currency: 'GBp',
+    factor: 100,
+    rootCurrency: 'GBP'
+  },
+  {
+    currency: 'ILA',
+    factor: 100,
+    rootCurrency: 'ILS'
+  },
+  {
+    currency: 'ZAc',
+    factor: 100,
+    rootCurrency: 'ZAR'
+  }
+];
+
 export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180';
 
 export const GATHER_ASSET_PROFILE_PROCESS = 'GATHER_ASSET_PROFILE';