Feature/introduce env variable data source exchange rates and data source import (#1910)

* Introduce env variables DATA_SOURCE_EXCHANGE_RATES and DATA_SOURCE_IMPORT

* Update changelog

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Split the environment variable `DATA_SOURCE_PRIMARY` in `DATA_SOURCE_EXCHANGE_RATES` and `DATA_SOURCE_IMPORT`
+
 ## 1.262.0 - 2023-04-29
 
 ### Added

apps/api/src/app/import/import.service.ts

@@ -15,7 +15,7 @@ import {
   OrderWithAccount
 } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
-import { Prisma, SymbolProfile } from '@prisma/client';
+import { DataSource, Prisma, SymbolProfile } from '@prisma/client';
 import Big from 'big.js';
 import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';
 import { v4 as uuidv4 } from 'uuid';
@@ -183,9 +183,10 @@ export class ImportService {
     for (const activity of activitiesDto) {
       if (!activity.dataSource) {
         if (activity.type === 'ITEM') {
-          activity.dataSource = 'MANUAL';
+          activity.dataSource = DataSource.MANUAL;
         } else {
-          activity.dataSource = this.dataProviderService.getPrimaryDataSource();
+          activity.dataSource =
+            this.dataProviderService.getDataSourceForImport();
         }
       }
 

apps/api/src/services/configuration/configuration.service.ts

@@ -16,7 +16,8 @@ export class ConfigurationService {
         default: 'USD'
       }),
       CACHE_TTL: num({ default: 1 }),
-      DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),
+      DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }),
+      DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }),
       DATA_SOURCES: json({
         default: [DataSource.COINGECKO, DataSource.MANUAL, DataSource.YAHOO]
       }),

apps/api/src/services/data-provider/data-provider.service.ts

@@ -58,6 +58,52 @@ export class DataProviderService {
     return false;
   }
 
+  public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{
+    [symbol: string]: Partial<SymbolProfile>;
+  }> {
+    const response: {
+      [symbol: string]: Partial<SymbolProfile>;
+    } = {};
+
+    const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
+
+    const promises = [];
+
+    for (const [dataSource, dataGatheringItems] of Object.entries(
+      itemsGroupedByDataSource
+    )) {
+      const symbols = dataGatheringItems.map((dataGatheringItem) => {
+        return dataGatheringItem.symbol;
+      });
+
+      for (const symbol of symbols) {
+        const promise = Promise.resolve(
+          this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol)
+        );
+
+        promises.push(
+          promise.then((symbolProfile) => {
+            response[symbol] = symbolProfile;
+          })
+        );
+      }
+    }
+
+    await Promise.all(promises);
+
+    return response;
+  }
+
+  public getDataSourceForExchangeRates(): DataSource {
+    return DataSource[
+      this.configurationService.get('DATA_SOURCE_EXCHANGE_RATES')
+    ];
+  }
+
+  public getDataSourceForImport(): DataSource {
+    return DataSource[this.configurationService.get('DATA_SOURCE_IMPORT')];
+  }
+
   public async getDividends({
     dataSource,
     from,
@@ -182,46 +228,6 @@ export class DataProviderService {
     return result;
   }
 
-  public getPrimaryDataSource(): DataSource {
-    return DataSource[this.configurationService.get('DATA_SOURCE_PRIMARY')];
-  }
-
-  public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{
-    [symbol: string]: Partial<SymbolProfile>;
-  }> {
-    const response: {
-      [symbol: string]: Partial<SymbolProfile>;
-    } = {};
-
-    const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
-
-    const promises = [];
-
-    for (const [dataSource, dataGatheringItems] of Object.entries(
-      itemsGroupedByDataSource
-    )) {
-      const symbols = dataGatheringItems.map((dataGatheringItem) => {
-        return dataGatheringItem.symbol;
-      });
-
-      for (const symbol of symbols) {
-        const promise = Promise.resolve(
-          this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol)
-        );
-
-        promises.push(
-          promise.then((symbolProfile) => {
-            response[symbol] = symbolProfile;
-          })
-        );
-      }
-    }
-
-    await Promise.all(promises);
-
-    return response;
-  }
-
   public async getQuotes(items: IDataGatheringItem[]): Promise<{
     [symbol: string]: IDataProviderResponse;
   }> {

apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts

@@ -5,7 +5,7 @@ import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DATE_FORMAT, isCurrency } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable, Logger } from '@nestjs/common';
 import {
@@ -15,17 +15,20 @@ import {
   SymbolProfile
 } from '@prisma/client';
 import bent from 'bent';
+import Big from 'big.js';
 import { format, isToday } from 'date-fns';
 
 @Injectable()
 export class EodHistoricalDataService implements DataProviderInterface {
   private apiKey: string;
+  private baseCurrency: string;
   private readonly URL = 'https://eodhistoricaldata.com/api';
 
   public constructor(
     private readonly configurationService: ConfigurationService
   ) {
     this.apiKey = this.configurationService.get('EOD_HISTORICAL_DATA_API_KEY');
+    this.baseCurrency = this.configurationService.get('BASE_CURRENCY');
   }
 
   public canHandle(symbol: string) {
@@ -70,9 +73,11 @@ export class EodHistoricalDataService implements DataProviderInterface {
   ): Promise<{
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }> {
+    const symbol = this.convertToEodSymbol(aSymbol);
+
     try {
       const get = bent(
-        `${this.URL}/eod/${aSymbol}?api_token=${
+        `${this.URL}/eod/${symbol}?api_token=${
           this.apiKey
         }&fmt=json&from=${format(from, DATE_FORMAT)}&to=${format(
           to,
@@ -87,14 +92,17 @@ export class EodHistoricalDataService implements DataProviderInterface {
 
       return response.reduce(
         (result, historicalItem, index, array) => {
-          result[aSymbol][historicalItem.date] = {
-            marketPrice: historicalItem.close,
+          result[this.convertFromEodSymbol(symbol)][historicalItem.date] = {
+            marketPrice: this.getConvertedValue({
+              symbol: aSymbol,
+              value: historicalItem.close
+            }),
             performance: historicalItem.open - historicalItem.close
           };
 
           return result;
         },
-        { [aSymbol]: {} }
+        { [this.convertFromEodSymbol(symbol)]: {} }
       );
     } catch (error) {
       throw new Error(
@@ -119,52 +127,87 @@ export class EodHistoricalDataService implements DataProviderInterface {
   public async getQuotes(
     aSymbols: string[]
   ): Promise<{ [symbol: string]: IDataProviderResponse }> {
-    if (aSymbols.length <= 0) {
+    const symbols = aSymbols.map((symbol) => {
+      return this.convertToEodSymbol(symbol);
+    });
+
+    if (symbols.length <= 0) {
       return {};
     }
 
     try {
       const get = bent(
-        `${this.URL}/real-time/${aSymbols[0]}?api_token=${
+        `${this.URL}/real-time/${symbols[0]}?api_token=${
           this.apiKey
-        }&fmt=json&s=${aSymbols.join(',')}`,
+        }&fmt=json&s=${symbols.join(',')}`,
         'GET',
         'json',
         200
       );
 
-      const [realTimeResponse, searchResponse] = await Promise.all([
-        get(),
-        this.search(aSymbols[0])
-      ]);
+      const realTimeResponse = await get();
 
       const quotes =
-        aSymbols.length === 1 ? [realTimeResponse] : realTimeResponse;
+        symbols.length === 1 ? [realTimeResponse] : realTimeResponse;
+
+      const searchResponse = await Promise.all(
+        symbols
+          .filter((symbol) => {
+            return !symbol.endsWith('.FOREX');
+          })
+          .map((symbol) => {
+            return this.search(symbol);
+          })
+      );
+
+      const lookupItems = searchResponse.flat().map(({ items }) => {
+        return items[0];
+      });
 
-      return quotes.reduce(
+      const response = quotes.reduce(
         (
           result: { [symbol: string]: IDataProviderResponse },
           { close, code, timestamp }
         ) => {
-          const currency = this.convertCurrency(
-            searchResponse?.items[0]?.currency
-          );
-
-          if (currency) {
-            result[code] = {
-              currency,
-              dataSource: DataSource.EOD_HISTORICAL_DATA,
-              marketPrice: close,
-              marketState: isToday(new Date(timestamp * 1000))
-                ? 'open'
-                : 'closed'
-            };
-          }
+          const currency = lookupItems.find((lookupItem) => {
+            return lookupItem.symbol === code;
+          })?.currency;
+
+          result[this.convertFromEodSymbol(code)] = {
+            currency: currency ?? this.baseCurrency,
+            dataSource: DataSource.EOD_HISTORICAL_DATA,
+            marketPrice: close,
+            marketState: isToday(new Date(timestamp * 1000)) ? 'open' : 'closed'
+          };
 
           return result;
         },
         {}
       );
+
+      if (response[`${this.baseCurrency}GBP`]) {
+        response[`${this.baseCurrency}GBp`] = {
+          ...response[`${this.baseCurrency}GBP`],
+          currency: `${this.baseCurrency}GBp`,
+          marketPrice: this.getConvertedValue({
+            symbol: `${this.baseCurrency}GBp`,
+            value: response[`${this.baseCurrency}GBP`].marketPrice
+          })
+        };
+      }
+
+      if (response[`${this.baseCurrency}ILS`]) {
+        response[`${this.baseCurrency}ILA`] = {
+          ...response[`${this.baseCurrency}ILS`],
+          currency: `${this.baseCurrency}ILA`,
+          marketPrice: this.getConvertedValue({
+            symbol: `${this.baseCurrency}ILA`,
+            value: response[`${this.baseCurrency}ILS`].marketPrice
+          })
+        };
+      }
+
+      return response;
     } catch (error) {
       Logger.error(error, 'EodHistoricalDataService');
     }
@@ -182,7 +225,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
     return {
       items: searchResult
         .filter(({ symbol }) => {
-          return !symbol.toLowerCase().endsWith('forex');
+          return !symbol.endsWith('.FOREX');
         })
         .map(
           ({
@@ -216,6 +259,60 @@ export class EodHistoricalDataService implements DataProviderInterface {
     return currency;
   }
 
+  private convertFromEodSymbol(aEodSymbol: string) {
+    let symbol = aEodSymbol;
+
+    if (symbol.endsWith('.FOREX')) {
+      symbol = symbol.replace('GBX', 'GBp');
+      symbol = symbol.replace('.FOREX', '');
+      symbol = `${this.baseCurrency}${symbol}`;
+    }
+
+    return symbol;
+  }
+
+  /**
+   * Converts a symbol to a EOD symbol
+   *
+   * Currency:  USDCHF  -> CHF.FOREX
+   */
+  private convertToEodSymbol(aSymbol: string) {
+    if (
+      aSymbol.startsWith(this.baseCurrency) &&
+      aSymbol.length > this.baseCurrency.length
+    ) {
+      if (
+        isCurrency(
+          aSymbol.substring(0, aSymbol.length - this.baseCurrency.length)
+        )
+      ) {
+        return `${aSymbol
+          .replace('GBp', 'GBX')
+          .replace(this.baseCurrency, '')}.FOREX`;
+      }
+    }
+
+    return aSymbol;
+  }
+
+  private getConvertedValue({
+    symbol,
+    value
+  }: {
+    symbol: string;
+    value: number;
+  }) {
+    if (symbol === `${this.baseCurrency}GBp`) {
+      // Convert GPB to GBp (pence)
+      return new Big(value).mul(100).toNumber();
+    } else if (symbol === `${this.baseCurrency}ILA`) {
+      // Convert ILS to ILA
+      return new Big(value).mul(100).toNumber();
+    }
+
+    return value;
+  }
+
   private async getSearchResult(aQuery: string): Promise<
     (LookupItem & {
       assetClass: AssetClass;

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@@ -203,9 +203,10 @@ export class YahooFinanceService implements DataProviderInterface {
           response[`${this.baseCurrency}GBp`] = {
             ...response[symbol],
             currency: 'GBp',
-            marketPrice: new Big(response[symbol].marketPrice)
-              .mul(100)
-              .toNumber()
+            marketPrice: this.getConvertedValue({
+              symbol: `${this.baseCurrency}GBp`,
+              value: response[symbol].marketPrice
+            })
           };
         } else if (
           symbol === `${this.baseCurrency}ILS` &&
@@ -215,9 +216,10 @@ export class YahooFinanceService implements DataProviderInterface {
           response[`${this.baseCurrency}ILA`] = {
             ...response[symbol],
             currency: 'ILA',
-            marketPrice: new Big(response[symbol].marketPrice)
-              .mul(100)
-              .toNumber()
+            marketPrice: this.getConvertedValue({
+              symbol: `${this.baseCurrency}ILA`,
+              value: response[symbol].marketPrice
+            })
           };
         } else if (
           symbol === `${this.baseCurrency}ZAR` &&
@@ -227,9 +229,10 @@ export class YahooFinanceService implements DataProviderInterface {
           response[`${this.baseCurrency}ZAc`] = {
             ...response[symbol],
             currency: 'ZAc',
-            marketPrice: new Big(response[symbol].marketPrice)
-              .mul(100)
-              .toNumber()
+            marketPrice: this.getConvertedValue({
+              symbol: `${this.baseCurrency}ZAc`,
+              value: response[symbol].marketPrice
+            })
           };
         }
       }

apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts

@@ -61,42 +61,41 @@ export class ExchangeRateDataService {
       getYesterday()
     );
 
-    // TODO: add fallback
-    /*if (Object.keys(result).length !== this.currencyPairs.length) {
+    if (Object.keys(result).length !== this.currencyPairs.length) {
       // Load currencies directly from data provider as a fallback
       // if historical data is not fully available
-      const historicalData = await this.dataProviderService.getQuotes(
+      const quotes = await this.dataProviderService.getQuotes(
         this.currencyPairs.map(({ dataSource, symbol }) => {
           return { dataSource, symbol };
         })
       );
 
-      Object.keys(historicalData).forEach((key) => {
-        if (isNumber(historicalData[key].marketPrice)) {
-          result[key] = {
+      for (const symbol of Object.keys(quotes)) {
+        if (isNumber(quotes[symbol].marketPrice)) {
+          result[symbol] = {
             [format(getYesterday(), DATE_FORMAT)]: {
-              marketPrice: historicalData[key].marketPrice
+              marketPrice: quotes[symbol].marketPrice
             }
           };
         }
-      });
-    }*/
+      }
+    }
 
     const resultExtended = result;
 
-    Object.keys(result).forEach((pair) => {
-      const [currency1, currency2] = pair.match(/.{1,3}/g);
-      const [date] = Object.keys(result[pair]);
+    for (const symbol of Object.keys(result)) {
+      const [currency1, currency2] = symbol.match(/.{1,3}/g);
+      const [date] = Object.keys(result[symbol]);
 
       // Calculate the opposite direction
       resultExtended[`${currency2}${currency1}`] = {
         [date]: {
-          marketPrice: 1 / result[pair][date].marketPrice
+          marketPrice: 1 / result[symbol][date].marketPrice
         }
       };
-    });
+    }
 
-    Object.keys(resultExtended).forEach((symbol) => {
+    for (const symbol of Object.keys(resultExtended)) {
       const [currency1, currency2] = symbol.match(/.{1,3}/g);
       const date = format(getYesterday(), DATE_FORMAT);
 
@@ -114,7 +113,7 @@ export class ExchangeRateDataService {
         this.exchangeRates[`${currency2}${currency1}`] =
           1 / this.exchangeRates[symbol];
       }
-    });
+    }
   }
 
   public toCurrency(
@@ -173,7 +172,8 @@ export class ExchangeRateDataService {
     let factor: number;
 
     if (aFromCurrency !== aToCurrency) {
-      const dataSource = this.dataProviderService.getPrimaryDataSource();
+      const dataSource =
+        this.dataProviderService.getDataSourceForExchangeRates();
       const symbol = `${aFromCurrency}${aToCurrency}`;
 
       const marketData = await this.marketDataService.get({
@@ -274,7 +274,7 @@ export class ExchangeRateDataService {
         return {
           currency1: this.baseCurrency,
           currency2: currency,
-          dataSource: this.dataProviderService.getPrimaryDataSource(),
+          dataSource: this.dataProviderService.getDataSourceForExchangeRates(),
           symbol: `${this.baseCurrency}${currency}`
         };
       });

apps/api/src/services/interfaces/environment.interface.ts

@@ -5,7 +5,8 @@ export interface Environment extends CleanedEnvAccessors {
   ALPHA_VANTAGE_API_KEY: string;
   BASE_CURRENCY: string;
   CACHE_TTL: number;
-  DATA_SOURCE_PRIMARY: string;
+  DATA_SOURCE_EXCHANGE_RATES: string;
+  DATA_SOURCE_IMPORT: string;
   DATA_SOURCES: string[];
   ENABLE_FEATURE_BLOG: boolean;
   ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;