From e500ccb61b10915a588d33434534a2558f93ff54 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 30 Apr 2023 18:26:34 +0200 Subject: [PATCH] Feature/introduce env variable data source exchange rates and data source import (#1910) * Introduce env variables DATA_SOURCE_EXCHANGE_RATES and DATA_SOURCE_IMPORT * Update changelog --- CHANGELOG.md | 6 + apps/api/src/app/import/import.service.ts | 7 +- .../configuration/configuration.service.ts | 3 +- .../data-provider/data-provider.service.ts | 86 +++++----- .../eod-historical-data.service.ts | 155 ++++++++++++++---- .../yahoo-finance/yahoo-finance.service.ts | 21 ++- .../exchange-rate-data.service.ts | 36 ++-- .../interfaces/environment.interface.ts | 3 +- 8 files changed, 216 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f93e48de..6974ddf5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Split the environment variable `DATA_SOURCE_PRIMARY` in `DATA_SOURCE_EXCHANGE_RATES` and `DATA_SOURCE_IMPORT` + ## 1.262.0 - 2023-04-29 ### Added diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index 7d0fda092..c3b8f63b3 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -15,7 +15,7 @@ import { OrderWithAccount } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; -import { Prisma, SymbolProfile } from '@prisma/client'; +import { DataSource, Prisma, SymbolProfile } from '@prisma/client'; import Big from 'big.js'; import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns'; import { v4 as uuidv4 } from 'uuid'; @@ -183,9 +183,10 @@ export class ImportService { for (const activity of activitiesDto) { if (!activity.dataSource) { if (activity.type === 'ITEM') { - activity.dataSource = 'MANUAL'; + activity.dataSource = DataSource.MANUAL; } else { - activity.dataSource = this.dataProviderService.getPrimaryDataSource(); + activity.dataSource = + this.dataProviderService.getDataSourceForImport(); } } diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts index df9d6fa24..eed51d1a8 100644 --- a/apps/api/src/services/configuration/configuration.service.ts +++ b/apps/api/src/services/configuration/configuration.service.ts @@ -16,7 +16,8 @@ export class ConfigurationService { default: 'USD' }), CACHE_TTL: num({ default: 1 }), - DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }), + DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }), + DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }), DATA_SOURCES: json({ default: [DataSource.COINGECKO, DataSource.MANUAL, DataSource.YAHOO] }), diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index db53d6341..472001db5 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -58,6 +58,52 @@ export class DataProviderService { return false; } + public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{ + [symbol: string]: Partial; + }> { + const response: { + [symbol: string]: Partial; + } = {}; + + const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource); + + const promises = []; + + for (const [dataSource, dataGatheringItems] of Object.entries( + itemsGroupedByDataSource + )) { + const symbols = dataGatheringItems.map((dataGatheringItem) => { + return dataGatheringItem.symbol; + }); + + for (const symbol of symbols) { + const promise = Promise.resolve( + this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol) + ); + + promises.push( + promise.then((symbolProfile) => { + response[symbol] = symbolProfile; + }) + ); + } + } + + await Promise.all(promises); + + return response; + } + + public getDataSourceForExchangeRates(): DataSource { + return DataSource[ + this.configurationService.get('DATA_SOURCE_EXCHANGE_RATES') + ]; + } + + public getDataSourceForImport(): DataSource { + return DataSource[this.configurationService.get('DATA_SOURCE_IMPORT')]; + } + public async getDividends({ dataSource, from, @@ -182,46 +228,6 @@ export class DataProviderService { return result; } - public getPrimaryDataSource(): DataSource { - return DataSource[this.configurationService.get('DATA_SOURCE_PRIMARY')]; - } - - public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{ - [symbol: string]: Partial; - }> { - const response: { - [symbol: string]: Partial; - } = {}; - - const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource); - - const promises = []; - - for (const [dataSource, dataGatheringItems] of Object.entries( - itemsGroupedByDataSource - )) { - const symbols = dataGatheringItems.map((dataGatheringItem) => { - return dataGatheringItem.symbol; - }); - - for (const symbol of symbols) { - const promise = Promise.resolve( - this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol) - ); - - promises.push( - promise.then((symbolProfile) => { - response[symbol] = symbolProfile; - }) - ); - } - } - - await Promise.all(promises); - - return response; - } - public async getQuotes(items: IDataGatheringItem[]): Promise<{ [symbol: string]: IDataProviderResponse; }> { diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts index 614ec46a8..f1b56f659 100644 --- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts +++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts @@ -5,7 +5,7 @@ import { IDataProviderHistoricalResponse, IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; -import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { DATE_FORMAT, isCurrency } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; import { @@ -15,17 +15,20 @@ import { SymbolProfile } from '@prisma/client'; import bent from 'bent'; +import Big from 'big.js'; import { format, isToday } from 'date-fns'; @Injectable() export class EodHistoricalDataService implements DataProviderInterface { private apiKey: string; + private baseCurrency: string; private readonly URL = 'https://eodhistoricaldata.com/api'; public constructor( private readonly configurationService: ConfigurationService ) { this.apiKey = this.configurationService.get('EOD_HISTORICAL_DATA_API_KEY'); + this.baseCurrency = this.configurationService.get('BASE_CURRENCY'); } public canHandle(symbol: string) { @@ -70,9 +73,11 @@ export class EodHistoricalDataService implements DataProviderInterface { ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { + const symbol = this.convertToEodSymbol(aSymbol); + try { const get = bent( - `${this.URL}/eod/${aSymbol}?api_token=${ + `${this.URL}/eod/${symbol}?api_token=${ this.apiKey }&fmt=json&from=${format(from, DATE_FORMAT)}&to=${format( to, @@ -87,14 +92,17 @@ export class EodHistoricalDataService implements DataProviderInterface { return response.reduce( (result, historicalItem, index, array) => { - result[aSymbol][historicalItem.date] = { - marketPrice: historicalItem.close, + result[this.convertFromEodSymbol(symbol)][historicalItem.date] = { + marketPrice: this.getConvertedValue({ + symbol: aSymbol, + value: historicalItem.close + }), performance: historicalItem.open - historicalItem.close }; return result; }, - { [aSymbol]: {} } + { [this.convertFromEodSymbol(symbol)]: {} } ); } catch (error) { throw new Error( @@ -119,52 +127,87 @@ export class EodHistoricalDataService implements DataProviderInterface { public async getQuotes( aSymbols: string[] ): Promise<{ [symbol: string]: IDataProviderResponse }> { - if (aSymbols.length <= 0) { + const symbols = aSymbols.map((symbol) => { + return this.convertToEodSymbol(symbol); + }); + + if (symbols.length <= 0) { return {}; } try { const get = bent( - `${this.URL}/real-time/${aSymbols[0]}?api_token=${ + `${this.URL}/real-time/${symbols[0]}?api_token=${ this.apiKey - }&fmt=json&s=${aSymbols.join(',')}`, + }&fmt=json&s=${symbols.join(',')}`, 'GET', 'json', 200 ); - const [realTimeResponse, searchResponse] = await Promise.all([ - get(), - this.search(aSymbols[0]) - ]); + const realTimeResponse = await get(); const quotes = - aSymbols.length === 1 ? [realTimeResponse] : realTimeResponse; + symbols.length === 1 ? [realTimeResponse] : realTimeResponse; + + const searchResponse = await Promise.all( + symbols + .filter((symbol) => { + return !symbol.endsWith('.FOREX'); + }) + .map((symbol) => { + return this.search(symbol); + }) + ); + + const lookupItems = searchResponse.flat().map(({ items }) => { + return items[0]; + }); - return quotes.reduce( + const response = quotes.reduce( ( result: { [symbol: string]: IDataProviderResponse }, { close, code, timestamp } ) => { - const currency = this.convertCurrency( - searchResponse?.items[0]?.currency - ); - - if (currency) { - result[code] = { - currency, - dataSource: DataSource.EOD_HISTORICAL_DATA, - marketPrice: close, - marketState: isToday(new Date(timestamp * 1000)) - ? 'open' - : 'closed' - }; - } + const currency = lookupItems.find((lookupItem) => { + return lookupItem.symbol === code; + })?.currency; + + result[this.convertFromEodSymbol(code)] = { + currency: currency ?? this.baseCurrency, + dataSource: DataSource.EOD_HISTORICAL_DATA, + marketPrice: close, + marketState: isToday(new Date(timestamp * 1000)) ? 'open' : 'closed' + }; return result; }, {} ); + + if (response[`${this.baseCurrency}GBP`]) { + response[`${this.baseCurrency}GBp`] = { + ...response[`${this.baseCurrency}GBP`], + currency: `${this.baseCurrency}GBp`, + marketPrice: this.getConvertedValue({ + symbol: `${this.baseCurrency}GBp`, + value: response[`${this.baseCurrency}GBP`].marketPrice + }) + }; + } + + if (response[`${this.baseCurrency}ILS`]) { + response[`${this.baseCurrency}ILA`] = { + ...response[`${this.baseCurrency}ILS`], + currency: `${this.baseCurrency}ILA`, + marketPrice: this.getConvertedValue({ + symbol: `${this.baseCurrency}ILA`, + value: response[`${this.baseCurrency}ILS`].marketPrice + }) + }; + } + + return response; } catch (error) { Logger.error(error, 'EodHistoricalDataService'); } @@ -182,7 +225,7 @@ export class EodHistoricalDataService implements DataProviderInterface { return { items: searchResult .filter(({ symbol }) => { - return !symbol.toLowerCase().endsWith('forex'); + return !symbol.endsWith('.FOREX'); }) .map( ({ @@ -216,6 +259,60 @@ export class EodHistoricalDataService implements DataProviderInterface { return currency; } + private convertFromEodSymbol(aEodSymbol: string) { + let symbol = aEodSymbol; + + if (symbol.endsWith('.FOREX')) { + symbol = symbol.replace('GBX', 'GBp'); + symbol = symbol.replace('.FOREX', ''); + symbol = `${this.baseCurrency}${symbol}`; + } + + return symbol; + } + + /** + * Converts a symbol to a EOD symbol + * + * Currency: USDCHF -> CHF.FOREX + */ + private convertToEodSymbol(aSymbol: string) { + if ( + aSymbol.startsWith(this.baseCurrency) && + aSymbol.length > this.baseCurrency.length + ) { + if ( + isCurrency( + aSymbol.substring(0, aSymbol.length - this.baseCurrency.length) + ) + ) { + return `${aSymbol + .replace('GBp', 'GBX') + .replace(this.baseCurrency, '')}.FOREX`; + } + } + + return aSymbol; + } + + private getConvertedValue({ + symbol, + value + }: { + symbol: string; + value: number; + }) { + if (symbol === `${this.baseCurrency}GBp`) { + // Convert GPB to GBp (pence) + return new Big(value).mul(100).toNumber(); + } else if (symbol === `${this.baseCurrency}ILA`) { + // Convert ILS to ILA + return new Big(value).mul(100).toNumber(); + } + + return value; + } + private async getSearchResult(aQuery: string): Promise< (LookupItem & { assetClass: AssetClass; diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index 9d20d9493..0671cf8c8 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -203,9 +203,10 @@ export class YahooFinanceService implements DataProviderInterface { response[`${this.baseCurrency}GBp`] = { ...response[symbol], currency: 'GBp', - marketPrice: new Big(response[symbol].marketPrice) - .mul(100) - .toNumber() + marketPrice: this.getConvertedValue({ + symbol: `${this.baseCurrency}GBp`, + value: response[symbol].marketPrice + }) }; } else if ( symbol === `${this.baseCurrency}ILS` && @@ -215,9 +216,10 @@ export class YahooFinanceService implements DataProviderInterface { response[`${this.baseCurrency}ILA`] = { ...response[symbol], currency: 'ILA', - marketPrice: new Big(response[symbol].marketPrice) - .mul(100) - .toNumber() + marketPrice: this.getConvertedValue({ + symbol: `${this.baseCurrency}ILA`, + value: response[symbol].marketPrice + }) }; } else if ( symbol === `${this.baseCurrency}ZAR` && @@ -227,9 +229,10 @@ export class YahooFinanceService implements DataProviderInterface { response[`${this.baseCurrency}ZAc`] = { ...response[symbol], currency: 'ZAc', - marketPrice: new Big(response[symbol].marketPrice) - .mul(100) - .toNumber() + marketPrice: this.getConvertedValue({ + symbol: `${this.baseCurrency}ZAc`, + value: response[symbol].marketPrice + }) }; } } diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts index 10aef9411..e39e926a9 100644 --- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts @@ -61,42 +61,41 @@ export class ExchangeRateDataService { getYesterday() ); - // TODO: add fallback - /*if (Object.keys(result).length !== this.currencyPairs.length) { + if (Object.keys(result).length !== this.currencyPairs.length) { // Load currencies directly from data provider as a fallback // if historical data is not fully available - const historicalData = await this.dataProviderService.getQuotes( + const quotes = await this.dataProviderService.getQuotes( this.currencyPairs.map(({ dataSource, symbol }) => { return { dataSource, symbol }; }) ); - Object.keys(historicalData).forEach((key) => { - if (isNumber(historicalData[key].marketPrice)) { - result[key] = { + for (const symbol of Object.keys(quotes)) { + if (isNumber(quotes[symbol].marketPrice)) { + result[symbol] = { [format(getYesterday(), DATE_FORMAT)]: { - marketPrice: historicalData[key].marketPrice + marketPrice: quotes[symbol].marketPrice } }; } - }); - }*/ + } + } const resultExtended = result; - Object.keys(result).forEach((pair) => { - const [currency1, currency2] = pair.match(/.{1,3}/g); - const [date] = Object.keys(result[pair]); + for (const symbol of Object.keys(result)) { + const [currency1, currency2] = symbol.match(/.{1,3}/g); + const [date] = Object.keys(result[symbol]); // Calculate the opposite direction resultExtended[`${currency2}${currency1}`] = { [date]: { - marketPrice: 1 / result[pair][date].marketPrice + marketPrice: 1 / result[symbol][date].marketPrice } }; - }); + } - Object.keys(resultExtended).forEach((symbol) => { + for (const symbol of Object.keys(resultExtended)) { const [currency1, currency2] = symbol.match(/.{1,3}/g); const date = format(getYesterday(), DATE_FORMAT); @@ -114,7 +113,7 @@ export class ExchangeRateDataService { this.exchangeRates[`${currency2}${currency1}`] = 1 / this.exchangeRates[symbol]; } - }); + } } public toCurrency( @@ -173,7 +172,8 @@ export class ExchangeRateDataService { let factor: number; if (aFromCurrency !== aToCurrency) { - const dataSource = this.dataProviderService.getPrimaryDataSource(); + const dataSource = + this.dataProviderService.getDataSourceForExchangeRates(); const symbol = `${aFromCurrency}${aToCurrency}`; const marketData = await this.marketDataService.get({ @@ -274,7 +274,7 @@ export class ExchangeRateDataService { return { currency1: this.baseCurrency, currency2: currency, - dataSource: this.dataProviderService.getPrimaryDataSource(), + dataSource: this.dataProviderService.getDataSourceForExchangeRates(), symbol: `${this.baseCurrency}${currency}` }; }); diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index b8f05e98c..586f19997 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -5,7 +5,8 @@ export interface Environment extends CleanedEnvAccessors { ALPHA_VANTAGE_API_KEY: string; BASE_CURRENCY: string; CACHE_TTL: number; - DATA_SOURCE_PRIMARY: string; + DATA_SOURCE_EXCHANGE_RATES: string; + DATA_SOURCE_IMPORT: string; DATA_SOURCES: string[]; ENABLE_FEATURE_BLOG: boolean; ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;