From fa47133f0d0cab847b2d41ef293027b85764e94d Mon Sep 17 00:00:00 2001 From: Thomas <4159106+dtslvr@users.noreply.github.com> Date: Wed, 23 Feb 2022 21:06:48 +0100 Subject: [PATCH] Migrate to yahoo-finance2 --- .../alpha-vantage/alpha-vantage.service.ts | 8 +-- .../data-provider/data-provider.service.ts | 2 +- .../ghostfolio-scraper-api.service.ts | 9 ++-- .../google-sheets/google-sheets.service.ts | 8 +-- .../interfaces/data-provider.interface.ts | 4 +- .../data-provider/manual/manual.service.ts | 2 +- .../rakuten-rapid-api.service.ts | 8 +-- .../yahoo-finance/interfaces/interfaces.ts | 11 ---- .../yahoo-finance/yahoo-finance.service.ts | 52 ++++++++----------- package.json | 1 + yarn.lock | 28 +++++++++- 11 files changed, 64 insertions(+), 69 deletions(-) diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts index 838f1ae6e..b0a1cc667 100644 --- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts +++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts @@ -36,18 +36,14 @@ export class AlphaVantageService implements DataProviderInterface { } public async getHistorical( - aSymbols: string[], + aSymbol: string, aGranularity: Granularity = 'day', from: Date, to: Date ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { - if (aSymbols.length <= 0) { - return {}; - } - - const symbol = aSymbols[0]; + const symbol = aSymbol; try { const historicalData: { diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 6ffd5b2dd..4281c3cb8 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -144,7 +144,7 @@ export class DataProviderService { if (dataProvider.canHandle(symbol)) { promises.push( dataProvider - .getHistorical([symbol], undefined, from, to) + .getHistorical(symbol, undefined, from, to) .then((data) => ({ data: data?.[symbol], symbol })) ); } diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts index a34f7cf92..a75dae7fb 100644 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts @@ -70,19 +70,16 @@ export class GhostfolioScraperApiService implements DataProviderInterface { } public async getHistorical( - aSymbols: string[], + aSymbol: string, aGranularity: Granularity = 'day', from: Date, to: Date ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { - if (aSymbols.length <= 0) { - return {}; - } - try { - const [symbol] = aSymbols; + const symbol = aSymbol; + const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles( [symbol] ); diff --git a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts index fff9db21e..079b023a3 100644 --- a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts +++ b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts @@ -73,19 +73,15 @@ export class GoogleSheetsService implements DataProviderInterface { } public async getHistorical( - aSymbols: string[], + aSymbol: string, aGranularity: Granularity = 'day', from: Date, to: Date ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { - if (aSymbols.length <= 0) { - return {}; - } - try { - const [symbol] = aSymbols; + const symbol = aSymbol; const sheet = await this.getSheet({ symbol, diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts index c5cf4c330..cdcdb3bd4 100644 --- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts @@ -12,13 +12,13 @@ export interface DataProviderInterface { get(aSymbols: string[]): Promise<{ [symbol: string]: IDataProviderResponse }>; getHistorical( - aSymbols: string[], + aSymbol: string, aGranularity: Granularity, from: Date, to: Date ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; - }>; + }>; // TODO: Return only one symbol getName(): DataSource; diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 3a486f897..b1407c932 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -23,7 +23,7 @@ export class ManualService implements DataProviderInterface { } public async getHistorical( - aSymbols: string[], + aSymbol: string, aGranularity: Granularity = 'day', from: Date, to: Date diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts index bdfc147dd..366ef6c97 100644 --- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts @@ -60,19 +60,15 @@ export class RakutenRapidApiService implements DataProviderInterface { } public async getHistorical( - aSymbols: string[], + aSymbol: string, aGranularity: Granularity = 'day', from: Date, to: Date ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { - if (aSymbols.length <= 0) { - return {}; - } - try { - const symbol = aSymbols[0]; + const symbol = aSymbol; if (symbol === ghostfolioFearAndGreedIndexSymbol) { const fgi = await this.getFearAndGreedIndex(); diff --git a/apps/api/src/services/data-provider/yahoo-finance/interfaces/interfaces.ts b/apps/api/src/services/data-provider/yahoo-finance/interfaces/interfaces.ts index d41a43d39..8f917f462 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/interfaces/interfaces.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/interfaces/interfaces.ts @@ -1,14 +1,3 @@ -export interface IYahooFinanceHistoricalResponse { - adjClose: number; - close: number; - date: Date; - high: number; - low: number; - open: number; - symbol: string; - volume: number; -} - export interface IYahooFinanceQuoteResponse { price: IYahooFinancePrice; summaryProfile: IYahooFinanceSummaryProfile; diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index 556cf71a1..1492c84b0 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -10,6 +10,7 @@ import Big from 'big.js'; import { countries } from 'countries-list'; import { addDays, format, isSameDay } from 'date-fns'; import * as yahooFinance from 'yahoo-finance'; +import yahooFinance2 from 'yahoo-finance2'; import { IDataProviderHistoricalResponse, @@ -18,7 +19,6 @@ import { } from '../../interfaces/interfaces'; import { DataProviderInterface } from '../interfaces/data-provider.interface'; import { - IYahooFinanceHistoricalResponse, IYahooFinancePrice, IYahooFinanceQuoteResponse } from './interfaces/interfaces'; @@ -162,56 +162,50 @@ export class YahooFinanceService implements DataProviderInterface { } public async getHistorical( - aSymbols: string[], + aSymbol: string, aGranularity: Granularity = 'day', from: Date, to: Date ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { - if (aSymbols.length <= 0) { - return {}; - } - if (isSameDay(from, to)) { to = addDays(to, 1); } - const yahooFinanceSymbols = aSymbols.map((symbol) => { - return this.convertToYahooFinanceSymbol(symbol); - }); + const yahooFinanceSymbol = this.convertToYahooFinanceSymbol(aSymbol); try { - const historicalData: { - [symbol: string]: IYahooFinanceHistoricalResponse[]; - } = await yahooFinance.historical({ - symbols: yahooFinanceSymbols, - from: format(from, DATE_FORMAT), - to: format(to, DATE_FORMAT) - }); + const historicalResult = await yahooFinance2.historical( + yahooFinanceSymbol, + { + interval: '1d', + period1: format(from, DATE_FORMAT), + period2: format(to, DATE_FORMAT) + } + ); const response: { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; } = {}; - for (const [yahooFinanceSymbol, timeSeries] of Object.entries( - historicalData - )) { - // Convert symbols back - const symbol = this.convertFromYahooFinanceSymbol(yahooFinanceSymbol); - response[symbol] = {}; + // Convert symbol back + const symbol = this.convertFromYahooFinanceSymbol(yahooFinanceSymbol); - timeSeries.forEach((timeSerie) => { - response[symbol][format(timeSerie.date, DATE_FORMAT)] = { - marketPrice: timeSerie.close, - performance: timeSerie.open - timeSerie.close - }; - }); + response[symbol] = {}; + + for (const historicalItem of historicalResult) { + response[symbol][format(historicalItem.date, DATE_FORMAT)] = { + marketPrice: historicalItem.close, + performance: historicalItem.open - historicalItem.close + }; } return response; } catch (error) { - Logger.error(error); + Logger.warn( + `Skipping yahooFinance2.getHistorical("${aSymbol}"): [${error.name}] ${error.message}` + ); return {}; } diff --git a/package.json b/package.json index 7f3a7f8a3..f4d64ad38 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,7 @@ "twitter-api-v2": "1.10.3", "uuid": "8.3.2", "yahoo-finance": "0.3.6", + "yahoo-finance2": "2.1.5", "zone.js": "0.11.4" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 756eca7bd..b95479030 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5720,6 +5720,16 @@ ajv-keywords@^5.0.0: dependencies: fast-deep-equal "^3.1.3" +ajv@8.10.0: + version "8.10.0" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.10.0.tgz#e573f719bd3af069017e3b66538ab968d040e54d" + integrity sha512-bzqAEZOjkrUMl2afH8dknrq5KEk2SrwdBROR+vH1EKVQTqaUbJVPdc/gEdggTMM0Se+s+Ja4ju4TlNcStKl2Hw== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + uri-js "^4.2.2" + ajv@8.6.3: version "8.6.3" resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.6.3.tgz#11a66527761dc3e9a3845ea775d2d3c0414e8764" @@ -13507,7 +13517,14 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== -moment-timezone@^0.5.10, moment-timezone@^0.5.x: +moment-timezone@^0.5.10: + version "0.5.34" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.34.tgz#a75938f7476b88f155d3504a9343f7519d9a405c" + integrity sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg== + dependencies: + moment ">= 2.9.0" + +moment-timezone@^0.5.x: version "0.5.33" resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.33.tgz#b252fd6bb57f341c9b59a5ab61a8e51a73bbd22c" integrity sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w== @@ -18764,6 +18781,15 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yahoo-finance2@2.1.5: + version "2.1.5" + resolved "https://registry.yarnpkg.com/yahoo-finance2/-/yahoo-finance2-2.1.5.tgz#f0ff4025b56d4b9c63a3edfd3b28df292c171e90" + integrity sha512-yN+BvaJ1IRWoOaZkZMHpH6Ll2NtKGeC9RRFDo6FcrC7llkCkL0RV/Fcvd4jbMIZ6pOhuZqsnxF0Mm7h5EZQIqg== + dependencies: + ajv "8.10.0" + ajv-formats "2.1.1" + node-fetch "^2.6.1" + yahoo-finance@0.3.6: version "0.3.6" resolved "https://registry.yarnpkg.com/yahoo-finance/-/yahoo-finance-0.3.6.tgz#c99fe8ff6c9a80babbb7e75881a244a862f6739f"