From 97961a78537398ed0fd262a478f328015e49268f Mon Sep 17 00:00:00 2001 From: Valentin Zickner <3200232+vzickner@users.noreply.github.com> Date: Thu, 27 May 2021 20:33:59 +0200 Subject: [PATCH] optimize fetching from multiple data sources (#123) * optimize fetching from multiple data sources * improve performance by executing data gathering promises in parallel * removed unused imports * rename hasHistoricalData to canHandle --- .../api/src/services/data-provider.service.ts | 80 ++++++------------- .../alpha-vantage/alpha-vantage.service.ts | 4 + .../ghostfolio-scraper-api.service.ts | 9 ++- .../rakuten-rapid-api.service.ts | 13 ++- .../yahoo-finance/yahoo-finance.service.ts | 4 + .../interfaces/data-provider.interface.ts | 2 + 6 files changed, 55 insertions(+), 57 deletions(-) diff --git a/apps/api/src/services/data-provider.service.ts b/apps/api/src/services/data-provider.service.ts index c38cb5925..4a6936def 100644 --- a/apps/api/src/services/data-provider.service.ts +++ b/apps/api/src/services/data-provider.service.ts @@ -1,6 +1,4 @@ -import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { - isCrypto, isGhostfolioScraperApiSymbol, isRakutenRapidApiSymbol } from '@ghostfolio/common/helper'; @@ -132,73 +130,45 @@ export class DataProviderService implements DataProviderInterface { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; } = {}; + const promises: Promise<{ + data: { [date: string]: IDataProviderHistoricalResponse }; + symbol: string; + }>[] = []; for (const { dataSource, symbol } of aDataGatheringItems) { - switch (dataSource) { - case DataSource.ALPHA_VANTAGE: { - if (this.configurationService.get('ALPHA_VANTAGE_API_KEY')) { - const data = await this.alphaVantageService.getHistorical( - [symbol], - undefined, - from, - to - ); - result[symbol] = data?.[symbol]; - } - break; - } - case DataSource.GHOSTFOLIO: { - if (isGhostfolioScraperApiSymbol(symbol)) { - const data = await this.ghostfolioScraperApiService.getHistorical( - [symbol], - undefined, - from, - to - ); - result[symbol] = data?.[symbol]; - } - break; - } - case DataSource.RAKUTEN: { - if ( - isRakutenRapidApiSymbol(symbol) && - this.configurationService.get('RAKUTEN_RAPID_API_KEY') - ) { - const data = await this.rakutenRapidApiService.getHistorical( - [symbol], - undefined, - from, - to - ); - result[symbol] = data?.[symbol]; - } - break; - } - case DataSource.YAHOO: { - const data = await this.yahooFinanceService.getHistorical( - [symbol], - undefined, - from, - to - ); - result[symbol] = data?.[symbol]; - break; - } + const dataProvider = this.getDataProvider(dataSource); + if (dataProvider.canHandle(symbol)) { + promises.push( + dataProvider + .getHistorical([symbol], undefined, from, to) + .then((data) => ({ data: data?.[symbol], symbol })) + ); } } + const allData = await Promise.all(promises); + for (const { data, symbol } of allData) { + result[symbol] = data; + } + return result; } public async search(aSymbol: string) { - return this.getDataProvider().search(aSymbol); + return this.getDataProvider( + this.configurationService.get('DATA_SOURCES')[0] + ).search(aSymbol); } - private getDataProvider() { - switch (this.configurationService.get('DATA_SOURCES')[0]) { + private getDataProvider(providerName: DataSource) { + switch (providerName) { case DataSource.ALPHA_VANTAGE: return this.alphaVantageService; case DataSource.YAHOO: return this.yahooFinanceService; + case DataSource.RAKUTEN: + return this.rakutenRapidApiService; + case DataSource.GHOSTFOLIO: + return this.ghostfolioScraperApiService; default: throw new Error('No data provider has been found.'); } diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts index 3046737e5..64e0b6c99 100644 --- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts +++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts @@ -24,6 +24,10 @@ export class AlphaVantageService implements DataProviderInterface { }); } + public canHandle(symbol: string) { + return this.configurationService.get('ALPHA_VANTAGE_API_KEY'); + } + public async get( aSymbols: string[] ): Promise<{ [symbol: string]: IDataProviderResponse }> { diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts index b6423f4a2..8b8e4290d 100644 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts @@ -1,4 +1,7 @@ -import { getYesterday } from '@ghostfolio/common/helper'; +import { + getYesterday, + isGhostfolioScraperApiSymbol +} from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; @@ -21,6 +24,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface { public constructor(private prisma: PrismaService) {} + public canHandle(symbol: string) { + return isGhostfolioScraperApiSymbol(symbol); + } + public async get( aSymbols: string[] ): Promise<{ [symbol: string]: IDataProviderResponse }> { diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts index 9f3943a8e..7af775f8b 100644 --- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts @@ -1,4 +1,8 @@ -import { getToday, getYesterday } from '@ghostfolio/common/helper'; +import { + getToday, + getYesterday, + isRakutenRapidApiSymbol +} from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; @@ -24,6 +28,13 @@ export class RakutenRapidApiService implements DataProviderInterface { private readonly configurationService: ConfigurationService ) {} + public canHandle(symbol: string) { + return ( + isRakutenRapidApiSymbol(symbol) && + this.configurationService.get('RAKUTEN_RAPID_API_KEY') + ); + } + public async get( aSymbols: string[] ): Promise<{ [symbol: string]: IDataProviderResponse }> { diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index 19a969cbd..b22d7e7cf 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -28,6 +28,10 @@ export class YahooFinanceService implements DataProviderInterface { public constructor() {} + public canHandle(symbol: string) { + return true; + } + public async get( aSymbols: string[] ): Promise<{ [symbol: string]: IDataProviderResponse }> { diff --git a/apps/api/src/services/interfaces/data-provider.interface.ts b/apps/api/src/services/interfaces/data-provider.interface.ts index cf861ad7b..b36be4cb4 100644 --- a/apps/api/src/services/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/interfaces/data-provider.interface.ts @@ -7,6 +7,8 @@ import { } from './interfaces'; export interface DataProviderInterface { + canHandle(symbol: string): boolean; + get(aSymbols: string[]): Promise<{ [symbol: string]: IDataProviderResponse }>; getHistorical(