diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index fd44f2426..7f10dc3a0 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -168,6 +168,7 @@ export class DataProviderService { const response: { [symbol: string]: IDataProviderResponse; } = {}; + const startTimeTotal = performance.now(); const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource); @@ -176,25 +177,59 @@ export class DataProviderService { for (const [dataSource, dataGatheringItems] of Object.entries( itemsGroupedByDataSource )) { + const dataProvider = this.getDataProvider(DataSource[dataSource]); + const symbols = dataGatheringItems.map((dataGatheringItem) => { return dataGatheringItem.symbol; }); - const promise = Promise.resolve( - this.getDataProvider(DataSource[dataSource]).getQuotes(symbols) - ); + const maximumNumberOfSymbolsPerRequest = + dataProvider.getMaxNumberOfSymbolsPerRequest?.() ?? + Number.MAX_SAFE_INTEGER; + for ( + let i = 0; + i < symbols.length; + i += maximumNumberOfSymbolsPerRequest + ) { + const startTimeDataSource = performance.now(); + + const symbolsChunk = symbols.slice( + i, + i + maximumNumberOfSymbolsPerRequest + ); - promises.push( - promise.then((result) => { - for (const [symbol, dataProviderResponse] of Object.entries(result)) { - response[symbol] = dataProviderResponse; - } - }) - ); + const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk)); + + promises.push( + promise.then((result) => { + for (const [symbol, dataProviderResponse] of Object.entries( + result + )) { + response[symbol] = dataProviderResponse; + } + + Logger.debug( + `Fetched ${symbolsChunk.length} quotes from ${dataSource} in ${( + (performance.now() - startTimeDataSource) / + 1000 + ).toFixed(3)} seconds` + ); + }) + ); + } } await Promise.all(promises); + Logger.debug('------------------------------------------------'); + Logger.debug( + `Fetched ${items.length} quotes in ${( + (performance.now() - startTimeTotal) / + 1000 + ).toFixed(3)} seconds` + ); + Logger.debug('================================================'); + return response; } diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts index 1dd3c7aff..c87c6ec3e 100644 --- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts +++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts @@ -81,6 +81,12 @@ export class EodHistoricalDataService implements DataProviderInterface { } } + public getMaxNumberOfSymbolsPerRequest() { + // It is not recommended using more than 15-20 tickers per request + // https://eodhistoricaldata.com/financial-apis/live-realtime-stocks-api + return 20; + } + public getName(): DataSource { return DataSource.EOD_HISTORICAL_DATA; } diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts index 16cf44603..6719f3099 100644 --- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts @@ -20,6 +20,8 @@ export interface DataProviderInterface { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }>; // TODO: Return only one symbol + getMaxNumberOfSymbolsPerRequest?(): number; + getName(): DataSource; getQuotes( diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index bddd529c2..d6a6a8412 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -208,6 +208,10 @@ export class YahooFinanceService implements DataProviderInterface { } } + public getMaxNumberOfSymbolsPerRequest() { + return 50; + } + public getName(): DataSource { return DataSource.YAHOO; }