From 57fd9bf091c6fab65c76f49a67571fb897bbc5d9 Mon Sep 17 00:00:00 2001 From: Thomas <4159106+dtslvr@users.noreply.github.com> Date: Wed, 19 May 2021 22:05:16 +0200 Subject: [PATCH] Respect data source in data gathering --- .../experimental/experimental.controller.ts | 4 +- apps/api/src/app/order/order.service.ts | 4 +- .../src/app/portfolio/portfolio.service.ts | 3 +- .../src/services/data-gathering.service.ts | 61 ++++++---- .../api/src/services/data-provider.service.ts | 109 +++++++++--------- .../api/src/services/interfaces/interfaces.ts | 6 + libs/common/src/lib/config.ts | 15 ++- 7 files changed, 116 insertions(+), 86 deletions(-) diff --git a/apps/api/src/app/experimental/experimental.controller.ts b/apps/api/src/app/experimental/experimental.controller.ts index cdddcee4e..f8535ac57 100644 --- a/apps/api/src/app/experimental/experimental.controller.ts +++ b/apps/api/src/app/experimental/experimental.controller.ts @@ -37,7 +37,9 @@ export class ExperimentalController { ); } - return benchmarks; + return benchmarks.map(({ symbol }) => { + return symbol; + }); } @Get('benchmarks/:symbol') diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 5f772375b..b933b054f 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -2,7 +2,7 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.se import { PrismaService } from '@ghostfolio/api/services/prisma.service'; import { OrderWithAccount } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; -import { Order, Prisma } from '@prisma/client'; +import { DataSource, Order, Prisma } from '@prisma/client'; import { CacheService } from '../cache/cache.service'; import { RedisCacheService } from '../redis-cache/redis-cache.service'; @@ -53,6 +53,7 @@ export class OrderService { // Gather symbol data of order in the background this.dataGatheringService.gatherSymbols([ { + dataSource: data.dataSource, date: data.date, symbol: data.symbol } @@ -90,6 +91,7 @@ export class OrderService { // Gather symbol data of order in the background this.dataGatheringService.gatherSymbols([ { + dataSource: data.dataSource, date: data.date, symbol: data.symbol } diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index 6ce441907..e1493cbd6 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -11,6 +11,7 @@ import { import { DateRange, RequestWithUser } from '@ghostfolio/common/types'; import { Inject, Injectable } from '@nestjs/common'; import { REQUEST } from '@nestjs/core'; +import { DataSource } from '@prisma/client'; import { add, format, @@ -289,7 +290,7 @@ export class PortfolioService { if (isEmpty(historicalData)) { historicalData = await this.dataProviderService.getHistoricalRaw( - [aSymbol], + [{ dataSource: DataSource.YAHOO, symbol: aSymbol }], portfolio.getMinDate(), new Date() ); diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index df84076ab..96ba41735 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -5,6 +5,7 @@ import { resetHours } from '@ghostfolio/common/helper'; import { Injectable } from '@nestjs/common'; +import { DataSource } from '@prisma/client'; import { differenceInHours, format, @@ -18,6 +19,7 @@ import { import { ConfigurationService } from './configuration.service'; import { DataProviderService } from './data-provider.service'; import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; +import { IDataGatheringItem } from './interfaces/interfaces'; import { PrismaService } from './prisma.service'; @Injectable() @@ -115,15 +117,13 @@ export class DataGatheringService { } } - public async gatherSymbols( - aSymbolsWithStartDate: { date: Date; symbol: string }[] - ) { + public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) { let hasError = false; - for (const { date, symbol } of aSymbolsWithStartDate) { + for (const { dataSource, date, symbol } of aSymbolsWithStartDate) { try { const historicalData = await this.dataProviderService.getHistoricalRaw( - [symbol], + [{ dataSource, symbol }], date, new Date() ); @@ -184,20 +184,24 @@ export class DataGatheringService { } } - public async getCustomSymbolsToGather(startDate?: Date) { + public async getCustomSymbolsToGather( + startDate?: Date + ): Promise { const scraperConfigurations = await this.ghostfolioScraperApi.getScraperConfigurations(); return scraperConfigurations.map((scraperConfiguration) => { return { + dataSource: DataSource.GHOSTFOLIO, date: startDate, symbol: scraperConfiguration.symbol }; }); } - private getBenchmarksToGather(startDate: Date) { - const benchmarksToGather = benchmarks.map((symbol) => { + private getBenchmarksToGather(startDate: Date): IDataGatheringItem[] { + const benchmarksToGather = benchmarks.map(({ dataSource, symbol }) => { return { + dataSource, symbol, date: startDate }; @@ -205,6 +209,7 @@ export class DataGatheringService { if (this.configurationService.get('ENABLE_FEATURE_FEAR_AND_GREED_INDEX')) { benchmarksToGather.push({ + dataSource: DataSource.RAKUTEN, date: startDate, symbol: 'GF.FEAR_AND_GREED_INDEX' }); @@ -213,16 +218,16 @@ export class DataGatheringService { return benchmarksToGather; } - private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> { + private async getSymbols7D(): Promise { const startDate = subDays(resetHours(new Date()), 7); const distinctOrders = await this.prisma.order.findMany({ distinct: ['symbol'], orderBy: [{ symbol: 'asc' }], - select: { symbol: true } + select: { dataSource: true, symbol: true } }); - const distinctOrdersWithDate = distinctOrders + const distinctOrdersWithDate: IDataGatheringItem[] = distinctOrders .filter((distinctOrder) => { return !isGhostfolioScraperApiSymbol(distinctOrder.symbol); }) @@ -233,12 +238,15 @@ export class DataGatheringService { }; }); - const currencyPairsToGather = currencyPairs.map((symbol) => { - return { - symbol, - date: startDate - }; - }); + const currencyPairsToGather = currencyPairs.map( + ({ dataSource, symbol }) => { + return { + dataSource, + symbol, + date: startDate + }; + } + ); const customSymbolsToGather = await this.getCustomSymbolsToGather( startDate @@ -252,24 +260,27 @@ export class DataGatheringService { ]; } - private async getSymbolsMax() { + private async getSymbolsMax(): Promise { const startDate = new Date(getUtc('2015-01-01')); const customSymbolsToGather = await this.getCustomSymbolsToGather( startDate ); - const currencyPairsToGather = currencyPairs.map((symbol) => { - return { - symbol, - date: startDate - }; - }); + const currencyPairsToGather = currencyPairs.map( + ({ dataSource, symbol }) => { + return { + dataSource, + symbol, + date: startDate + }; + } + ); const distinctOrders = await this.prisma.order.findMany({ distinct: ['symbol'], orderBy: [{ date: 'asc' }], - select: { date: true, symbol: true } + select: { dataSource: true, date: true, symbol: true } }); return [ diff --git a/apps/api/src/services/data-provider.service.ts b/apps/api/src/services/data-provider.service.ts index 2183a514f..c38cb5925 100644 --- a/apps/api/src/services/data-provider.service.ts +++ b/apps/api/src/services/data-provider.service.ts @@ -16,6 +16,7 @@ import { RakutenRapidApiService } from './data-provider/rakuten-rapid-api/rakute import { YahooFinanceService } from './data-provider/yahoo-finance/yahoo-finance.service'; import { DataProviderInterface } from './interfaces/data-provider.interface'; import { + IDataGatheringItem, IDataProviderHistoricalResponse, IDataProviderResponse } from './interfaces/interfaces'; @@ -121,69 +122,71 @@ export class DataProviderService implements DataProviderInterface { } public async getHistoricalRaw( - aSymbols: string[], + aDataGatheringItems: IDataGatheringItem[], from: Date, to: Date ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { - const filteredSymbols = aSymbols.filter((symbol) => { - return !isGhostfolioScraperApiSymbol(symbol); - }); - - const dataOfYahoo = await this.yahooFinanceService.getHistorical( - filteredSymbols, - undefined, - from, - to - ); - - if (aSymbols.length === 1) { - const symbol = aSymbols[0]; - - if ( - isCrypto(symbol) && - this.configurationService.get('ALPHA_VANTAGE_API_KEY') - ) { - // Merge data from Yahoo with data from Alpha Vantage - const dataOfAlphaVantage = await this.alphaVantageService.getHistorical( - [symbol], - undefined, - from, - to - ); + const result: { + [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; + } = {}; - return { - [symbol]: { - ...dataOfYahoo[symbol], - ...dataOfAlphaVantage[symbol] + for (const { dataSource, symbol } of aDataGatheringItems) { + switch (dataSource) { + case DataSource.ALPHA_VANTAGE: { + if (this.configurationService.get('ALPHA_VANTAGE_API_KEY')) { + const data = await this.alphaVantageService.getHistorical( + [symbol], + undefined, + from, + to + ); + result[symbol] = data?.[symbol]; } - }; - } else if (isGhostfolioScraperApiSymbol(symbol)) { - const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical( - [symbol], - undefined, - from, - to - ); - - return dataOfGhostfolioScraperApi; - } else if ( - isRakutenRapidApiSymbol(symbol) && - this.configurationService.get('RAKUTEN_RAPID_API_KEY') - ) { - const dataOfRakutenRapidApi = await this.rakutenRapidApiService.getHistorical( - [symbol], - undefined, - from, - to - ); - - return dataOfRakutenRapidApi; + break; + } + case DataSource.GHOSTFOLIO: { + if (isGhostfolioScraperApiSymbol(symbol)) { + const data = await this.ghostfolioScraperApiService.getHistorical( + [symbol], + undefined, + from, + to + ); + result[symbol] = data?.[symbol]; + } + break; + } + case DataSource.RAKUTEN: { + if ( + isRakutenRapidApiSymbol(symbol) && + this.configurationService.get('RAKUTEN_RAPID_API_KEY') + ) { + const data = await this.rakutenRapidApiService.getHistorical( + [symbol], + undefined, + from, + to + ); + result[symbol] = data?.[symbol]; + } + break; + } + case DataSource.YAHOO: { + const data = await this.yahooFinanceService.getHistorical( + [symbol], + undefined, + from, + to + ); + result[symbol] = data?.[symbol]; + break; + } } } - return dataOfYahoo; + return result; } public async search(aSymbol: string) { diff --git a/apps/api/src/services/interfaces/interfaces.ts b/apps/api/src/services/interfaces/interfaces.ts index 82f1381c3..bea217384 100644 --- a/apps/api/src/services/interfaces/interfaces.ts +++ b/apps/api/src/services/interfaces/interfaces.ts @@ -65,6 +65,12 @@ export interface IDataProviderResponse { url?: string; } +export interface IDataGatheringItem { + dataSource: DataSource; + date?: Date; + symbol: string; +} + export type Industry = typeof Industry[keyof typeof Industry]; export type MarketState = typeof MarketState[keyof typeof MarketState]; diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 3c724127c..c9f523b9f 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -1,13 +1,18 @@ +import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; +import { DataSource } from '@prisma/client'; + import { Currency } from '.prisma/client'; export const baseCurrency = Currency.CHF; -export const benchmarks = ['VOO']; +export const benchmarks: Partial[] = [ + { dataSource: DataSource.YAHOO, symbol: 'VOO' } +]; -export const currencyPairs = [ - `${Currency.USD}${Currency.EUR}`, - `${Currency.USD}${Currency.GBP}`, - `${Currency.USD}${Currency.CHF}` +export const currencyPairs: Partial[] = [ + { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.EUR}` }, + { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.GBP}` }, + { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.CHF}` } ]; export const ghostfolioScraperApiSymbolPrefix = '_GF_';