From faccc97fe9d8e25b561b0badba815a5702a190a0 Mon Sep 17 00:00:00 2001 From: Thomas <4159106+dtslvr@users.noreply.github.com> Date: Sat, 23 Oct 2021 11:50:17 +0200 Subject: [PATCH] Move data enhancer from data provider to data gathering --- apps/api/src/app/cache/cache.module.ts | 8 +++++- apps/api/src/app/info/info.module.ts | 2 ++ .../api/src/services/data-gathering.module.ts | 14 +++++++++-- .../src/services/data-gathering.service.ts | 25 ++++++++++++++----- .../data-enhancer/data-enhancer.module.ts | 8 ++++++ .../data-provider/data-provider.module.ts | 7 ------ .../data-provider/data-provider.service.ts | 18 +------------ 7 files changed, 49 insertions(+), 33 deletions(-) create mode 100644 apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts diff --git a/apps/api/src/app/cache/cache.module.ts b/apps/api/src/app/cache/cache.module.ts index 663d9926b..b346ee444 100644 --- a/apps/api/src/app/cache/cache.module.ts +++ b/apps/api/src/app/cache/cache.module.ts @@ -1,6 +1,7 @@ import { CacheService } from '@ghostfolio/api/app/cache/cache.service'; import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module'; import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; +import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module'; @@ -10,7 +11,12 @@ import { Module } from '@nestjs/common'; import { CacheController } from './cache.controller'; @Module({ - imports: [DataProviderModule, ExchangeRateDataModule, RedisCacheModule], + imports: [ + DataGatheringModule, + DataProviderModule, + ExchangeRateDataModule, + RedisCacheModule + ], controllers: [CacheController], providers: [ CacheService, diff --git a/apps/api/src/app/info/info.module.ts b/apps/api/src/app/info/info.module.ts index d2fb4ff38..0db322a85 100644 --- a/apps/api/src/app/info/info.module.ts +++ b/apps/api/src/app/info/info.module.ts @@ -1,4 +1,5 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; +import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module'; @@ -11,6 +12,7 @@ import { InfoService } from './info.service'; @Module({ imports: [ + DataGatheringModule, DataProviderModule, ExchangeRateDataModule, JwtModule.register({ diff --git a/apps/api/src/services/data-gathering.module.ts b/apps/api/src/services/data-gathering.module.ts index adf48a33b..8aafbb666 100644 --- a/apps/api/src/services/data-gathering.module.ts +++ b/apps/api/src/services/data-gathering.module.ts @@ -1,19 +1,29 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; +import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma.module'; import { Module } from '@nestjs/common'; +import { DataEnhancerModule } from './data-provider/data-enhancer/data-enhancer.module'; import { ExchangeRateDataModule } from './exchange-rate-data.module'; @Module({ imports: [ ConfigurationModule, + DataEnhancerModule, DataProviderModule, ExchangeRateDataModule, PrismaModule ], - providers: [DataGatheringService], - exports: [DataGatheringService] + providers: [ + { + inject: [TrackinsightDataEnhancerService], + provide: 'DataEnhancers', + useFactory: (trackinsight) => [trackinsight] + }, + DataGatheringService + ], + exports: ['DataEnhancers', DataGatheringService] }) export class DataGatheringModule {} diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 35a9a4e63..65e08fcf5 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -3,7 +3,7 @@ import { ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config'; import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper'; -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; import { differenceInHours, @@ -18,6 +18,7 @@ import { import { ConfigurationService } from './configuration.service'; import { DataProviderService } from './data-provider/data-provider.service'; import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; +import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface'; import { ExchangeRateDataService } from './exchange-rate-data.service'; import { IDataGatheringItem } from './interfaces/interfaces'; import { PrismaService } from './prisma.service'; @@ -26,6 +27,8 @@ import { PrismaService } from './prisma.service'; export class DataGatheringService { public constructor( private readonly configurationService: ConfigurationService, + @Inject('DataEnhancers') + private readonly dataEnhancers: DataEnhancerInterface[], private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, private readonly ghostfolioScraperApi: GhostfolioScraperApiService, @@ -130,9 +133,19 @@ export class DataGatheringService { const currentData = await this.dataProviderService.get(dataGatheringItems); - for (const [ - symbol, - { + for (const [symbol, response] of Object.entries(currentData)) { + for (const dataEnhancer of this.dataEnhancers) { + try { + currentData[symbol] = await dataEnhancer.enhance({ + response, + symbol + }); + } catch (error) { + console.error(`Failed to enhance data for symbol ${symbol}`, error); + } + } + + const { assetClass, assetSubClass, countries, @@ -140,8 +153,8 @@ export class DataGatheringService { dataSource, name, sectors - } - ] of Object.entries(currentData)) { + } = currentData[symbol]; + try { await this.prismaService.symbolProfile.upsert({ create: { diff --git a/apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts b/apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts new file mode 100644 index 000000000..fcdaad67c --- /dev/null +++ b/apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts @@ -0,0 +1,8 @@ +import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service'; +import { Module } from '@nestjs/common'; + +@Module({ + exports: [TrackinsightDataEnhancerService], + providers: [TrackinsightDataEnhancerService] +}) +export class DataEnhancerModule {} diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index 234d4d694..cacf2bf80 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -1,6 +1,5 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module'; import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module'; -import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service'; import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service'; import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; @@ -17,13 +16,7 @@ import { DataProviderService } from './data-provider.service'; DataProviderService, GhostfolioScraperApiService, RakutenRapidApiService, - TrackinsightDataEnhancerService, YahooFinanceService, - { - inject: [TrackinsightDataEnhancerService], - provide: 'DataEnhancers', - useFactory: (trackinsight) => [trackinsight] - }, { inject: [ AlphaVantageService, diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index ff975251a..b70b8dd59 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -1,6 +1,5 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; -import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface'; import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; import { IDataGatheringItem, @@ -19,8 +18,6 @@ import { isEmpty } from 'lodash'; export class DataProviderService { public constructor( private readonly configurationService: ConfigurationService, - @Inject('DataEnhancers') - private readonly dataEnhancers: DataEnhancerInterface[], @Inject('DataProviderInterfaces') private readonly dataProviderInterfaces: DataProviderInterface[], private readonly prismaService: PrismaService @@ -42,20 +39,7 @@ export class DataProviderService { const promises = []; for (const symbol of Object.keys(response)) { - let promise = Promise.resolve(response[symbol]); - for (const dataEnhancer of this.dataEnhancers) { - promise = promise.then((currentResponse) => - dataEnhancer - .enhance({ symbol, response: currentResponse }) - .catch((error) => { - console.error( - `Failed to enhance data for symbol ${symbol}`, - error - ); - return currentResponse; - }) - ); - } + const promise = Promise.resolve(response[symbol]); promises.push( promise.then((currentResponse) => (response[symbol] = currentResponse)) );