Feature/move data enhancer from data provider to data gathering (#437)

* Move data enhancers from data provider to data enhancer module

* Update changelog

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Moved the data enhancer calls from the data provider (`get()`) to the data gathering service to reduce traffic to 3rd party data providers
+- Changed the profile data gathering from every 12 hours to once every weekend
+
 ## 1.64.0 - 21.10.2021
 
 ### Added

apps/api/src/app/cache/cache.module.ts

@@ -1,6 +1,7 @@
 import { CacheService } from '@ghostfolio/api/app/cache/cache.service';
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
 import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
@@ -10,7 +11,12 @@ import { Module } from '@nestjs/common';
 import { CacheController } from './cache.controller';
 
 @Module({
-  imports: [DataProviderModule, ExchangeRateDataModule, RedisCacheModule],
+  imports: [
+    DataGatheringModule,
+    DataProviderModule,
+    ExchangeRateDataModule,
+    RedisCacheModule
+  ],
   controllers: [CacheController],
   providers: [
     CacheService,

apps/api/src/app/info/info.module.ts

@@ -1,4 +1,5 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
 import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
@@ -11,6 +12,7 @@ import { InfoService } from './info.service';
 
 @Module({
   imports: [
+    DataGatheringModule,
     DataProviderModule,
     ExchangeRateDataModule,
     JwtModule.register({

apps/api/src/app/portfolio/current-rate.service.spec.ts

@@ -72,7 +72,7 @@ describe('CurrentRateService', () => {
   let marketDataService: MarketDataService;
 
   beforeAll(async () => {
-    dataProviderService = new DataProviderService(null, [], [], null);
+    dataProviderService = new DataProviderService(null, [], null);
     exchangeRateDataService = new ExchangeRateDataService(null, null);
     marketDataService = new MarketDataService(null);
 

apps/api/src/services/cron.service.ts

@@ -18,7 +18,11 @@ export class CronService {
 
   @Cron(CronExpression.EVERY_12_HOURS)
   public async runEveryTwelveHours() {
-    await this.dataGatheringService.gatherProfileData();
     await this.exchangeRateDataService.loadCurrencies();
   }
+
+  @Cron(CronExpression.EVERY_WEEKEND)
+  public async runEveryWeekend() {
+    await this.dataGatheringService.gatherProfileData();
+  }
 }

apps/api/src/services/data-gathering.module.ts

@@ -1,5 +1,6 @@
 import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
+import { DataEnhancerModule } from '@ghostfolio/api/services/data-provider/data-enhancer/data-enhancer.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
 import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
 import { Module } from '@nestjs/common';
@@ -9,11 +10,12 @@ import { ExchangeRateDataModule } from './exchange-rate-data.module';
 @Module({
   imports: [
     ConfigurationModule,
+    DataEnhancerModule,
     DataProviderModule,
     ExchangeRateDataModule,
     PrismaModule
   ],
   providers: [DataGatheringService],
-  exports: [DataGatheringService]
+  exports: [DataEnhancerModule, DataGatheringService]
 })
 export class DataGatheringModule {}

apps/api/src/services/data-gathering.service.ts

@@ -3,7 +3,7 @@ import {
   ghostfolioFearAndGreedIndexSymbol
 } from '@ghostfolio/common/config';
 import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
 import {
   differenceInHours,
@@ -18,6 +18,7 @@ import {
 import { ConfigurationService } from './configuration.service';
 import { DataProviderService } from './data-provider/data-provider.service';
 import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
+import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface';
 import { ExchangeRateDataService } from './exchange-rate-data.service';
 import { IDataGatheringItem } from './interfaces/interfaces';
 import { PrismaService } from './prisma.service';
@@ -26,6 +27,8 @@ import { PrismaService } from './prisma.service';
 export class DataGatheringService {
   public constructor(
     private readonly configurationService: ConfigurationService,
+    @Inject('DataEnhancers')
+    private readonly dataEnhancers: DataEnhancerInterface[],
     private readonly dataProviderService: DataProviderService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly ghostfolioScraperApi: GhostfolioScraperApiService,
@@ -130,9 +133,19 @@ export class DataGatheringService {
 
     const currentData = await this.dataProviderService.get(dataGatheringItems);
 
-    for (const [
-      symbol,
-      {
+    for (const [symbol, response] of Object.entries(currentData)) {
+      for (const dataEnhancer of this.dataEnhancers) {
+        try {
+          currentData[symbol] = await dataEnhancer.enhance({
+            response,
+            symbol
+          });
+        } catch (error) {
+          console.error(`Failed to enhance data for symbol ${symbol}`, error);
+        }
+      }
+
+      const {
         assetClass,
         assetSubClass,
         countries,
@@ -140,8 +153,8 @@ export class DataGatheringService {
         dataSource,
         name,
         sectors
-      }
-    ] of Object.entries(currentData)) {
+      } = currentData[symbol];
+
       try {
         await this.prismaService.symbolProfile.upsert({
           create: {

apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts

@@ -0,0 +1,15 @@
+import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service';
+import { Module } from '@nestjs/common';
+
+@Module({
+  exports: ['DataEnhancers', TrackinsightDataEnhancerService],
+  providers: [
+    {
+      inject: [TrackinsightDataEnhancerService],
+      provide: 'DataEnhancers',
+      useFactory: (trackinsight) => [trackinsight]
+    },
+    TrackinsightDataEnhancerService
+  ]
+})
+export class DataEnhancerModule {}

apps/api/src/services/data-provider/data-provider.module.ts

@@ -1,6 +1,5 @@
 import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
 import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
-import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service';
 import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
@@ -17,13 +16,7 @@ import { DataProviderService } from './data-provider.service';
     DataProviderService,
     GhostfolioScraperApiService,
     RakutenRapidApiService,
-    TrackinsightDataEnhancerService,
     YahooFinanceService,
-    {
-      inject: [TrackinsightDataEnhancerService],
-      provide: 'DataEnhancers',
-      useFactory: (trackinsight) => [trackinsight]
-    },
     {
       inject: [
         AlphaVantageService,

apps/api/src/services/data-provider/data-provider.service.ts

@@ -1,6 +1,5 @@
 import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
-import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
 import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
 import {
   IDataGatheringItem,
@@ -19,8 +18,6 @@ import { isEmpty } from 'lodash';
 export class DataProviderService {
   public constructor(
     private readonly configurationService: ConfigurationService,
-    @Inject('DataEnhancers')
-    private readonly dataEnhancers: DataEnhancerInterface[],
     @Inject('DataProviderInterfaces')
     private readonly dataProviderInterfaces: DataProviderInterface[],
     private readonly prismaService: PrismaService
@@ -42,20 +39,7 @@ export class DataProviderService {
 
     const promises = [];
     for (const symbol of Object.keys(response)) {
-      let promise = Promise.resolve(response[symbol]);
-      for (const dataEnhancer of this.dataEnhancers) {
-        promise = promise.then((currentResponse) =>
-          dataEnhancer
-            .enhance({ symbol, response: currentResponse })
-            .catch((error) => {
-              console.error(
-                `Failed to enhance data for symbol ${symbol}`,
-                error
-              );
-              return currentResponse;
-            })
-        );
-      }
+      const promise = Promise.resolve(response[symbol]);
       promises.push(
         promise.then((currentResponse) => (response[symbol] = currentResponse))
       );