Respect data source in data gathering

apps/api/src/app/experimental/experimental.controller.ts

@@ -37,7 +37,9 @@ export class ExperimentalController {
       );
     }
 
-    return benchmarks;
+    return benchmarks.map(({ symbol }) => {
+      return symbol;
+    });
   }
 
   @Get('benchmarks/:symbol')

apps/api/src/app/order/order.service.ts

@@ -2,7 +2,7 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.se
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { OrderWithAccount } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
-import { Order, Prisma } from '@prisma/client';
+import { DataSource, Order, Prisma } from '@prisma/client';
 
 import { CacheService } from '../cache/cache.service';
 import { RedisCacheService } from '../redis-cache/redis-cache.service';
@@ -53,6 +53,7 @@ export class OrderService {
     // Gather symbol data of order in the background
     this.dataGatheringService.gatherSymbols([
       {
+        dataSource: data.dataSource,
         date: <Date>data.date,
         symbol: data.symbol
       }
@@ -90,6 +91,7 @@ export class OrderService {
     // Gather symbol data of order in the background
     this.dataGatheringService.gatherSymbols([
       {
+        dataSource: <DataSource>data.dataSource,
         date: <Date>data.date,
         symbol: <string>data.symbol
       }

apps/api/src/app/portfolio/portfolio.service.ts

@@ -11,6 +11,7 @@ import {
 import { DateRange, RequestWithUser } from '@ghostfolio/common/types';
 import { Inject, Injectable } from '@nestjs/common';
 import { REQUEST } from '@nestjs/core';
+import { DataSource } from '@prisma/client';
 import {
   add,
   format,
@@ -289,7 +290,7 @@ export class PortfolioService {
 
       if (isEmpty(historicalData)) {
         historicalData = await this.dataProviderService.getHistoricalRaw(
-          [aSymbol],
+          [{ dataSource: DataSource.YAHOO, symbol: aSymbol }],
           portfolio.getMinDate(),
           new Date()
         );

apps/api/src/services/data-gathering.service.ts

@@ -5,6 +5,7 @@ import {
   resetHours
 } from '@ghostfolio/common/helper';
 import { Injectable } from '@nestjs/common';
+import { DataSource } from '@prisma/client';
 import {
   differenceInHours,
   format,
@@ -18,6 +19,7 @@ import {
 import { ConfigurationService } from './configuration.service';
 import { DataProviderService } from './data-provider.service';
 import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
+import { IDataGatheringItem } from './interfaces/interfaces';
 import { PrismaService } from './prisma.service';
 
 @Injectable()
@@ -115,15 +117,13 @@ export class DataGatheringService {
     }
   }
 
-  public async gatherSymbols(
-    aSymbolsWithStartDate: { date: Date; symbol: string }[]
-  ) {
+  public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
     let hasError = false;
 
-    for (const { date, symbol } of aSymbolsWithStartDate) {
+    for (const { dataSource, date, symbol } of aSymbolsWithStartDate) {
       try {
         const historicalData = await this.dataProviderService.getHistoricalRaw(
-          [symbol],
+          [{ dataSource, symbol }],
           date,
           new Date()
         );
@@ -184,20 +184,24 @@ export class DataGatheringService {
     }
   }
 
-  public async getCustomSymbolsToGather(startDate?: Date) {
+  public async getCustomSymbolsToGather(
+    startDate?: Date
+  ): Promise<IDataGatheringItem[]> {
     const scraperConfigurations = await this.ghostfolioScraperApi.getScraperConfigurations();
 
     return scraperConfigurations.map((scraperConfiguration) => {
       return {
+        dataSource: DataSource.GHOSTFOLIO,
         date: startDate,
         symbol: scraperConfiguration.symbol
       };
     });
   }
 
-  private getBenchmarksToGather(startDate: Date) {
-    const benchmarksToGather = benchmarks.map((symbol) => {
+  private getBenchmarksToGather(startDate: Date): IDataGatheringItem[] {
+    const benchmarksToGather = benchmarks.map(({ dataSource, symbol }) => {
       return {
+        dataSource,
         symbol,
         date: startDate
       };
@@ -205,6 +209,7 @@ export class DataGatheringService {
 
     if (this.configurationService.get('ENABLE_FEATURE_FEAR_AND_GREED_INDEX')) {
       benchmarksToGather.push({
+        dataSource: DataSource.RAKUTEN,
         date: startDate,
         symbol: 'GF.FEAR_AND_GREED_INDEX'
       });
@@ -213,16 +218,16 @@ export class DataGatheringService {
     return benchmarksToGather;
   }
 
-  private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> {
+  private async getSymbols7D(): Promise<IDataGatheringItem[]> {
     const startDate = subDays(resetHours(new Date()), 7);
 
     const distinctOrders = await this.prisma.order.findMany({
       distinct: ['symbol'],
       orderBy: [{ symbol: 'asc' }],
-      select: { symbol: true }
+      select: { dataSource: true, symbol: true }
     });
 
-    const distinctOrdersWithDate = distinctOrders
+    const distinctOrdersWithDate: IDataGatheringItem[] = distinctOrders
       .filter((distinctOrder) => {
         return !isGhostfolioScraperApiSymbol(distinctOrder.symbol);
       })
@@ -233,12 +238,15 @@ export class DataGatheringService {
         };
       });
 
-    const currencyPairsToGather = currencyPairs.map((symbol) => {
-      return {
-        symbol,
-        date: startDate
-      };
-    });
+    const currencyPairsToGather = currencyPairs.map(
+      ({ dataSource, symbol }) => {
+        return {
+          dataSource,
+          symbol,
+          date: startDate
+        };
+      }
+    );
 
     const customSymbolsToGather = await this.getCustomSymbolsToGather(
       startDate
@@ -252,24 +260,27 @@ export class DataGatheringService {
     ];
   }
 
-  private async getSymbolsMax() {
+  private async getSymbolsMax(): Promise<IDataGatheringItem[]> {
     const startDate = new Date(getUtc('2015-01-01'));
 
     const customSymbolsToGather = await this.getCustomSymbolsToGather(
       startDate
     );
 
-    const currencyPairsToGather = currencyPairs.map((symbol) => {
-      return {
-        symbol,
-        date: startDate
-      };
-    });
+    const currencyPairsToGather = currencyPairs.map(
+      ({ dataSource, symbol }) => {
+        return {
+          dataSource,
+          symbol,
+          date: startDate
+        };
+      }
+    );
 
     const distinctOrders = await this.prisma.order.findMany({
       distinct: ['symbol'],
       orderBy: [{ date: 'asc' }],
-      select: { date: true, symbol: true }
+      select: { dataSource: true, date: true, symbol: true }
     });
 
     return [

apps/api/src/services/data-provider.service.ts

@@ -16,6 +16,7 @@ import { RakutenRapidApiService } from './data-provider/rakuten-rapid-api/rakute
 import { YahooFinanceService } from './data-provider/yahoo-finance/yahoo-finance.service';
 import { DataProviderInterface } from './interfaces/data-provider.interface';
 import {
+  IDataGatheringItem,
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from './interfaces/interfaces';
@@ -121,69 +122,71 @@ export class DataProviderService implements DataProviderInterface {
   }
 
   public async getHistoricalRaw(
-    aSymbols: string[],
+    aDataGatheringItems: IDataGatheringItem[],
     from: Date,
     to: Date
   ): Promise<{
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }> {
-    const filteredSymbols = aSymbols.filter((symbol) => {
-      return !isGhostfolioScraperApiSymbol(symbol);
-    });
-
-    const dataOfYahoo = await this.yahooFinanceService.getHistorical(
-      filteredSymbols,
-      undefined,
-      from,
-      to
-    );
-
-    if (aSymbols.length === 1) {
-      const symbol = aSymbols[0];
-
-      if (
-        isCrypto(symbol) &&
-        this.configurationService.get('ALPHA_VANTAGE_API_KEY')
-      ) {
-        // Merge data from Yahoo with data from Alpha Vantage
-        const dataOfAlphaVantage = await this.alphaVantageService.getHistorical(
-          [symbol],
-          undefined,
-          from,
-          to
-        );
+    const result: {
+      [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
+    } = {};
 
-        return {
-          [symbol]: {
-            ...dataOfYahoo[symbol],
-            ...dataOfAlphaVantage[symbol]
+    for (const { dataSource, symbol } of aDataGatheringItems) {
+      switch (dataSource) {
+        case DataSource.ALPHA_VANTAGE: {
+          if (this.configurationService.get('ALPHA_VANTAGE_API_KEY')) {
+            const data = await this.alphaVantageService.getHistorical(
+              [symbol],
+              undefined,
+              from,
+              to
+            );
+            result[symbol] = data?.[symbol];
           }
-        };
-      } else if (isGhostfolioScraperApiSymbol(symbol)) {
-        const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical(
-          [symbol],
-          undefined,
-          from,
-          to
-        );
-
-        return dataOfGhostfolioScraperApi;
-      } else if (
-        isRakutenRapidApiSymbol(symbol) &&
-        this.configurationService.get('RAKUTEN_RAPID_API_KEY')
-      ) {
-        const dataOfRakutenRapidApi = await this.rakutenRapidApiService.getHistorical(
-          [symbol],
-          undefined,
-          from,
-          to
-        );
-
-        return dataOfRakutenRapidApi;
+          break;
+        }
+        case DataSource.GHOSTFOLIO: {
+          if (isGhostfolioScraperApiSymbol(symbol)) {
+            const data = await this.ghostfolioScraperApiService.getHistorical(
+              [symbol],
+              undefined,
+              from,
+              to
+            );
+            result[symbol] = data?.[symbol];
+          }
+          break;
+        }
+        case DataSource.RAKUTEN: {
+          if (
+            isRakutenRapidApiSymbol(symbol) &&
+            this.configurationService.get('RAKUTEN_RAPID_API_KEY')
+          ) {
+            const data = await this.rakutenRapidApiService.getHistorical(
+              [symbol],
+              undefined,
+              from,
+              to
+            );
+            result[symbol] = data?.[symbol];
+          }
+          break;
+        }
+        case DataSource.YAHOO: {
+          const data = await this.yahooFinanceService.getHistorical(
+            [symbol],
+            undefined,
+            from,
+            to
+          );
+          result[symbol] = data?.[symbol];
+          break;
+        }
       }
     }
 
-    return dataOfYahoo;
+    return result;
   }
 
   public async search(aSymbol: string) {

apps/api/src/services/interfaces/interfaces.ts

@@ -65,6 +65,12 @@ export interface IDataProviderResponse {
   url?: string;
 }
 
+export interface IDataGatheringItem {
+  dataSource: DataSource;
+  date?: Date;
+  symbol: string;
+}
+
 export type Industry = typeof Industry[keyof typeof Industry];
 
 export type MarketState = typeof MarketState[keyof typeof MarketState];

libs/common/src/lib/config.ts

@@ -1,13 +1,18 @@
+import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
+import { DataSource } from '@prisma/client';
+
 import { Currency } from '.prisma/client';
 
 export const baseCurrency = Currency.CHF;
 
-export const benchmarks = ['VOO'];
+export const benchmarks: Partial<IDataGatheringItem>[] = [
+  { dataSource: DataSource.YAHOO, symbol: 'VOO' }
+];
 
-export const currencyPairs = [
-  `${Currency.USD}${Currency.EUR}`,
-  `${Currency.USD}${Currency.GBP}`,
-  `${Currency.USD}${Currency.CHF}`
+export const currencyPairs: Partial<IDataGatheringItem>[] = [
+  { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.EUR}` },
+  { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.GBP}` },
+  { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.CHF}` }
 ];
 
 export const ghostfolioScraperApiSymbolPrefix = '_GF_';