Feature/refactor search functionality (#105)

* Refactor search functionality

* Update changelog

* Improvements after code review

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Hid unknown exchange in the position overview
 - Disable the base currency selector for the demo user
 - Refactored the portfolio unit tests to work without database
+- Refactored the search functionality of the data management (aligned with data source)
 - Renamed shared helper to `@ghostfolio/common/helper`
 - Moved shared interfaces to `@ghostfolio/common/interfaces`
 - Moved shared types to `@ghostfolio/common/types`

apps/api/src/app/symbol/interfaces/lookup-item.interface.ts

@@ -1,4 +1,7 @@
+import { DataSource } from '@prisma/client';
+
 export interface LookupItem {
+  dataSource: DataSource;
   name: string;
   symbol: string;
 }

apps/api/src/app/symbol/symbol.controller.ts

@@ -28,9 +28,12 @@ export class SymbolController {
    */
   @Get('lookup')
   @UseGuards(AuthGuard('jwt'))
-  public async lookupSymbol(@Query() { query }): Promise<LookupItem[]> {
+  public async lookupSymbol(
+    @Query() { query = '' }
+  ): Promise<{ items: LookupItem[] }> {
     try {
-      return this.symbolService.lookup(query);
+      const encodedQuery = encodeURIComponent(query.toLowerCase());
+      return this.symbolService.lookup(encodedQuery);
     } catch {
       throw new HttpException(
         getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR),

apps/api/src/app/symbol/symbol.service.ts

@@ -1,10 +1,8 @@
-import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
 import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { convertFromYahooSymbol } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
 import { Injectable } from '@nestjs/common';
-import { Currency } from '@prisma/client';
+import { Currency, DataSource } from '@prisma/client';
-import * as bent from 'bent';
 
 import { LookupItem } from './interfaces/lookup-item.interface';
 import { SymbolItem } from './interfaces/symbol-item.interface';
@@ -27,62 +25,30 @@ export class SymbolService {
     };
   }
 
-  public async lookup(aQuery = ''): Promise<LookupItem[]> {
+  public async lookup(aQuery: string): Promise<{ items: LookupItem[] }> {
-    const query = aQuery.toLowerCase();
+    const results: { items: LookupItem[] } = { items: [] };
-    const results: LookupItem[] = [];
 
-    if (!query) {
+    if (!aQuery) {
       return results;
     }
 
-    const get = bent(
-      `https://query1.finance.yahoo.com/v1/finance/search?q=${query}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
-      'GET',
-      'json',
-      200
-    );
-
-    // Add custom symbols
-    const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations();
-    scraperConfigurations.forEach((scraperConfiguration) => {
-      if (scraperConfiguration.name.toLowerCase().startsWith(query)) {
-        results.push({
-          name: scraperConfiguration.name,
-          symbol: scraperConfiguration.symbol
-        });
-      }
-    });
-
     try {
-      const { quotes } = await get();
+      const { items } = await this.dataProviderService.search(aQuery);
-
+      results.items = items;
-      const searchResult = quotes
+
-        .filter(({ isYahooFinance }) => {
+      // Add custom symbols
-          return isYahooFinance;
+      const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations();
-        })
+      scraperConfigurations.forEach((scraperConfiguration) => {
-        .filter(({ quoteType }) => {
+        if (scraperConfiguration.name.toLowerCase().startsWith(aQuery)) {
-          return (
+          results.items.push({
-            quoteType === 'CRYPTOCURRENCY' ||
+            dataSource: DataSource.GHOSTFOLIO,
-            quoteType === 'EQUITY' ||
+            name: scraperConfiguration.name,
-            quoteType === 'ETF'
+            symbol: scraperConfiguration.symbol
-          );
+          });
-        })
+        }
-        .filter(({ quoteType, symbol }) => {
+      });
-          if (quoteType === 'CRYPTOCURRENCY') {
-            // Only allow cryptocurrencies in USD
-            return symbol.includes('USD');
-          }
 
-          return true;
+      return results;
-        })
-        .map(({ longname, shortname, symbol }) => {
-          return {
-            name: longname || shortname,
-            symbol: convertFromYahooSymbol(symbol)
-          };
-        });
-
-      return results.concat(searchResult);
     } catch (error) {
       console.error(error);
 

apps/api/src/services/configuration.service.ts

@@ -1,7 +1,8 @@
 import { Injectable } from '@nestjs/common';
-import { bool, cleanEnv, num, port, str } from 'envalid';
+import { bool, cleanEnv, json, num, port, str } from 'envalid';
 
 import { Environment } from './interfaces/environment.interface';
+import { DataSource } from '.prisma/client';
 
 @Injectable()
 export class ConfigurationService {
@@ -12,6 +13,7 @@ export class ConfigurationService {
       ACCESS_TOKEN_SALT: str(),
       ALPHA_VANTAGE_API_KEY: str({ default: '' }),
       CACHE_TTL: num({ default: 1 }),
+      DATA_SOURCES: json({ default: JSON.stringify([DataSource.YAHOO]) }),
       ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }),
       ENABLE_FEATURE_FEAR_AND_GREED_INDEX: bool({ default: false }),
       ENABLE_FEATURE_SOCIAL_LOGIN: bool({ default: false }),

apps/api/src/services/data-provider.service.ts

@@ -1,3 +1,4 @@
+import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import {
   isCrypto,
   isGhostfolioScraperApiSymbol,
@@ -5,7 +6,7 @@ import {
 } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
-import { MarketData } from '@prisma/client';
+import { DataSource, MarketData } from '@prisma/client';
 import { format } from 'date-fns';
 
 import { ConfigurationService } from './configuration.service';
@@ -184,4 +185,19 @@ export class DataProviderService implements DataProviderInterface {
 
     return dataOfYahoo;
   }
+
+  public async search(aSymbol: string) {
+    return this.getDataProvider().search(aSymbol);
+  }
+
+  private getDataProvider() {
+    switch (this.configurationService.get('DATA_SOURCES')[0]) {
+      case DataSource.ALPHA_VANTAGE:
+        return this.alphaVantageService;
+      case DataSource.YAHOO:
+        return this.yahooFinanceService;
+      default:
+        throw new Error('No data provider has been found.');
+    }
+  }
 }

apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts

@@ -1,5 +1,7 @@
+import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
+import { DataSource } from '@prisma/client';
 import { isAfter, isBefore, parse } from 'date-fns';
 
 import { ConfigurationService } from '../../configuration.service';
@@ -77,7 +79,17 @@ export class AlphaVantageService implements DataProviderInterface {
     }
   }
 
-  public search(aSymbol: string) {
+  public async search(aSymbol: string): Promise<{ items: LookupItem[] }> {
-    return this.alphaVantage.data.search(aSymbol);
+    const result = await this.alphaVantage.data.search(aSymbol);
+
+    return {
+      items: result?.bestMatches?.map((bestMatch) => {
+        return {
+          dataSource: DataSource.ALPHA_VANTAGE,
+          name: bestMatch['2. name'],
+          symbol: bestMatch['1. symbol']
+        };
+      })
+    };
   }
 }

apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@@ -117,6 +117,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     return [];
   }
 
+  public async search(aSymbol: string) {
+    return { items: [] };
+  }
+
   private extractNumberFromString(aString: string): number {
     try {
       const [numberString] = aString.match(

apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts

@@ -117,6 +117,14 @@ export class RakutenRapidApiService implements DataProviderInterface {
     return {};
   }
 
+  public async search(aSymbol: string) {
+    return { items: [] };
+  }
+
+  public setPrisma(aPrismaService: PrismaService) {
+    this.prisma = aPrismaService;
+  }
+
   private async getFearAndGreedIndex(): Promise<{
     now: { value: number; valueText: string };
     previousClose: { value: number; valueText: string };
@@ -147,8 +155,4 @@ export class RakutenRapidApiService implements DataProviderInterface {
       return undefined;
     }
   }
-
-  public setPrisma(aPrismaService: PrismaService) {
-    this.prisma = aPrismaService;
-  }
 }

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.spec.ts___

@@ -1,24 +0,0 @@
-/*
-  import { Test } from '@nestjs/testing';
-
-  import { YahooFinanceService } from './yahoo-finance.service';
-
-  describe('AppService', () => {
-    let service: YahooFinanceService;
-
-    beforeAll(async () => {
-      const app = await Test.createTestingModule({
-        imports: [],
-        providers: [YahooFinanceService]
-      }).compile();
-
-      service = app.get<YahooFinanceService>(YahooFinanceService);
-    });
-
-    describe('get', () => {
-      it('should return data for USDCHF', () => {
-        expect(service.get(['USDCHF'])).toEqual('{}');
-      });
-    });
-  });
-*/

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@@ -1,8 +1,10 @@
+import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import { UNKNOWN_KEY } from '@ghostfolio/common/config';
 import { isCrypto, isCurrency, parseCurrency } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
+import * as bent from 'bent';
 import { format } from 'date-fns';
 import * as yahooFinance from 'yahoo-finance';
 
@@ -22,6 +24,8 @@ import {
 
 @Injectable()
 export class YahooFinanceService implements DataProviderInterface {
+  private yahooFinanceHostname = 'https://query1.finance.yahoo.com';
+
   public constructor() {}
 
   public async get(
@@ -136,6 +140,49 @@ export class YahooFinanceService implements DataProviderInterface {
     }
   }
 
+  public async search(aSymbol: string): Promise<{ items: LookupItem[] }> {
+    let items = [];
+
+    try {
+      const get = bent(
+        `${this.yahooFinanceHostname}/v1/finance/search?q=${aSymbol}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
+        'GET',
+        'json',
+        200
+      );
+
+      const result = await get();
+      items = result.quotes
+        .filter((quote) => {
+          return quote.isYahooFinance;
+        })
+        .filter(({ quoteType }) => {
+          return (
+            quoteType === 'CRYPTOCURRENCY' ||
+            quoteType === 'EQUITY' ||
+            quoteType === 'ETF'
+          );
+        })
+        .filter(({ quoteType, symbol }) => {
+          if (quoteType === 'CRYPTOCURRENCY') {
+            // Only allow cryptocurrencies in USD
+            return symbol.includes('USD');
+          }
+
+          return true;
+        })
+        .map(({ longname, shortname, symbol }) => {
+          return {
+            dataSource: DataSource.YAHOO,
+            name: longname || shortname,
+            symbol: convertFromYahooSymbol(symbol)
+          };
+        });
+    } catch {}
+
+    return { items };
+  }
+
   /**
    * Converts a symbol to a Yahoo symbol
    *

apps/api/src/services/interfaces/data-provider.interface.ts

@@ -1,3 +1,4 @@
+import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import { Granularity } from '@ghostfolio/common/types';
 
 import {
@@ -16,4 +17,6 @@ export interface DataProviderInterface {
   ): Promise<{
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }>;
+
+  search(aSymbol: string): Promise<{ items: LookupItem[] }>;
 }

apps/api/src/services/interfaces/environment.interface.ts

@@ -4,6 +4,7 @@ export interface Environment extends CleanedEnvAccessors {
   ACCESS_TOKEN_SALT: string;
   ALPHA_VANTAGE_API_KEY: string;
   CACHE_TTL: number;
+  DATA_SOURCES: string | string[]; // string is not correct, error in envalid?
   ENABLE_FEATURE_CUSTOM_SYMBOLS: boolean;
   ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;
   ENABLE_FEATURE_SOCIAL_LOGIN: boolean;

apps/client/src/app/services/data.service.ts

@@ -102,7 +102,13 @@ export class DataService {
   }
 
   public fetchSymbols(aQuery: string) {
-    return this.http.get<LookupItem[]>(`/api/symbol/lookup?query=${aQuery}`);
+    return this.http
+      .get<{ items: LookupItem[] }>(`/api/symbol/lookup?query=${aQuery}`)
+      .pipe(
+        map((respose) => {
+          return respose.items;
+        })
+      );
   }
 
   public fetchOrders(): Observable<OrderModel[]> {

prisma/schema.prisma

@@ -127,6 +127,7 @@ enum Currency {
 }
 
 enum DataSource {
+  ALPHA_VANTAGE
   GHOSTFOLIO
   RAKUTEN
   YAHOO