From 9d303303d43433d217a8330b368c2210329dbc2c Mon Sep 17 00:00:00 2001 From: Thomas <4159106+dtslvr@users.noreply.github.com> Date: Mon, 17 May 2021 21:49:42 +0200 Subject: [PATCH] Refactor search functionality --- .../interfaces/lookup-item.interface.ts | 3 + apps/api/src/app/symbol/symbol.service.ts | 69 +++++-------------- .../api/src/services/configuration.service.ts | 4 +- .../api/src/services/data-provider.service.ts | 21 +++++- .../alpha-vantage/alpha-vantage.service.ts | 16 ++++- .../ghostfolio-scraper-api.service.ts | 4 ++ .../rakuten-rapid-api.service.ts | 12 ++-- .../yahoo-finance.service.spec.ts___ | 24 ------- .../yahoo-finance/yahoo-finance.service.ts | 45 ++++++++++++ .../interfaces/data-provider.interface.ts | 3 + .../interfaces/environment.interface.ts | 1 + apps/client/src/app/services/data.service.ts | 8 ++- prisma/schema.prisma | 1 + 13 files changed, 127 insertions(+), 84 deletions(-) delete mode 100644 apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.spec.ts___ diff --git a/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts b/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts index 689f69b91..e99cb893c 100644 --- a/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts +++ b/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts @@ -1,4 +1,7 @@ +import { DataSource } from '@prisma/client'; + export interface LookupItem { + dataSource: DataSource; name: string; symbol: string; } diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index 9da9488b9..3e487d60b 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -1,10 +1,8 @@ -import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; import { DataProviderService } from '@ghostfolio/api/services/data-provider.service'; import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; import { convertFromYahooSymbol } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; import { Injectable } from '@nestjs/common'; -import { Currency } from '@prisma/client'; -import * as bent from 'bent'; +import { Currency, DataSource } from '@prisma/client'; import { LookupItem } from './interfaces/lookup-item.interface'; import { SymbolItem } from './interfaces/symbol-item.interface'; @@ -27,62 +25,31 @@ export class SymbolService { }; } - public async lookup(aQuery = ''): Promise { + public async lookup(aQuery = ''): Promise<{ items: LookupItem[] }> { const query = aQuery.toLowerCase(); - const results: LookupItem[] = []; + const results: { items: LookupItem[] } = { items: [] }; if (!query) { return results; } - const get = bent( - `https://query1.finance.yahoo.com/v1/finance/search?q=${query}&lang=en-US®ion=US"esCount=8&newsCount=0&enableFuzzyQuery=false"esQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`, - 'GET', - 'json', - 200 - ); - - // Add custom symbols - const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations(); - scraperConfigurations.forEach((scraperConfiguration) => { - if (scraperConfiguration.name.toLowerCase().startsWith(query)) { - results.push({ - name: scraperConfiguration.name, - symbol: scraperConfiguration.symbol - }); - } - }); - try { - const { quotes } = await get(); - - const searchResult = quotes - .filter(({ isYahooFinance }) => { - return isYahooFinance; - }) - .filter(({ quoteType }) => { - return ( - quoteType === 'CRYPTOCURRENCY' || - quoteType === 'EQUITY' || - quoteType === 'ETF' - ); - }) - .filter(({ quoteType, symbol }) => { - if (quoteType === 'CRYPTOCURRENCY') { - // Only allow cryptocurrencies in USD - return symbol.includes('USD'); - } + const { items } = await this.dataProviderService.search(aQuery); + results.items = items; + + // Add custom symbols + const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations(); + scraperConfigurations.forEach((scraperConfiguration) => { + if (scraperConfiguration.name.toLowerCase().startsWith(query)) { + results.items.push({ + dataSource: DataSource.GHOSTFOLIO, + name: scraperConfiguration.name, + symbol: scraperConfiguration.symbol + }); + } + }); - return true; - }) - .map(({ longname, shortname, symbol }) => { - return { - name: longname || shortname, - symbol: convertFromYahooSymbol(symbol) - }; - }); - - return results.concat(searchResult); + return results; } catch (error) { console.error(error); diff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.ts index 6a5af44d5..a52bf85cd 100644 --- a/apps/api/src/services/configuration.service.ts +++ b/apps/api/src/services/configuration.service.ts @@ -1,7 +1,8 @@ import { Injectable } from '@nestjs/common'; -import { bool, cleanEnv, num, port, str } from 'envalid'; +import { bool, cleanEnv, json, num, port, str } from 'envalid'; import { Environment } from './interfaces/environment.interface'; +import { DataSource } from '.prisma/client'; @Injectable() export class ConfigurationService { @@ -12,6 +13,7 @@ export class ConfigurationService { ACCESS_TOKEN_SALT: str(), ALPHA_VANTAGE_API_KEY: str({ default: '' }), CACHE_TTL: num({ default: 1 }), + DATA_SOURCES: json({ default: JSON.stringify([DataSource.YAHOO]) }), ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }), ENABLE_FEATURE_FEAR_AND_GREED_INDEX: bool({ default: false }), ENABLE_FEATURE_SOCIAL_LOGIN: bool({ default: false }), diff --git a/apps/api/src/services/data-provider.service.ts b/apps/api/src/services/data-provider.service.ts index e5d82a7a1..e2d6af5f4 100644 --- a/apps/api/src/services/data-provider.service.ts +++ b/apps/api/src/services/data-provider.service.ts @@ -1,3 +1,4 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { isCrypto, isGhostfolioScraperApiSymbol, @@ -5,7 +6,7 @@ import { } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; -import { MarketData } from '@prisma/client'; +import { DataSource, MarketData } from '@prisma/client'; import { format } from 'date-fns'; import { ConfigurationService } from './configuration.service'; @@ -184,4 +185,22 @@ export class DataProviderService implements DataProviderInterface { return dataOfYahoo; } + + public async search(aSymbol: string) { + let results: { items: LookupItem[] } = { items: [] }; + + switch (this.configurationService.get('DATA_SOURCES')[0]) { + case DataSource.ALPHA_VANTAGE: + results = await this.alphaVantageService.search(aSymbol); + break; + case DataSource.YAHOO: + results = await this.yahooFinanceService.search(aSymbol); + break; + default: + console.error('No data provider has been found.'); + throw new Error('No data provider has been found.'); + } + + return results; + } } diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts index 696aee3c4..3046737e5 100644 --- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts +++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts @@ -1,5 +1,7 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; +import { DataSource } from '@prisma/client'; import { isAfter, isBefore, parse } from 'date-fns'; import { ConfigurationService } from '../../configuration.service'; @@ -77,7 +79,17 @@ export class AlphaVantageService implements DataProviderInterface { } } - public search(aSymbol: string) { - return this.alphaVantage.data.search(aSymbol); + public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { + const result = await this.alphaVantage.data.search(aSymbol); + + return { + items: result?.bestMatches?.map((bestMatch) => { + return { + dataSource: DataSource.ALPHA_VANTAGE, + name: bestMatch['2. name'], + symbol: bestMatch['1. symbol'] + }; + }) + }; } } diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts index 4568bf9a2..b6423f4a2 100644 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts @@ -117,6 +117,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface { return []; } + public async search(aSymbol: string) { + return { items: [] }; + } + private extractNumberFromString(aString: string): number { try { const [numberString] = aString.match( diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts index 1d31c0298..9f3943a8e 100644 --- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts @@ -117,6 +117,14 @@ export class RakutenRapidApiService implements DataProviderInterface { return {}; } + public async search(aSymbol: string) { + return { items: [] }; + } + + public setPrisma(aPrismaService: PrismaService) { + this.prisma = aPrismaService; + } + private async getFearAndGreedIndex(): Promise<{ now: { value: number; valueText: string }; previousClose: { value: number; valueText: string }; @@ -147,8 +155,4 @@ export class RakutenRapidApiService implements DataProviderInterface { return undefined; } } - - public setPrisma(aPrismaService: PrismaService) { - this.prisma = aPrismaService; - } } diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.spec.ts___ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.spec.ts___ deleted file mode 100644 index b18d9eaff..000000000 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.spec.ts___ +++ /dev/null @@ -1,24 +0,0 @@ -/* - import { Test } from '@nestjs/testing'; - - import { YahooFinanceService } from './yahoo-finance.service'; - - describe('AppService', () => { - let service: YahooFinanceService; - - beforeAll(async () => { - const app = await Test.createTestingModule({ - imports: [], - providers: [YahooFinanceService] - }).compile(); - - service = app.get(YahooFinanceService); - }); - - describe('get', () => { - it('should return data for USDCHF', () => { - expect(service.get(['USDCHF'])).toEqual('{}'); - }); - }); - }); -*/ diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index d20446f29..29134d904 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -1,8 +1,10 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { UNKNOWN_KEY } from '@ghostfolio/common/config'; import { isCrypto, isCurrency, parseCurrency } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; +import * as bent from 'bent'; import { format } from 'date-fns'; import * as yahooFinance from 'yahoo-finance'; @@ -136,6 +138,49 @@ export class YahooFinanceService implements DataProviderInterface { } } + public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { + let items = []; + + try { + const get = bent( + `https://query1.finance.yahoo.com/v1/finance/search?q=${aSymbol}&lang=en-US®ion=US"esCount=8&newsCount=0&enableFuzzyQuery=false"esQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`, + 'GET', + 'json', + 200 + ); + + const result = await get(); + items = result?.quotes + ?.filter((quote) => { + return quote.isYahooFinance; + }) + .filter(({ quoteType }) => { + return ( + quoteType === 'CRYPTOCURRENCY' || + quoteType === 'EQUITY' || + quoteType === 'ETF' + ); + }) + .filter(({ quoteType, symbol }) => { + if (quoteType === 'CRYPTOCURRENCY') { + // Only allow cryptocurrencies in USD + return symbol.includes('USD'); + } + + return true; + }) + .map(({ longname, shortname, symbol }) => { + return { + dataSource: DataSource.YAHOO, + name: longname || shortname, + symbol: convertFromYahooSymbol(symbol) + }; + }); + } catch {} + + return { items }; + } + /** * Converts a symbol to a Yahoo symbol * diff --git a/apps/api/src/services/interfaces/data-provider.interface.ts b/apps/api/src/services/interfaces/data-provider.interface.ts index c3a431dce..cf861ad7b 100644 --- a/apps/api/src/services/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/interfaces/data-provider.interface.ts @@ -1,3 +1,4 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { Granularity } from '@ghostfolio/common/types'; import { @@ -16,4 +17,6 @@ export interface DataProviderInterface { ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }>; + + search(aSymbol: string): Promise<{ items: LookupItem[] }>; } diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index ab73a0fe1..f5dae437e 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -4,6 +4,7 @@ export interface Environment extends CleanedEnvAccessors { ACCESS_TOKEN_SALT: string; ALPHA_VANTAGE_API_KEY: string; CACHE_TTL: number; + DATA_SOURCES: string[]; ENABLE_FEATURE_CUSTOM_SYMBOLS: boolean; ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean; ENABLE_FEATURE_SOCIAL_LOGIN: boolean; diff --git a/apps/client/src/app/services/data.service.ts b/apps/client/src/app/services/data.service.ts index 78c0f6cb0..d9d0fe112 100644 --- a/apps/client/src/app/services/data.service.ts +++ b/apps/client/src/app/services/data.service.ts @@ -83,7 +83,13 @@ export class DataService { } public fetchSymbols(aQuery: string) { - return this.http.get(`/api/symbol/lookup?query=${aQuery}`); + return this.http + .get<{ items: LookupItem[] }>(`/api/symbol/lookup?query=${aQuery}`) + .pipe( + map((respose) => { + return respose.items; + }) + ); } public fetchOrders(): Observable { diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4f173a39a..eb7a084c9 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -127,6 +127,7 @@ enum Currency { } enum DataSource { + ALPHA_VANTAGE GHOSTFOLIO RAKUTEN YAHOO