Add scraper symbols to symbol lookup results (#51)

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+- Added the generic scraper symbols to the symbol lookup results
+
 ## 0.93.0 - 26.04.2021
 
 ### Changed

apps/api/src/app/symbol/symbol.service.ts

@@ -1,4 +1,6 @@
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
+import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { convertFromYahooSymbol } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
 import { Injectable } from '@nestjs/common';
 import { Currency } from '@prisma/client';
@@ -10,7 +12,8 @@ import { SymbolItem } from './interfaces/symbol-item.interface';
 @Injectable()
 export class SymbolService {
   public constructor(
-    private readonly dataProviderService: DataProviderService
+    private readonly dataProviderService: DataProviderService,
+    private readonly ghostfolioScraperApiService: GhostfolioScraperApiService
   ) {}
 
   public async get(aSymbol: string): Promise<SymbolItem> {
@@ -23,18 +26,36 @@ export class SymbolService {
     };
   }
 
-  public async lookup(aQuery: string): Promise<LookupItem[]> {
+  public async lookup(aQuery = ''): Promise<LookupItem[]> {
+    const query = aQuery.toLowerCase();
+    const results: LookupItem[] = [];
+
+    if (!query) {
+      return results;
+    }
+
     const get = bent(
-      `https://query1.finance.yahoo.com/v1/finance/search?q=${aQuery}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
+      `https://query1.finance.yahoo.com/v1/finance/search?q=${query}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
       'GET',
       'json',
       200
     );
 
+    // Add custom symbols
+    const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations();
+    scraperConfigurations.forEach((scraperConfiguration) => {
+      if (scraperConfiguration.name.toLowerCase().startsWith(query)) {
+        results.push({
+          name: scraperConfiguration.name,
+          symbol: scraperConfiguration.symbol
+        });
+      }
+    });
+
     try {
       const { quotes } = await get();
 
-      return quotes
+      const searchResult = quotes
         .filter(({ isYahooFinance }) => {
           return isYahooFinance;
         })
@@ -59,6 +80,8 @@ export class SymbolService {
             symbol: convertFromYahooSymbol(symbol)
           };
         });
+
+      return results.concat(searchResult);
     } catch (error) {
       console.error(error);
 

apps/api/src/services/data-gathering.service.ts

@@ -18,6 +18,7 @@ import {
 
 import { ConfigurationService } from './configuration.service';
 import { DataProviderService } from './data-provider.service';
+import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { PrismaService } from './prisma.service';
 
 @Injectable()
@@ -25,6 +26,7 @@ export class DataGatheringService {
   public constructor(
     private readonly configurationService: ConfigurationService,
     private readonly dataProviderService: DataProviderService,
+    private readonly ghostfolioScraperApi: GhostfolioScraperApiService,
     private prisma: PrismaService
   ) {}
 
@@ -183,6 +185,17 @@ export class DataGatheringService {
     }
   }
 
+  public async getCustomSymbolsToGather(startDate?: Date) {
+    const scraperConfigurations = await this.ghostfolioScraperApi.getScraperConfigurations();
+
+    return scraperConfigurations.map((scraperConfiguration) => {
+      return {
+        date: startDate,
+        symbol: scraperConfiguration.symbol
+      };
+    });
+  }
+
   private getBenchmarksToGather(startDate: Date) {
     const benchmarksToGather = benchmarks.map((symbol) => {
       return {
@@ -201,32 +214,6 @@ export class DataGatheringService {
     return benchmarksToGather;
   }
 
-  private async getCustomSymbolsToGather(startDate: Date) {
-    const customSymbolsToGather = [];
-
-    if (this.configurationService.get('ENABLE_FEATURE_CUSTOM_SYMBOLS')) {
-      try {
-        const {
-          value: scraperConfigString
-        } = await this.prisma.property.findFirst({
-          select: {
-            value: true
-          },
-          where: { key: 'SCRAPER_CONFIG' }
-        });
-
-        JSON.parse(scraperConfigString).forEach((item) => {
-          customSymbolsToGather.push({
-            date: startDate,
-            symbol: item.symbol
-          });
-        });
-      } catch {}
-    }
-
-    return customSymbolsToGather;
-  }
-
   private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> {
     const startDate = subDays(resetHours(new Date()), 7);
 

apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@@ -12,6 +12,7 @@ import {
   MarketState
 } from '../../interfaces/interfaces';
 import { PrismaService } from '../../prisma.service';
+import { ScraperConfig } from './interfaces/scraper-config.interface';
 
 @Injectable()
 export class GhostfolioScraperApiService implements DataProviderInterface {
@@ -29,7 +30,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     try {
       const symbol = aSymbols[0];
 
-      const scraperConfig = await this.getScraperConfig(symbol);
+      const scraperConfig = await this.getScraperConfigurationBySymbol(symbol);
 
       const { marketPrice } = await this.prisma.marketData.findFirst({
         orderBy: {
@@ -70,15 +71,17 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     try {
       const symbol = aSymbols[0];
 
-      const scraperConfig = await this.getScraperConfig(symbol);
+      const scraperConfiguration = await this.getScraperConfigurationBySymbol(
+        symbol
+      );
 
-      const get = bent(scraperConfig?.url, 'GET', 'string', 200, {});
+      const get = bent(scraperConfiguration?.url, 'GET', 'string', 200, {});
 
       const html = await get();
       const $ = cheerio.load(html);
 
       const value = this.extractNumberFromString(
-        $(scraperConfig?.selector).text()
+        $(scraperConfiguration?.selector).text()
       );
 
       return {
@@ -95,18 +98,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     return {};
   }
 
-  private extractNumberFromString(aString: string): number {
+  public async getScraperConfigurations(): Promise<ScraperConfig[]> {
-    try {
-      const [numberString] = aString.match(
-        GhostfolioScraperApiService.NUMERIC_REGEXP
-      );
-      return parseFloat(numberString.trim());
-    } catch {
-      return undefined;
-    }
-  }
-
-  private async getScraperConfig(aSymbol: string) {
     try {
       const {
         value: scraperConfigString
@@ -117,11 +109,27 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
         where: { key: 'SCRAPER_CONFIG' }
       });
 
-      return JSON.parse(scraperConfigString).find((item) => {
+      return JSON.parse(scraperConfigString);
-        return item.symbol === aSymbol;
-      });
     } catch {}
 
-    return {};
+    return [];
+  }
+
+  private extractNumberFromString(aString: string): number {
+    try {
+      const [numberString] = aString.match(
+        GhostfolioScraperApiService.NUMERIC_REGEXP
+      );
+      return parseFloat(numberString.trim());
+    } catch {
+      return undefined;
+    }
+  }
+
+  private async getScraperConfigurationBySymbol(aSymbol: string) {
+    const scraperConfigurations = await this.getScraperConfigurations();
+    return scraperConfigurations.find((scraperConfiguration) => {
+      return scraperConfiguration.symbol === aSymbol;
+    });
   }
 }

apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-config.interface.ts

@@ -0,0 +1,9 @@
+import { Currency } from '.prisma/client';
+
+export interface ScraperConfig {
+  currency: Currency;
+  name: string;
+  selector: string;
+  symbol: string;
+  url: string;
+}

apps/client/src/app/pages/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.html

@@ -25,7 +25,7 @@
               class="autocomplete"
               [value]="lookupItem.symbol"
             >
-              <span class="mr-2 symbol">{{ lookupItem.symbol }}</span
+              <span class="mr-2 symbol">{{ lookupItem.symbol | gfSymbol }}</span
               ><span><b>{{ lookupItem.name }}</b></span>
             </mat-option>
           </ng-container>

apps/client/src/app/pages/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.module.ts

@@ -9,6 +9,7 @@ import { MatFormFieldModule } from '@angular/material/form-field';
 import { MatInputModule } from '@angular/material/input';
 import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
 import { MatSelectModule } from '@angular/material/select';
+import { GfSymbolModule } from '@ghostfolio/client/pipes/symbol/symbol.module';
 
 import { CreateOrUpdateTransactionDialog } from './create-or-update-transaction-dialog.component';
 
@@ -17,6 +18,7 @@ import { CreateOrUpdateTransactionDialog } from './create-or-update-transaction-
   exports: [],
   imports: [
     CommonModule,
+    GfSymbolModule,
     FormsModule,
     MatAutocompleteModule,
     MatButtonModule,

apps/client/src/app/pages/transactions/create-or-update-transaction-dialog/interfaces/interfaces.ts

@@ -1,6 +1,7 @@
-import { Order } from '../../interfaces/order.interface';
 import { Account } from '@prisma/client';
 
+import { Order } from '../../interfaces/order.interface';
+
 export interface CreateOrUpdateTransactionDialogParams {
   accountId: string;
   accounts: Account[];