Added support for testing scraping

.gitignore

@@ -40,3 +40,4 @@ yarn-error.log
 # System Files
 .DS_Store
 Thumbs.db
+.nx/cache

apps/api/src/app/admin/admin.controller.ts

@@ -45,6 +45,7 @@ import { AdminService } from './admin.service';
 import { UpdateAssetProfileDto } from './update-asset-profile.dto';
 import { UpdateBulkMarketDataDto } from './update-bulk-market-data.dto';
 import { UpdateMarketDataDto } from './update-market-data.dto';
+import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
 
 @Controller('admin')
 export class AdminController {
@@ -52,6 +53,7 @@ export class AdminController {
     private readonly adminService: AdminService,
     private readonly apiService: ApiService,
     private readonly dataGatheringService: DataGatheringService,
+    private readonly manualService: ManualService,
     private readonly marketDataService: MarketDataService,
     @Inject(REQUEST) private readonly request: RequestWithUser
   ) {}
@@ -246,8 +248,8 @@ export class AdminController {
     }
 
     if (dataSource === 'MANUAL' && isDryRun && isToday(date)) {
-      // TODO
+      //const marketData = await this.manualService.scrape()
-      Logger.log(`Check ${symbol} via scraperConfiguration`);
+      Logger.log(`Check ${symbol} via scraperConfiguration`, dataSource);
     }
 
     return this.dataGatheringService.gatherSymbolForDate({
@@ -258,6 +260,17 @@ export class AdminController {
     });
   }
 
+  @Post('test-scraper')
+  @UseGuards(AuthGuard('jwt'))
+  public async testScraper(
+    @Body() data: { config: string }
+  ): Promise<{ price: number }> {
+    const {url, selector} = JSON.parse(data.config);
+
+    let price = await this.manualService.scrape(url ,selector);
+    return {price: price};
+  }
+
   @Get('market-data')
   @UseGuards(AuthGuard('jwt'))
   public async getMarketData(

apps/api/src/app/admin/admin.module.ts

@@ -26,10 +26,11 @@ import { QueueModule } from './queue/queue.module';
     PropertyModule,
     QueueModule,
     SubscriptionModule,
-    SymbolProfileModule
+    SymbolProfileModule,
   ],
   controllers: [AdminController],
   providers: [AdminService],
-  exports: [AdminService]
+  exports: [AdminService],
+
 })
 export class AdminModule {}

apps/api/src/services/data-provider/data-provider.module.ts

@@ -29,6 +29,7 @@ import { DataProviderService } from './data-provider.service';
     PropertyModule,
     RedisCacheModule,
     SymbolProfileModule
+
   ],
   providers: [
     AlphaVantageService,
@@ -74,6 +75,6 @@ import { DataProviderService } from './data-provider.service';
     },
     YahooFinanceDataEnhancerService
   ],
-  exports: [DataProviderService, YahooFinanceService]
+  exports: [DataProviderService, YahooFinanceService, ManualService]
 })
 export class DataProviderModule {}

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -1,17 +1,10 @@
 import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
-import {
+import { IDataProviderHistoricalResponse, IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces';
-  IDataProviderHistoricalResponse,
-  IDataProviderResponse
-} from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
-import {
+import { DATE_FORMAT, extractNumberFromString, getYesterday } from '@ghostfolio/common/helper';
-  DATE_FORMAT,
-  extractNumberFromString,
-  getYesterday
-} from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable, Logger } from '@nestjs/common';
 import { DataSource, SymbolProfile } from '@prisma/client';
@@ -75,6 +68,7 @@ export class ManualService implements DataProviderInterface {
       url
       } = symbolProfile.scraperConfiguration ?? {};
 
+      Logger.log(symbolProfile);
       if (defaultMarketPrice) {
         const historical: {
           [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
@@ -95,23 +89,10 @@ export class ManualService implements DataProviderInterface {
       } else if (selector === undefined || url === undefined) {
         return {};
       }
+      Logger.log("Here");
 
-      const abortController = new AbortController();
-
-      setTimeout(() => {
-        abortController.abort();
-      }, DEFAULT_REQUEST_TIMEOUT);
-
-      const { body } = await got(url, {
-        headers,
-        // @ts-ignore
-        signal: abortController.signal
-      });
-
-      const $ = cheerio.load(body);
-
-      const value = extractNumberFromString($(selector).text());
 
+      const value = await this.scrape(url, selector, headers);
       return {
         [symbol]: {
           [format(getYesterday(), DATE_FORMAT)]: {
@@ -129,6 +110,26 @@ export class ManualService implements DataProviderInterface {
     }
   }
 
+  public async scrape(url: string, selector: string, headers = {}): Promise<number>{
+
+    const abortController = new AbortController();
+
+    const { body } = await got(url, {
+      headers,
+      // @ts-ignore
+      signal: abortController.signal
+    });
+
+    setTimeout(() => {
+      abortController.abort();
+    }, DEFAULT_REQUEST_TIMEOUT);
+    const $ = cheerio.load(body);
+
+    return extractNumberFromString($(selector).first().text());
+
+  }
+
+
   public getName(): DataSource {
     return DataSource.MANUAL;
   }

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts

@@ -240,21 +240,17 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
       });
   }
 
-  public onTestScraper(symbol: string) {
+  public onTestScraper() {
-    const today = new Date();
+
     this.adminService
-      .gatherSymbol({
+      .testScrapeConfig(this.assetProfileForm.controls['scraperConfiguration'].value)
-        dataSource: 'MANUAL',
-        date: today,
-        isDryRun: true,
-        symbol
-      })
       .pipe(takeUntil(this.unsubscribeSubject))
       .subscribe((response) => {
-        if (response !== null) {
+        if (response && response.price) {
-          alert($localize`Please try again.`);
+          console.log(response);
+          alert($localize`Current Market Price is:` + ' ' + response.price);
         } else {
-          alert($localize`Current Market Price is:` + ' ' + response);
+          alert($localize`Please try again.`);
         }
       });
   }

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html

@@ -241,6 +241,7 @@
             formControlName="scraperConfiguration"
             matInput
             type="text"
+            val
             (keyup.enter)="$event.stopPropagation()"
           ></textarea>
           <button
@@ -248,7 +249,7 @@
             mat-flat-button
             type="button"
             [disabled]="assetProfileForm.controls['scraperConfiguration'].value === '{}'"
-            (click)="onTestScraper(assetProfile?.symbol)"
+            (click)="onTestScraper()"
           >
             <ng-container i18n>Test</ng-container>
           </button>

apps/client/src/app/services/admin.service.ts

@@ -187,6 +187,10 @@ export class AdminService {
     return this.http.post<MarketData | void>(url, {});
   }
 
+  public testScrapeConfig(config: string){
+    return this.http.post<any>(`/api/v1/admin/test-scraper`, {config});
+  }
+
   public fetchSymbolForDate({
     dataSource,
     date,