From 9156a3eae3a95b71c7ac6ad394b55b837c3e3a57 Mon Sep 17 00:00:00 2001 From: Hugo Persson Date: Sat, 23 Dec 2023 12:32:33 +0100 Subject: [PATCH] Added support for testing scraping --- .gitignore | 1 + apps/api/src/app/admin/admin.controller.ts | 17 ++++++- apps/api/src/app/admin/admin.module.ts | 5 +- .../data-provider/data-provider.module.ts | 3 +- .../data-provider/manual/manual.service.ts | 51 ++++++++++--------- .../asset-profile-dialog.component.ts | 18 +++---- .../asset-profile-dialog.html | 3 +- apps/client/src/app/services/admin.service.ts | 4 ++ 8 files changed, 60 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index 307d7e9e2..e4e767dde 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ yarn-error.log # System Files .DS_Store Thumbs.db +.nx/cache diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 0ee935b62..43c3af90f 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -45,6 +45,7 @@ import { AdminService } from './admin.service'; import { UpdateAssetProfileDto } from './update-asset-profile.dto'; import { UpdateBulkMarketDataDto } from './update-bulk-market-data.dto'; import { UpdateMarketDataDto } from './update-market-data.dto'; +import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service'; @Controller('admin') export class AdminController { @@ -52,6 +53,7 @@ export class AdminController { private readonly adminService: AdminService, private readonly apiService: ApiService, private readonly dataGatheringService: DataGatheringService, + private readonly manualService: ManualService, private readonly marketDataService: MarketDataService, @Inject(REQUEST) private readonly request: RequestWithUser ) {} @@ -246,8 +248,8 @@ export class AdminController { } if (dataSource === 'MANUAL' && isDryRun && isToday(date)) { - // TODO - Logger.log(`Check ${symbol} via scraperConfiguration`); + //const marketData = await this.manualService.scrape() + Logger.log(`Check ${symbol} via scraperConfiguration`, dataSource); } return this.dataGatheringService.gatherSymbolForDate({ @@ -258,6 +260,17 @@ export class AdminController { }); } + @Post('test-scraper') + @UseGuards(AuthGuard('jwt')) + public async testScraper( + @Body() data: { config: string } + ): Promise<{ price: number }> { + const {url, selector} = JSON.parse(data.config); + + let price = await this.manualService.scrape(url ,selector); + return {price: price}; + } + @Get('market-data') @UseGuards(AuthGuard('jwt')) public async getMarketData( diff --git a/apps/api/src/app/admin/admin.module.ts b/apps/api/src/app/admin/admin.module.ts index 079af87fa..e0d01d363 100644 --- a/apps/api/src/app/admin/admin.module.ts +++ b/apps/api/src/app/admin/admin.module.ts @@ -26,10 +26,11 @@ import { QueueModule } from './queue/queue.module'; PropertyModule, QueueModule, SubscriptionModule, - SymbolProfileModule + SymbolProfileModule, ], controllers: [AdminController], providers: [AdminService], - exports: [AdminService] + exports: [AdminService], + }) export class AdminModule {} diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index b3a219a50..8b00dc713 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -29,6 +29,7 @@ import { DataProviderService } from './data-provider.service'; PropertyModule, RedisCacheModule, SymbolProfileModule + ], providers: [ AlphaVantageService, @@ -74,6 +75,6 @@ import { DataProviderService } from './data-provider.service'; }, YahooFinanceDataEnhancerService ], - exports: [DataProviderService, YahooFinanceService] + exports: [DataProviderService, YahooFinanceService, ManualService] }) export class DataProviderModule {} diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 5c84a9c92..9625aff2f 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -1,17 +1,10 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; -import { - IDataProviderHistoricalResponse, - IDataProviderResponse -} from '@ghostfolio/api/services/interfaces/interfaces'; +import { IDataProviderHistoricalResponse, IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config'; -import { - DATE_FORMAT, - extractNumberFromString, - getYesterday -} from '@ghostfolio/common/helper'; +import { DATE_FORMAT, extractNumberFromString, getYesterday } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; @@ -72,9 +65,10 @@ export class ManualService implements DataProviderInterface { defaultMarketPrice, headers = {}, selector, - url + url } = symbolProfile.scraperConfiguration ?? {}; + Logger.log(symbolProfile); if (defaultMarketPrice) { const historical: { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; @@ -95,23 +89,10 @@ export class ManualService implements DataProviderInterface { } else if (selector === undefined || url === undefined) { return {}; } + Logger.log("Here"); - const abortController = new AbortController(); - - setTimeout(() => { - abortController.abort(); - }, DEFAULT_REQUEST_TIMEOUT); - - const { body } = await got(url, { - headers, - // @ts-ignore - signal: abortController.signal - }); - - const $ = cheerio.load(body); - - const value = extractNumberFromString($(selector).text()); + const value = await this.scrape(url, selector, headers); return { [symbol]: { [format(getYesterday(), DATE_FORMAT)]: { @@ -129,6 +110,26 @@ export class ManualService implements DataProviderInterface { } } + public async scrape(url: string, selector: string, headers = {}): Promise{ + + const abortController = new AbortController(); + + const { body } = await got(url, { + headers, + // @ts-ignore + signal: abortController.signal + }); + + setTimeout(() => { + abortController.abort(); + }, DEFAULT_REQUEST_TIMEOUT); + const $ = cheerio.load(body); + + return extractNumberFromString($(selector).first().text()); + + } + + public getName(): DataSource { return DataSource.MANUAL; } diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts index f7c4fc57e..b0f5f3ac4 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts @@ -240,21 +240,17 @@ export class AssetProfileDialog implements OnDestroy, OnInit { }); } - public onTestScraper(symbol: string) { - const today = new Date(); + public onTestScraper() { + this.adminService - .gatherSymbol({ - dataSource: 'MANUAL', - date: today, - isDryRun: true, - symbol - }) + .testScrapeConfig(this.assetProfileForm.controls['scraperConfiguration'].value) .pipe(takeUntil(this.unsubscribeSubject)) .subscribe((response) => { - if (response !== null) { - alert($localize`Please try again.`); + if (response && response.price) { + console.log(response); + alert($localize`Current Market Price is:` + ' ' + response.price); } else { - alert($localize`Current Market Price is:` + ' ' + response); + alert($localize`Please try again.`); } }); } diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html index 99fe741a4..a832f32d0 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html @@ -241,6 +241,7 @@ formControlName="scraperConfiguration" matInput type="text" + val (keyup.enter)="$event.stopPropagation()" > diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.ts index 49e990b8c..ed88fe375 100644 --- a/apps/client/src/app/services/admin.service.ts +++ b/apps/client/src/app/services/admin.service.ts @@ -187,6 +187,10 @@ export class AdminService { return this.http.post(url, {}); } + public testScrapeConfig(config: string){ + return this.http.post(`/api/v1/admin/test-scraper`, {config}); + } + public fetchSymbolForDate({ dataSource, date,