From 069006145a4e146430347426a6a79731db293e4f Mon Sep 17 00:00:00 2001 From: Thomas <4159106+dtslvr@users.noreply.github.com> Date: Tue, 20 Apr 2021 21:52:01 +0200 Subject: [PATCH] Improve scraper (#28) --- CHANGELOG.md | 10 ++ .../src/app/portfolio/portfolio.service.ts | 94 ++++++++-------- apps/api/src/app/user/user.service.ts | 28 ++--- apps/api/src/models/portfolio.ts | 104 +++++++++--------- .../src/services/data-gathering.service.ts | 17 ++- .../api/src/services/data-provider.service.ts | 39 +++++-- .../ghostfolio-scraper-api.service.ts | 49 ++++++--- apps/client/src/app/app-routing.module.ts | 3 +- .../position/position.component.html | 4 +- .../src/app/pages/home/home-page.component.ts | 23 ++-- libs/helper/src/lib/helper.ts | 4 +- 11 files changed, 210 insertions(+), 165 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c8254318..b0797d765 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Reverted the restoring of the scroll position when opening a new page + +### Fixed + +- Fixed some issues in the generic scraper + ## 0.87.0 - 19.04.2021 ### Added diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index 4f4e9894d..6c697e325 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -48,53 +48,6 @@ export class PortfolioService { private readonly userService: UserService ) {} - private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) { - let currentDate = new Date(); - - const normalizedMinDate = - getDate(aMinDate) === 1 - ? aMinDate - : add(setDate(aMinDate, 1), { months: 1 }); - - const year = getYear(currentDate); - const month = getMonth(currentDate); - const day = getDate(currentDate); - - currentDate = new Date(Date.UTC(year, month, day, 0)); - - switch (aDateRange) { - case '1d': - return sub(currentDate, { - days: 1 - }); - case 'ytd': - currentDate = setDate(currentDate, 1); - currentDate = setMonth(currentDate, 0); - return isAfter(currentDate, normalizedMinDate) - ? currentDate - : undefined; - case '1y': - currentDate = setDate(currentDate, 1); - currentDate = sub(currentDate, { - years: 1 - }); - return isAfter(currentDate, normalizedMinDate) - ? currentDate - : undefined; - case '5y': - currentDate = setDate(currentDate, 1); - currentDate = sub(currentDate, { - years: 5 - }); - return isAfter(currentDate, normalizedMinDate) - ? currentDate - : undefined; - default: - // Gets handled as all data - return undefined; - } - } - public async createPortfolio(aUserId: string): Promise { let portfolio: Portfolio; let stringifiedPortfolio = await this.redisCacheService.get( @@ -382,4 +335,51 @@ export class PortfolioService { symbol: aSymbol }; } + + private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) { + let currentDate = new Date(); + + const normalizedMinDate = + getDate(aMinDate) === 1 + ? aMinDate + : add(setDate(aMinDate, 1), { months: 1 }); + + const year = getYear(currentDate); + const month = getMonth(currentDate); + const day = getDate(currentDate); + + currentDate = new Date(Date.UTC(year, month, day, 0)); + + switch (aDateRange) { + case '1d': + return sub(currentDate, { + days: 1 + }); + case 'ytd': + currentDate = setDate(currentDate, 1); + currentDate = setMonth(currentDate, 0); + return isAfter(currentDate, normalizedMinDate) + ? currentDate + : undefined; + case '1y': + currentDate = setDate(currentDate, 1); + currentDate = sub(currentDate, { + years: 1 + }); + return isAfter(currentDate, normalizedMinDate) + ? currentDate + : undefined; + case '5y': + currentDate = setDate(currentDate, 1); + currentDate = sub(currentDate, { + years: 5 + }); + return isAfter(currentDate, normalizedMinDate) + ? currentDate + : undefined; + default: + // Gets handled as all data + return undefined; + } + } } diff --git a/apps/api/src/app/user/user.service.ts b/apps/api/src/app/user/user.service.ts index 82035da5b..a1eab374c 100644 --- a/apps/api/src/app/user/user.service.ts +++ b/apps/api/src/app/user/user.service.ts @@ -44,10 +44,6 @@ export class UserService { currentPermissions.push(permissions.accessFearAndGreedIndex); } - if (this.configurationService.get('ENABLE_FEATURE_SOCIAL_LOGIN')) { - currentPermissions.push(permissions.useSocialLogin); - } - return { alias, id, @@ -162,18 +158,6 @@ export class UserService { }); } - private getRandomString(length: number) { - const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; - const result = []; - - for (let i = 0; i < length; i++) { - result.push( - characters.charAt(Math.floor(Math.random() * characters.length)) - ); - } - return result.join(''); - } - public async updateUserSettings({ currency, userId @@ -200,4 +184,16 @@ export class UserService { return; } + + private getRandomString(length: number) { + const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; + const result = []; + + for (let i = 0; i < length; i++) { + result.push( + characters.charAt(Math.floor(Math.random() * characters.length)) + ); + } + return result.join(''); + } } diff --git a/apps/api/src/models/portfolio.ts b/apps/api/src/models/portfolio.ts index 4c73a45c4..e5b0efa64 100644 --- a/apps/api/src/models/portfolio.ts +++ b/apps/api/src/models/portfolio.ts @@ -79,7 +79,9 @@ export class Portfolio implements PortfolioInterface { investmentInOriginalCurrency: portfolioItemsYesterday?.positions[symbol] ?.investmentInOriginalCurrency, - marketPrice: currentData[symbol]?.marketPrice, + marketPrice: + currentData[symbol]?.marketPrice ?? + portfolioItemsYesterday.positions[symbol]?.marketPrice, quantity: portfolioItemsYesterday?.positions[symbol]?.quantity }; }); @@ -158,53 +160,6 @@ export class Portfolio implements PortfolioInterface { return this; } - private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) { - let currentDate = new Date(); - - const normalizedMinDate = - getDate(aMinDate) === 1 - ? aMinDate - : add(setDate(aMinDate, 1), { months: 1 }); - - const year = getYear(currentDate); - const month = getMonth(currentDate); - const day = getDate(currentDate); - - currentDate = new Date(Date.UTC(year, month, day, 0)); - - switch (aDateRange) { - case '1d': - return sub(currentDate, { - days: 1 - }); - case 'ytd': - currentDate = setDate(currentDate, 1); - currentDate = setMonth(currentDate, 0); - return isAfter(currentDate, normalizedMinDate) - ? currentDate - : undefined; - case '1y': - currentDate = setDate(currentDate, 1); - currentDate = sub(currentDate, { - years: 1 - }); - return isAfter(currentDate, normalizedMinDate) - ? currentDate - : undefined; - case '5y': - currentDate = setDate(currentDate, 1); - currentDate = sub(currentDate, { - years: 5 - }); - return isAfter(currentDate, normalizedMinDate) - ? currentDate - : undefined; - default: - // Gets handled as all data - return undefined; - } - } - public get(aDate?: Date): PortfolioItem[] { if (aDate) { const filteredPortfolio = this.portfolioItems.find((item) => { @@ -528,12 +483,6 @@ export class Portfolio implements PortfolioInterface { return this.orders; } - private getOrdersByType(aFilter: string[]) { - return this.orders.filter((order) => { - return aFilter.includes(order.getType()); - }); - } - public getValue(aDate = getToday()) { const positions = this.getPositions(aDate); let value = 0; @@ -692,6 +641,53 @@ export class Portfolio implements PortfolioInterface { this.updatePortfolioItems(); } + private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) { + let currentDate = new Date(); + + const normalizedMinDate = + getDate(aMinDate) === 1 + ? aMinDate + : add(setDate(aMinDate, 1), { months: 1 }); + + const year = getYear(currentDate); + const month = getMonth(currentDate); + const day = getDate(currentDate); + + currentDate = new Date(Date.UTC(year, month, day, 0)); + + switch (aDateRange) { + case '1d': + return sub(currentDate, { + days: 1 + }); + case 'ytd': + currentDate = setDate(currentDate, 1); + currentDate = setMonth(currentDate, 0); + return isAfter(currentDate, normalizedMinDate) + ? currentDate + : undefined; + case '1y': + currentDate = setDate(currentDate, 1); + currentDate = sub(currentDate, { + years: 1 + }); + return isAfter(currentDate, normalizedMinDate) + ? currentDate + : undefined; + case '5y': + currentDate = setDate(currentDate, 1); + currentDate = sub(currentDate, { + years: 5 + }); + return isAfter(currentDate, normalizedMinDate) + ? currentDate + : undefined; + default: + // Gets handled as all data + return undefined; + } + } + private updatePortfolioItems() { // console.time('update-portfolio-items'); diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 2b36494a8..6d96e0a55 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -2,6 +2,7 @@ import { benchmarks, currencyPairs, getUtc, + isGhostfolioScraperApiSymbol, resetHours } from '@ghostfolio/helper'; import { Injectable } from '@nestjs/common'; @@ -235,12 +236,16 @@ export class DataGatheringService { select: { symbol: true } }); - const distinctOrdersWithDate = distinctOrders.map((distinctOrder) => { - return { - ...distinctOrder, - date: startDate - }; - }); + const distinctOrdersWithDate = distinctOrders + .filter((distinctOrder) => { + return !isGhostfolioScraperApiSymbol(distinctOrder.symbol); + }) + .map((distinctOrder) => { + return { + ...distinctOrder, + date: startDate + }; + }); const currencyPairsToGather = currencyPairs.map((symbol) => { return { diff --git a/apps/api/src/services/data-provider.service.ts b/apps/api/src/services/data-provider.service.ts index c3fce9dca..827b22946 100644 --- a/apps/api/src/services/data-provider.service.ts +++ b/apps/api/src/services/data-provider.service.ts @@ -1,7 +1,7 @@ import { isCrypto, - isGhostfolioScraperApi, - isRakutenRapidApi + isGhostfolioScraperApiSymbol, + isRakutenRapidApiSymbol } from '@ghostfolio/helper'; import { Injectable } from '@nestjs/common'; import { MarketData } from '@prisma/client'; @@ -39,14 +39,33 @@ export class DataProviderService implements DataProviderInterface { if (aSymbols.length === 1) { const symbol = aSymbols[0]; - if (isGhostfolioScraperApi(symbol)) { + if (isGhostfolioScraperApiSymbol(symbol)) { return this.ghostfolioScraperApiService.get(aSymbols); - } else if (isRakutenRapidApi(symbol)) { + } else if (isRakutenRapidApiSymbol(symbol)) { return this.rakutenRapidApiService.get(aSymbols); } } - return this.yahooFinanceService.get(aSymbols); + const yahooFinanceSymbols = aSymbols.filter((symbol) => { + return !isGhostfolioScraperApiSymbol(symbol); + }); + + const response = await this.yahooFinanceService.get(yahooFinanceSymbols); + + const ghostfolioScraperApiSymbols = aSymbols.filter((symbol) => { + return isGhostfolioScraperApiSymbol(symbol); + }); + + for (const symbol of ghostfolioScraperApiSymbols) { + if (symbol) { + const ghostfolioScraperApiResult = await this.ghostfolioScraperApiService.get( + [symbol] + ); + response[symbol] = ghostfolioScraperApiResult[symbol]; + } + } + + return response; } public async getHistorical( @@ -107,8 +126,12 @@ export class DataProviderService implements DataProviderInterface { ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { + const filteredSymbols = aSymbols.filter((symbol) => { + return !isGhostfolioScraperApiSymbol(symbol); + }); + const dataOfYahoo = await this.yahooFinanceService.getHistorical( - aSymbols, + filteredSymbols, undefined, from, to @@ -135,7 +158,7 @@ export class DataProviderService implements DataProviderInterface { ...dataOfAlphaVantage[symbol] } }; - } else if (isGhostfolioScraperApi(symbol)) { + } else if (isGhostfolioScraperApiSymbol(symbol)) { const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical( [symbol], undefined, @@ -145,7 +168,7 @@ export class DataProviderService implements DataProviderInterface { return dataOfGhostfolioScraperApi; } else if ( - isRakutenRapidApi(symbol) && + isRakutenRapidApiSymbol(symbol) && this.configurationService.get('RAKUTEN_RAPID_API_KEY') ) { const dataOfRakutenRapidApi = await this.rakutenRapidApiService.getHistorical( diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts index 6b6947d94..55c3db0cb 100644 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts @@ -11,7 +11,6 @@ import { IDataProviderResponse } from '../../interfaces/interfaces'; import { PrismaService } from '../../prisma.service'; -import { Currency } from '.prisma/client'; @Injectable() export class GhostfolioScraperApiService implements DataProviderInterface { @@ -26,6 +25,9 @@ export class GhostfolioScraperApiService implements DataProviderInterface { try { const symbol = aSymbols[0]; + + const scraperConfig = await this.getScraperConfig(symbol); + const { marketPrice } = await this.prisma.marketData.findFirst({ orderBy: { date: 'desc' @@ -38,9 +40,9 @@ export class GhostfolioScraperApiService implements DataProviderInterface { return { [symbol]: { marketPrice, - currency: Currency.CHF, - isMarketOpen: true, - name: symbol + currency: scraperConfig?.currency, + isMarketOpen: false, + name: scraperConfig?.name } }; } catch (error) { @@ -65,25 +67,17 @@ export class GhostfolioScraperApiService implements DataProviderInterface { try { const symbol = aSymbols[0]; - const { - value: scraperConfigString - } = await this.prisma.property.findFirst({ - select: { - value: true - }, - where: { key: 'SCRAPER_CONFIG' } - }); - - const scraperConfig = JSON.parse(scraperConfigString).find((item) => { - return item.symbol === symbol; - }); + const scraperConfig = await this.getScraperConfig(symbol); - const get = bent(scraperConfig.url, 'GET', 'string', 200, {}); + const get = bent(scraperConfig?.url, 'GET', 'string', 200, {}); const html = await get(); const $ = cheerio.load(html); - const string = $(scraperConfig.selector).text().replace('CHF', '').trim(); + const string = $(scraperConfig?.selector) + .text() + .replace('CHF', '') + .trim(); const value = parseFloat(string); @@ -100,4 +94,23 @@ export class GhostfolioScraperApiService implements DataProviderInterface { return {}; } + + private async getScraperConfig(aSymbol: string) { + try { + const { + value: scraperConfigString + } = await this.prisma.property.findFirst({ + select: { + value: true + }, + where: { key: 'SCRAPER_CONFIG' } + }); + + return JSON.parse(scraperConfigString).find((item) => { + return item.symbol === aSymbol; + }); + } catch {} + + return {}; + } } diff --git a/apps/client/src/app/app-routing.module.ts b/apps/client/src/app/app-routing.module.ts index dfa2c3623..145744e91 100644 --- a/apps/client/src/app/app-routing.module.ts +++ b/apps/client/src/app/app-routing.module.ts @@ -81,8 +81,7 @@ const routes: Routes = [ { preloadingStrategy: ModulePreloadService, // enableTracing: true // <-- debugging purposes only - relativeLinkResolution: 'legacy', - scrollPositionRestoration: 'enabled' + relativeLinkResolution: 'legacy' } ) ], diff --git a/apps/client/src/app/components/position/position.component.html b/apps/client/src/app/components/position/position.component.html index 1c75f7267..1c3be1b31 100644 --- a/apps/client/src/app/components/position/position.component.html +++ b/apps/client/src/app/components/position/position.component.html @@ -40,7 +40,9 @@ class="ml-1" [url]="position?.url" > - ({{ position?.exchange }}) + ({{ position.exchange }})
{ + this.fearAndGreedIndex = marketPrice; + + this.cd.markForCheck(); + }); + } + this.hasPermissionToReadForeignPortfolio = hasPermission( user.permissions, permissions.readForeignPortfolio @@ -180,17 +192,6 @@ export class HomePageComponent implements OnDestroy, OnInit { this.cd.markForCheck(); }); - if (this.hasPermissionToAccessFearAndGreedIndex) { - this.dataService - .fetchSymbolItem('GF.FEAR_AND_GREED_INDEX') - .pipe(takeUntil(this.unsubscribeSubject)) - .subscribe(({ marketPrice }) => { - this.fearAndGreedIndex = marketPrice; - - this.cd.markForCheck(); - }); - } - this.cd.markForCheck(); } diff --git a/libs/helper/src/lib/helper.ts b/libs/helper/src/lib/helper.ts index d77780fcb..ba0966f09 100644 --- a/libs/helper/src/lib/helper.ts +++ b/libs/helper/src/lib/helper.ts @@ -66,11 +66,11 @@ export function isCurrency(aSymbol = '') { ); } -export function isGhostfolioScraperApi(aSymbol = '') { +export function isGhostfolioScraperApiSymbol(aSymbol = '') { return aSymbol.startsWith('[GF]'); } -export function isRakutenRapidApi(aSymbol = '') { +export function isRakutenRapidApiSymbol(aSymbol = '') { return aSymbol === 'GF.FEAR_AND_GREED_INDEX'; }