From 0cbf275a2e2cd3508148f069b663b9aec7077235 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Thu, 16 Feb 2023 16:25:23 +0100 Subject: [PATCH] Feature/eliminate ghostfolio scraper api service (#1706) * Eliminate GhostfolioScraperApiService * Update changelog --- CHANGELOG.md | 6 + .../api/src/services/configuration.service.ts | 2 +- .../src/services/data-gathering.service.ts | 1 - .../data-provider/data-provider.module.ts | 11 +- .../ghostfolio-scraper-api.service.ts | 194 ------------------ .../migration.sql | 2 + 6 files changed, 10 insertions(+), 206 deletions(-) delete mode 100644 apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts create mode 100644 prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fd7d738b..d80726239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Eliminated the `GhostfolioScraperApiService` + ## 1.234.0 - 2023-02-15 ### Added diff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.ts index 7ea284a1e..60e36623e 100644 --- a/apps/api/src/services/configuration.service.ts +++ b/apps/api/src/services/configuration.service.ts @@ -19,7 +19,7 @@ export class ConfigurationService { CACHE_TTL: num({ default: 1 }), DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }), DATA_SOURCES: json({ - default: [DataSource.GHOSTFOLIO, DataSource.MANUAL, DataSource.YAHOO] + default: [DataSource.MANUAL, DataSource.YAHOO] }), ENABLE_FEATURE_BLOG: bool({ default: false }), ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }), diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index e977c9c07..6e8ceb808 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -278,7 +278,6 @@ export class DataGatheringService { return symbolProfiles .filter(({ dataSource }) => { return ( - dataSource !== DataSource.GHOSTFOLIO && dataSource !== DataSource.MANUAL && dataSource !== DataSource.RAPID_API ); diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index 07982e698..abafe6189 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -2,7 +2,6 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.modu import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module'; import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service'; import { EodHistoricalDataService } from '@ghostfolio/api/services/data-provider/eod-historical-data/eod-historical-data.service'; -import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service'; import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service'; import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service'; @@ -24,7 +23,6 @@ import { DataProviderService } from './data-provider.service'; AlphaVantageService, DataProviderService, EodHistoricalDataService, - GhostfolioScraperApiService, GoogleSheetsService, ManualService, RapidApiService, @@ -33,7 +31,6 @@ import { DataProviderService } from './data-provider.service'; inject: [ AlphaVantageService, EodHistoricalDataService, - GhostfolioScraperApiService, GoogleSheetsService, ManualService, RapidApiService, @@ -43,7 +40,6 @@ import { DataProviderService } from './data-provider.service'; useFactory: ( alphaVantageService, eodHistoricalDataService, - ghostfolioScraperApiService, googleSheetsService, manualService, rapidApiService, @@ -51,7 +47,6 @@ import { DataProviderService } from './data-provider.service'; ) => [ alphaVantageService, eodHistoricalDataService, - ghostfolioScraperApiService, googleSheetsService, manualService, rapidApiService, @@ -59,10 +54,6 @@ import { DataProviderService } from './data-provider.service'; ] } ], - exports: [ - DataProviderService, - GhostfolioScraperApiService, - YahooFinanceService - ] + exports: [DataProviderService, YahooFinanceService] }) export class DataProviderModule {} diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts deleted file mode 100644 index d1049ffb1..000000000 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; -import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; -import { - IDataProviderHistoricalResponse, - IDataProviderResponse -} from '@ghostfolio/api/services/interfaces/interfaces'; -import { PrismaService } from '@ghostfolio/api/services/prisma.service'; -import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service'; -import { - DATE_FORMAT, - extractNumberFromString, - getYesterday -} from '@ghostfolio/common/helper'; -import { Granularity } from '@ghostfolio/common/types'; -import { Injectable, Logger } from '@nestjs/common'; -import { DataSource, SymbolProfile } from '@prisma/client'; -import bent from 'bent'; -import * as cheerio from 'cheerio'; -import { addDays, format, isBefore } from 'date-fns'; - -@Injectable() -export class GhostfolioScraperApiService implements DataProviderInterface { - public constructor( - private readonly prismaService: PrismaService, - private readonly symbolProfileService: SymbolProfileService - ) {} - - public canHandle(symbol: string) { - return true; - } - - public async getAssetProfile( - aSymbol: string - ): Promise> { - return { - dataSource: this.getName() - }; - } - - public async getDividends({ - from, - granularity = 'day', - symbol, - to - }: { - from: Date; - granularity: Granularity; - symbol: string; - to: Date; - }) { - return {}; - } - - public async getHistorical( - aSymbol: string, - aGranularity: Granularity = 'day', - from: Date, - to: Date - ): Promise<{ - [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; - }> { - try { - const symbol = aSymbol; - - const [symbolProfile] = - await this.symbolProfileService.getSymbolProfilesBySymbols([symbol]); - const { defaultMarketPrice, selector, url } = - symbolProfile.scraperConfiguration ?? {}; - - if (defaultMarketPrice) { - const historical: { - [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; - } = { - [symbol]: {} - }; - let date = from; - - while (isBefore(date, to)) { - historical[symbol][format(date, DATE_FORMAT)] = { - marketPrice: defaultMarketPrice - }; - - date = addDays(date, 1); - } - - return historical; - } else if (selector === undefined || url === undefined) { - return {}; - } - - const get = bent(url, 'GET', 'string', 200, {}); - - const html = await get(); - const $ = cheerio.load(html); - - const value = extractNumberFromString($(selector).text()); - - return { - [symbol]: { - [format(getYesterday(), DATE_FORMAT)]: { - marketPrice: value - } - } - }; - } catch (error) { - throw new Error( - `Could not get historical market data for ${aSymbol} (${this.getName()}) from ${format( - from, - DATE_FORMAT - )} to ${format(to, DATE_FORMAT)}: [${error.name}] ${error.message}` - ); - } - } - - public getName(): DataSource { - return DataSource.GHOSTFOLIO; - } - - public async getQuotes( - aSymbols: string[] - ): Promise<{ [symbol: string]: IDataProviderResponse }> { - const response: { [symbol: string]: IDataProviderResponse } = {}; - - if (aSymbols.length <= 0) { - return response; - } - - try { - const symbolProfiles = - await this.symbolProfileService.getSymbolProfilesBySymbols(aSymbols); - - const marketData = await this.prismaService.marketData.findMany({ - distinct: ['symbol'], - orderBy: { - date: 'desc' - }, - take: aSymbols.length, - where: { - symbol: { - in: aSymbols - } - } - }); - - for (const symbolProfile of symbolProfiles) { - response[symbolProfile.symbol] = { - currency: symbolProfile.currency, - dataSource: this.getName(), - marketPrice: marketData.find((marketDataItem) => { - return marketDataItem.symbol === symbolProfile.symbol; - })?.marketPrice, - marketState: 'delayed' - }; - } - - return response; - } catch (error) { - Logger.error(error, 'GhostfolioScraperApiService'); - } - - return {}; - } - - public async search(aQuery: string): Promise<{ items: LookupItem[] }> { - const items = await this.prismaService.symbolProfile.findMany({ - select: { - currency: true, - dataSource: true, - name: true, - symbol: true - }, - where: { - OR: [ - { - dataSource: this.getName(), - name: { - mode: 'insensitive', - startsWith: aQuery - } - }, - { - dataSource: this.getName(), - symbol: { - mode: 'insensitive', - startsWith: aQuery - } - } - ] - } - }); - - return { items }; - } -} diff --git a/prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql b/prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql new file mode 100644 index 000000000..fbf129be4 --- /dev/null +++ b/prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql @@ -0,0 +1,2 @@ +UPDATE "MarketData" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO'; +UPDATE "SymbolProfile" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO';