From 893e76f83f5d29fb38a3f3139be8b0e1e73f4708 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Mon, 5 Feb 2024 19:55:39 +0100 Subject: [PATCH] Feature/provide data provider info in search (#2958) * Provide data provider info in search * Update changelog --- CHANGELOG.md | 1 + apps/api/src/app/import/import.controller.ts | 5 +- apps/api/src/app/import/import.service.ts | 50 +++++++---- .../interfaces/lookup-item.interface.ts | 2 + .../alpha-vantage/alpha-vantage.service.ts | 8 ++ .../coingecko/coingecko.service.ts | 15 ++-- .../data-provider/data-provider.service.ts | 85 +++++++++---------- .../eod-historical-data.service.ts | 10 ++- .../financial-modeling-prep.service.ts | 15 ++-- .../google-sheets/google-sheets.service.ts | 13 ++- .../interfaces/data-provider.interface.ts | 3 + .../data-provider/manual/manual.service.ts | 17 +++- .../rapid-api/rapid-api.service.ts | 7 ++ .../yahoo-finance/yahoo-finance.service.ts | 8 ++ .../data-provider-info.interface.ts | 5 +- .../activities-table-lazy.component.html | 4 +- .../symbol-autocomplete.component.html | 11 ++- .../symbol-autocomplete.module.ts | 2 + 18 files changed, 171 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73cc26f15..fcb81ecb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Extended the assistant by an asset class selector (experimental) +- Added the data provider information to the search endpoint ### Changed diff --git a/apps/api/src/app/import/import.controller.ts b/apps/api/src/app/import/import.controller.ts index 3697afb25..d04607de4 100644 --- a/apps/api/src/app/import/import.controller.ts +++ b/apps/api/src/app/import/import.controller.ts @@ -64,16 +64,13 @@ export class ImportController { maxActivitiesToImport = Number.MAX_SAFE_INTEGER; } - const userCurrency = this.request.user.Settings.settings.baseCurrency; - try { const activities = await this.importService.import({ isDryRun, maxActivitiesToImport, - userCurrency, accountsDto: importData.accounts ?? [], activitiesDto: importData.activities, - userId: this.request.user.id + user: this.request.user }); return { activities }; diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index d2daa4338..5416bffbd 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -21,7 +21,8 @@ import { import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { AccountWithPlatform, - OrderWithAccount + OrderWithAccount, + UserWithSettings } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; import { DataSource, Prisma, SymbolProfile } from '@prisma/client'; @@ -138,17 +139,16 @@ export class ImportService { activitiesDto, isDryRun = false, maxActivitiesToImport, - userCurrency, - userId + user }: { accountsDto: Partial[]; activitiesDto: Partial[]; isDryRun?: boolean; maxActivitiesToImport: number; - userCurrency: string; - userId: string; + user: UserWithSettings; }): Promise { const accountIdMapping: { [oldAccountId: string]: string } = {}; + const userCurrency = user.Settings.settings.baseCurrency; if (!isDryRun && accountsDto?.length) { const [existingAccounts, existingPlatforms] = await Promise.all([ @@ -171,7 +171,7 @@ export class ImportService { ); // If there is no account or if the account belongs to a different user then create a new account - if (!accountWithSameId || accountWithSameId.userId !== userId) { + if (!accountWithSameId || accountWithSameId.userId !== user.id) { let oldAccountId: string; const platformId = account.platformId; @@ -184,7 +184,7 @@ export class ImportService { let accountObject: Prisma.AccountCreateInput = { ...account, - User: { connect: { id: userId } } + User: { connect: { id: user.id } } }; if ( @@ -200,7 +200,7 @@ export class ImportService { const newAccount = await this.accountService.createAccount( accountObject, - userId + user.id ); // Store the new to old account ID mappings for updating activities @@ -231,16 +231,17 @@ export class ImportService { const assetProfiles = await this.validateActivities({ activitiesDto, - maxActivitiesToImport + maxActivitiesToImport, + user }); const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({ activitiesDto, userCurrency, - userId + userId: user.id }); - const accounts = (await this.accountService.getAccounts(userId)).map( + const accounts = (await this.accountService.getAccounts(user.id)).map( ({ id, name }) => { return { id, name }; } @@ -345,7 +346,6 @@ export class ImportService { quantity, type, unitPrice, - userId, accountId: validatedAccount?.id, accountUserId: undefined, createdAt: new Date(), @@ -374,7 +374,8 @@ export class ImportService { }, Account: validatedAccount, symbolProfileId: undefined, - updatedAt: new Date() + updatedAt: new Date(), + userId: user.id }; } else { if (error) { @@ -388,7 +389,6 @@ export class ImportService { quantity, type, unitPrice, - userId, accountId: validatedAccount?.id, SymbolProfile: { connectOrCreate: { @@ -406,7 +406,8 @@ export class ImportService { } }, updateAccountBalance: false, - User: { connect: { id: userId } } + User: { connect: { id: user.id } }, + userId: user.id }); } @@ -553,10 +554,12 @@ export class ImportService { private async validateActivities({ activitiesDto, - maxActivitiesToImport + maxActivitiesToImport, + user }: { activitiesDto: Partial[]; maxActivitiesToImport: number; + user: UserWithSettings; }) { if (activitiesDto?.length > maxActivitiesToImport) { throw new Error(`Too many activities (${maxActivitiesToImport} at most)`); @@ -583,6 +586,21 @@ export class ImportService { ); } + if ( + this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION') && + user.subscription.type === 'Basic' + ) { + const dataProvider = this.dataProviderService.getDataProvider( + DataSource[dataSource] + ); + + if (dataProvider.getDataProviderInfo().isPremium) { + throw new Error( + `activities.${index}.dataSource ("${dataSource}") is not valid` + ); + } + } + const assetProfile = { currency, ...( diff --git a/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts b/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts index e9c90b0bc..1931d1bfe 100644 --- a/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts +++ b/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts @@ -1,9 +1,11 @@ +import { DataProviderInfo } from '@ghostfolio/common/interfaces'; import { AssetClass, AssetSubClass, DataSource } from '@prisma/client'; export interface LookupItem { assetClass: AssetClass; assetSubClass: AssetSubClass; currency: string; + dataProviderInfo: DataProviderInfo; dataSource: DataSource; name: string; symbol: string; diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts index 53b882fda..6ab790159 100644 --- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts +++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts @@ -12,6 +12,7 @@ import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { DataProviderInfo } from '@ghostfolio/common/interfaces'; import { Injectable } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; import * as Alphavantage from 'alphavantage'; @@ -44,6 +45,12 @@ export class AlphaVantageService implements DataProviderInterface { }; } + public getDataProviderInfo(): DataProviderInfo { + return { + isPremium: false + }; + } + public async getDividends({}: GetDividendsParams) { return {}; } @@ -118,6 +125,7 @@ export class AlphaVantageService implements DataProviderInterface { assetClass: undefined, assetSubClass: undefined, currency: bestMatch['8. currency'], + dataProviderInfo: this.getDataProviderInfo(), dataSource: this.getName(), name: bestMatch['2. name'], symbol: bestMatch['1. symbol'] diff --git a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts index 6c6d73a8c..c9fd47a1d 100644 --- a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts +++ b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts @@ -91,6 +91,14 @@ export class CoinGeckoService implements DataProviderInterface { return response; } + public getDataProviderInfo(): DataProviderInfo { + return { + isPremium: false, + name: 'CoinGecko', + url: 'https://coingecko.com' + }; + } + public async getDividends({}: GetDividendsParams) { return {}; } @@ -252,11 +260,4 @@ export class CoinGeckoService implements DataProviderInterface { return { items }; } - - private getDataProviderInfo(): DataProviderInfo { - return { - name: 'CoinGecko', - url: 'https://coingecko.com' - }; - } } diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 57d970fd6..8868501d9 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -107,6 +107,31 @@ export class DataProviderService { return response; } + public getDataProvider(providerName: DataSource) { + for (const dataProviderInterface of this.dataProviderInterfaces) { + if (this.dataProviderMapping[dataProviderInterface.getName()]) { + const mappedDataProviderInterface = this.dataProviderInterfaces.find( + (currentDataProviderInterface) => { + return ( + currentDataProviderInterface.getName() === + this.dataProviderMapping[dataProviderInterface.getName()] + ); + } + ); + + if (mappedDataProviderInterface) { + return mappedDataProviderInterface; + } + } + + if (dataProviderInterface.getName() === providerName) { + return dataProviderInterface; + } + } + + throw new Error('No data provider has been found.'); + } + public getDataSourceForExchangeRates(): DataSource { return DataSource[ this.configurationService.get('DATA_SOURCE_EXCHANGE_RATES') @@ -520,20 +545,15 @@ export class DataProviderService { return { items: lookupItems }; } - let dataSources = this.configurationService.get('DATA_SOURCES'); - - if ( - this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION') && - user.subscription.type === 'Basic' - ) { - dataSources = dataSources.filter((dataSource) => { - return !this.isPremiumDataSource(DataSource[dataSource]); + let dataProviderServices = this.configurationService + .get('DATA_SOURCES') + .map((dataSource) => { + return this.getDataProvider(DataSource[dataSource]); }); - } - for (const dataSource of dataSources) { + for (const dataProviderService of dataProviderServices) { promises.push( - this.getDataProvider(DataSource[dataSource]).search({ + dataProviderService.search({ includeIndices, query }) @@ -555,6 +575,16 @@ export class DataProviderService { }) .sort(({ name: name1 }, { name: name2 }) => { return name1?.toLowerCase().localeCompare(name2?.toLowerCase()); + }) + .map((lookupItem) => { + if ( + !this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION') || + user.subscription.type === 'Premium' + ) { + lookupItem.dataProviderInfo.isPremium = false; + } + + return lookupItem; }); return { @@ -562,31 +592,6 @@ export class DataProviderService { }; } - private getDataProvider(providerName: DataSource) { - for (const dataProviderInterface of this.dataProviderInterfaces) { - if (this.dataProviderMapping[dataProviderInterface.getName()]) { - const mappedDataProviderInterface = this.dataProviderInterfaces.find( - (currentDataProviderInterface) => { - return ( - currentDataProviderInterface.getName() === - this.dataProviderMapping[dataProviderInterface.getName()] - ); - } - ); - - if (mappedDataProviderInterface) { - return mappedDataProviderInterface; - } - } - - if (dataProviderInterface.getName() === providerName) { - return dataProviderInterface; - } - } - - throw new Error('No data provider has been found.'); - } - private hasCurrency({ currency, dataGatheringItems @@ -602,14 +607,6 @@ export class DataProviderService { }); } - private isPremiumDataSource(aDataSource: DataSource) { - const premiumDataSources: DataSource[] = [ - DataSource.EOD_HISTORICAL_DATA, - DataSource.FINANCIAL_MODELING_PREP - ]; - return premiumDataSources.includes(aDataSource); - } - private transformHistoricalData({ allData, currency, diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts index d9ee298be..c46055999 100644 --- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts +++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts @@ -16,6 +16,7 @@ import { REPLACE_NAME_PARTS } from '@ghostfolio/common/config'; import { DATE_FORMAT, isCurrency } from '@ghostfolio/common/helper'; +import { DataProviderInfo } from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { AssetClass, @@ -58,6 +59,12 @@ export class EodHistoricalDataService implements DataProviderInterface { }; } + public getDataProviderInfo(): DataProviderInfo { + return { + isPremium: true + }; + } + public async getDividends({ from, requestTimeout = this.configurationService.get('REQUEST_TIMEOUT'), @@ -312,7 +319,8 @@ export class EodHistoricalDataService implements DataProviderInterface { dataSource, name, symbol, - currency: this.convertCurrency(currency) + currency: this.convertCurrency(currency), + dataProviderInfo: this.getDataProviderInfo() }; } ) diff --git a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts index 81cb96e90..fe1b18cc1 100644 --- a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts +++ b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts @@ -45,6 +45,14 @@ export class FinancialModelingPrepService implements DataProviderInterface { }; } + public getDataProviderInfo(): DataProviderInfo { + return { + isPremium: true, + name: 'Financial Modeling Prep', + url: 'https://financialmodelingprep.com/developer/docs' + }; + } + public async getDividends({}: GetDividendsParams) { return {}; } @@ -202,11 +210,4 @@ export class FinancialModelingPrepService implements DataProviderInterface { return { items }; } - - private getDataProviderInfo(): DataProviderInfo { - return { - name: 'Financial Modeling Prep', - url: 'https://financialmodelingprep.com/developer/docs' - }; - } } diff --git a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts index d52d09adf..d9f4115e0 100644 --- a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts +++ b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts @@ -14,6 +14,7 @@ import { import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper'; +import { DataProviderInfo } from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; import { format } from 'date-fns'; @@ -40,6 +41,12 @@ export class GoogleSheetsService implements DataProviderInterface { }; } + public getDataProviderInfo(): DataProviderInfo { + return { + isPremium: false + }; + } + public async getDividends({}: GetDividendsParams) { return {}; } @@ -177,7 +184,11 @@ export class GoogleSheetsService implements DataProviderInterface { } }); - return { items }; + return { + items: items.map((item) => { + return { ...item, dataProviderInfo: this.getDataProviderInfo() }; + }) + }; } private async getSheet({ diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts index 044836d82..924605f09 100644 --- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts @@ -3,6 +3,7 @@ import { IDataProviderHistoricalResponse, IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; +import { DataProviderInfo } from '@ghostfolio/common/interfaces'; import { Granularity } from '@ghostfolio/common/types'; import { DataSource, SymbolProfile } from '@prisma/client'; @@ -11,6 +12,8 @@ export interface DataProviderInterface { getAssetProfile(aSymbol: string): Promise>; + getDataProviderInfo(): DataProviderInfo; + getDividends({ from, granularity, symbol, to }: GetDividendsParams): Promise<{ [date: string]: IDataProviderHistoricalResponse; }>; diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 74acc48e1..33d93bbe2 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -18,7 +18,10 @@ import { extractNumberFromString, getYesterday } from '@ghostfolio/common/helper'; -import { ScraperConfiguration } from '@ghostfolio/common/interfaces'; +import { + DataProviderInfo, + ScraperConfiguration +} from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; import * as cheerio from 'cheerio'; @@ -59,6 +62,12 @@ export class ManualService implements DataProviderInterface { return assetProfile; } + public getDataProviderInfo(): DataProviderInfo { + return { + isPremium: false + }; + } + public async getDividends({}: GetDividendsParams) { return {}; } @@ -214,7 +223,11 @@ export class ManualService implements DataProviderInterface { return !isUUID(symbol); }); - return { items }; + return { + items: items.map((item) => { + return { ...item, dataProviderInfo: this.getDataProviderInfo() }; + }) + }; } public async test(scraperConfiguration: ScraperConfiguration) { diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts index d9b4bd0e4..31c813180 100644 --- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts @@ -13,6 +13,7 @@ import { } from '@ghostfolio/api/services/interfaces/interfaces'; import { ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config'; import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper'; +import { DataProviderInfo } from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; import { format } from 'date-fns'; @@ -37,6 +38,12 @@ export class RapidApiService implements DataProviderInterface { }; } + public getDataProviderInfo(): DataProviderInfo { + return { + isPremium: false + }; + } + public async getDividends({}: GetDividendsParams) { return {}; } diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index 47869d3e8..335162023 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -14,6 +14,7 @@ import { } from '@ghostfolio/api/services/interfaces/interfaces'; import { DEFAULT_CURRENCY } from '@ghostfolio/common/config'; import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { DataProviderInfo } from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; import { addDays, format, isSameDay } from 'date-fns'; @@ -47,6 +48,12 @@ export class YahooFinanceService implements DataProviderInterface { }; } + public getDataProviderInfo(): DataProviderInfo { + return { + isPremium: false + }; + } + public async getDividends({ from, granularity = 'day', @@ -283,6 +290,7 @@ export class YahooFinanceService implements DataProviderInterface { assetSubClass, symbol, currency: marketDataItem.currency, + dataProviderInfo: this.getDataProviderInfo(), dataSource: this.getName(), name: this.yahooFinanceDataEnhancerService.formatName({ longName: quote.longname, diff --git a/libs/common/src/lib/interfaces/data-provider-info.interface.ts b/libs/common/src/lib/interfaces/data-provider-info.interface.ts index 59f3a0b69..79d7d6940 100644 --- a/libs/common/src/lib/interfaces/data-provider-info.interface.ts +++ b/libs/common/src/lib/interfaces/data-provider-info.interface.ts @@ -1,4 +1,5 @@ export interface DataProviderInfo { - name: string; - url: string; + isPremium: boolean; + name?: string; + url?: string; } diff --git a/libs/ui/src/lib/activities-table-lazy/activities-table-lazy.component.html b/libs/ui/src/lib/activities-table-lazy/activities-table-lazy.component.html index bc2851e97..543e496f8 100644 --- a/libs/ui/src/lib/activities-table-lazy/activities-table-lazy.component.html +++ b/libs/ui/src/lib/activities-table-lazy/activities-table-lazy.component.html @@ -129,8 +129,8 @@ Name - -
+ +
{{ element.SymbolProfile?.name }} - {{ lookupItem.name }} -
+ {{ lookupItem.name }} + @if (lookupItem.dataProviderInfo.isPremium) { + + } + {{ lookupItem.symbol | gfSymbol }} ยท {{ lookupItem.currency }} diff --git a/libs/ui/src/lib/symbol-autocomplete/symbol-autocomplete.module.ts b/libs/ui/src/lib/symbol-autocomplete/symbol-autocomplete.module.ts index d7b1ed2f8..259403043 100644 --- a/libs/ui/src/lib/symbol-autocomplete/symbol-autocomplete.module.ts +++ b/libs/ui/src/lib/symbol-autocomplete/symbol-autocomplete.module.ts @@ -7,6 +7,7 @@ import { MatInputModule } from '@angular/material/input'; import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { GfSymbolModule } from '@ghostfolio/client/pipes/symbol/symbol.module'; import { SymbolAutocompleteComponent } from '@ghostfolio/ui/symbol-autocomplete/symbol-autocomplete.component'; +import { GfPremiumIndicatorModule } from '@ghostfolio/ui/premium-indicator'; @NgModule({ declarations: [SymbolAutocompleteComponent], @@ -14,6 +15,7 @@ import { SymbolAutocompleteComponent } from '@ghostfolio/ui/symbol-autocomplete/ imports: [ CommonModule, FormsModule, + GfPremiumIndicatorModule, GfSymbolModule, MatAutocompleteModule, MatFormFieldModule,