diff --git a/.gitignore b/.gitignore index ab31ae269..8071bf0b9 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ npm-debug.log .env.prod .github/instructions/nx.instructions.md .nx/cache +.nx/migrate-runs .nx/polygraph .nx/self-healing .nx/workspace-data diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e3ae9649..9dfeb76fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,16 +5,95 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## 3.17.0 - 2026-06-26 + +### Added + +- Added `zod` as a root dependency to resolve peer dependency warnings + +### Changed + +- Improved the error message styling in the import activities dialog +- Improved the grantee display in the access table to share the portfolio +- Improved the country mapping for data providers +- Upgraded `bull-board` from version `7.2.1` to `8.0.1` +- Upgraded `Nx` from version `22.7.5` to `23.0.1` +- Upgraded `prettier` from version `3.8.3` to `3.8.4` + +### Fixed + +- Improved the table headers’ alignment in the queue jobs table of the admin control panel + +## 3.16.0 - 2026-06-24 + +### Added + +- Extended the user account settings with a copy-to-clipboard button for the user id +- Added pagination to the platform management of the admin control panel +- Added pagination to the tag management of the admin control panel +- Extended the asset profile details dialog of the admin control panel with a copy-to-clipboard button for the ISIN number +- Extended the asset profile details dialog of the admin control panel with a copy-to-clipboard button for the symbol + +### Changed + +- Improved the throughput of the market data gathering queue by applying the rate limit per data source +- Decreased the rate limiter duration of the market data gathering queue jobs from 4 to 3 seconds +- Removed the deprecated `SymbolProfile` field from the endpoint `GET api/v1/portfolio/holding/:dataSource/:symbol` +- Upgraded `@simplewebauthn/browser` and `@simplewebauthn/server` from version `13.2.2` to `13.3` + +### Fixed + +- Fixed an issue with hourly market data updates not refreshing prices for asset profiles with `MANUAL` data source +- Fixed an issue with the log context formatting in the performance logging service + +## 3.15.1 - 2026-06-23 + +### Changed + +- Improved the dynamic numerical precision for various values in the account detail dialog on mobile +- Improved the dynamic numerical precision for various values in the holding detail dialog on mobile +- Upgraded `@internationalized/number` from version `3.6.6` to `3.6.7` + +### Fixed + +- Fixed an issue where symbols with special characters caused API request failures by URL encoding the symbol +- Fixed the disabled state of the delete action in the asset profiles actions menu of the historical market data table in the admin control panel +- Fixed the persistence of an empty `locale` string in the scraper configuration +- Fixed a transaction timeout that prevented gathering historical market data for symbols with a long history +- Fixed an exception in various portfolio endpoints when historical exchange rate data is missing + +## 3.14.0 - 2026-06-22 + +### Added + +- Exposed the `ENABLE_FEATURE_CRON` environment variable to control scheduled cron job execution +- Exposed the `PROCESSOR_GATHER_STATISTICS_CONCURRENCY` environment variable to control the concurrency of the statistics gathering queue processor + +### Changed + +- Consolidated the exchange rates to be gathered with hourly market data +- Improved the language localization for German (`de`) +- Upgraded `@openrouter/ai-sdk-provider` from version `2.9.0` to `2.9.1` +- Upgraded `undici` from version `7.24.4` to `8.5.0` + +### Fixed + +- Fixed an issue in the data provider service where asset profiles and historical data could be missing for symbols that exist in multiple data sources by keying the responses by the asset profile identifier +- Resolved an exception in the benchmarks service when the current market price is unavailable + +## 3.13.0 - 2026-06-20 ### Added - Added an icon to indicate external links in the page tabs component - Added the Korean (`ko`) language to the footer +- Added a data gathering frequency (`DAILY` or `HOURLY`) to the asset profile to control the market data gathering interval ### Changed +- Changed the _Fear & Greed Index_ (market mood) in the markets overview to use the stored market data instead of a live quote - Moved the endpoint to get the asset profiles from `GET api/v1/admin/market-data` to `GET api/v1/asset-profiles` +- Moved the endpoint to get the asset profile details from `GET api/v1/market-data/:dataSource/:symbol` to `GET api/v1/asset-profiles/:dataSource/:symbol` - Added the selected asset profile count to the delete menu item of the historical market data table in the admin control panel - Added the selected asset profile count to the deletion confirmation dialog of the historical market data table in the admin control panel - Improved the sorting to be case-insensitive in the platform management of the admin control panel @@ -25,6 +104,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed an issue with the localization of the country names +- Fixed an issue in the data provider service where quotes could be missing for symbols that exist in multiple data sources by keying the quotes response by the asset profile identifier ## 3.12.0 - 2026-06-17 diff --git a/apps/api/src/app/activities/activities.service.ts b/apps/api/src/app/activities/activities.service.ts index f57507e3d..44ab67e45 100644 --- a/apps/api/src/app/activities/activities.service.ts +++ b/apps/api/src/app/activities/activities.service.ts @@ -746,10 +746,10 @@ export class ActivitiesService { const value = new Big(order.quantity).mul(order.unitPrice).toNumber(); const [ - feeInAssetProfileCurrency, - feeInBaseCurrency, - unitPriceInAssetProfileCurrency, - valueInBaseCurrency + feeInAssetProfileCurrency = 0, + feeInBaseCurrency = 0, + unitPriceInAssetProfileCurrency = 0, + valueInBaseCurrency = 0 ] = await Promise.all([ this.exchangeRateDataService.toCurrencyAtDate( order.fee, diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 9eb045fba..a6f6e8b6f 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -86,8 +86,8 @@ export class AdminController { @HasPermission(permissions.accessAdminControl) @Post('gather') @UseGuards(AuthGuard('jwt'), HasPermissionGuard) - public async gather7Days(): Promise { - this.dataGatheringService.gather7Days(); + public async gatherRecentMarketData(): Promise { + this.dataGatheringService.gatherRecentMarketData(); } @HasPermission(permissions.accessAdminControl) diff --git a/apps/api/src/app/admin/admin.module.ts b/apps/api/src/app/admin/admin.module.ts index 0cd4d3c16..2d5c734dc 100644 --- a/apps/api/src/app/admin/admin.module.ts +++ b/apps/api/src/app/admin/admin.module.ts @@ -1,4 +1,3 @@ -import { ActivitiesModule } from '@ghostfolio/api/app/activities/activities.module'; import { TransformDataSourceInRequestModule } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.module'; import { BenchmarkModule } from '@ghostfolio/api/services/benchmark/benchmark.module'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; @@ -19,7 +18,6 @@ import { QueueModule } from './queue/queue.module'; @Module({ imports: [ - ActivitiesModule, BenchmarkModule, ConfigurationModule, DataGatheringQueueModule, diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts index f08fb6a37..e50c0c77f 100644 --- a/apps/api/src/app/admin/admin.service.ts +++ b/apps/api/src/app/admin/admin.service.ts @@ -1,4 +1,3 @@ -import { ActivitiesService } from '@ghostfolio/api/app/activities/activities.service'; import { environment } from '@ghostfolio/api/environments/environment'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; @@ -12,14 +11,15 @@ import { PROPERTY_IS_READ_ONLY_MODE, PROPERTY_IS_USER_SIGNUP_ENABLED } from '@ghostfolio/common/config'; -import { getCurrencyFromSymbol, isCurrency } from '@ghostfolio/common/helper'; +import { + getAssetProfileIdentifier, + getCurrencyFromSymbol +} from '@ghostfolio/common/helper'; import { AdminData, - AdminMarketDataDetails, AdminUserResponse, AdminUsersResponse, - AssetProfileIdentifier, - EnhancedSymbolProfile + AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { @@ -42,7 +42,6 @@ import { StatusCodes, getReasonPhrase } from 'http-status-codes'; @Injectable() export class AdminService { public constructor( - private readonly activitiesService: ActivitiesService, private readonly configurationService: ConfigurationService, private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, @@ -72,14 +71,17 @@ export class AdminService { { dataSource, symbol } ]); - if (!assetProfiles[symbol]?.currency) { + const assetProfile = + assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (!assetProfile?.currency) { throw new BadRequestException( `Asset profile not found for ${symbol} (${dataSource})` ); } return this.symbolProfileService.add( - assetProfiles[symbol] as Prisma.SymbolProfileCreateInput + assetProfile as Prisma.SymbolProfileCreateInput ); } catch (error) { if ( @@ -176,61 +178,6 @@ export class AdminService { }; } - public async getMarketDataBySymbol({ - dataSource, - symbol - }: AssetProfileIdentifier): Promise { - let activitiesCount: EnhancedSymbolProfile['activitiesCount'] = 0; - let currency: EnhancedSymbolProfile['currency'] = '-'; - let dateOfFirstActivity: EnhancedSymbolProfile['dateOfFirstActivity']; - - const isCurrencyAssetProfile = isCurrency(getCurrencyFromSymbol(symbol)); - - if (isCurrencyAssetProfile) { - currency = getCurrencyFromSymbol(symbol); - ({ activitiesCount, dateOfFirstActivity } = - await this.activitiesService.getStatisticsByCurrency(currency)); - } - - const [[assetProfile], marketData] = await Promise.all([ - this.symbolProfileService.getSymbolProfiles([ - { - dataSource, - symbol - } - ]), - this.marketDataService.marketDataItems({ - orderBy: { - date: 'asc' - }, - where: { - dataSource, - symbol - } - }) - ]); - - if (assetProfile) { - assetProfile.dataProviderInfo = this.dataProviderService - .getDataProvider(assetProfile.dataSource) - .getDataProviderInfo(); - } - - return { - marketData, - assetProfile: assetProfile ?? { - activitiesCount, - currency, - dataSource, - dateOfFirstActivity, - symbol, - assetClass: isCurrencyAssetProfile ? AssetClass.LIQUIDITY : undefined, - assetSubClass: isCurrencyAssetProfile ? AssetSubClass.CASH : undefined, - isActive: true - } - }; - } - public async getUser(id: string): Promise { const [user] = await this.getUsersWithAnalytics({ where: { id } @@ -241,7 +188,7 @@ export class AdminService { } if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) { - const subscriptions = await this.prismaService.subscription.findMany({ + user.subscriptions = await this.prismaService.subscription.findMany({ orderBy: { expiresAt: 'desc' }, @@ -249,13 +196,6 @@ export class AdminService { userId: id } }); - - user.subscriptions = subscriptions.map((subscription) => { - return { - ...subscription, - price: subscription.price ?? 0 - }; - }); } return user; @@ -287,6 +227,7 @@ export class AdminService { comment, countries, currency, + dataGatheringFrequency, dataSource: newDataSource, holdings, isActive, @@ -370,6 +311,7 @@ export class AdminService { const updatedSymbolProfile: Prisma.SymbolProfileUpdateInput = { comment, currency, + dataGatheringFrequency, dataSource, isActive, scraperConfiguration, diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.ts index f47b3d3a1..d31834eea 100644 --- a/apps/api/src/app/admin/queue/queue.service.ts +++ b/apps/api/src/app/admin/queue/queue.service.ts @@ -56,7 +56,7 @@ export class QueueService { } public async getJobs({ - limit = 1000, + limit = 5000, status = QUEUE_JOB_STATUS_LIST }: { limit?: number; diff --git a/apps/api/src/app/asset/asset.controller.ts b/apps/api/src/app/asset/asset.controller.ts index 3b2031084..de04b5c81 100644 --- a/apps/api/src/app/asset/asset.controller.ts +++ b/apps/api/src/app/asset/asset.controller.ts @@ -1,4 +1,4 @@ -import { AdminService } from '@ghostfolio/api/app/admin/admin.service'; +import { AssetProfilesService } from '@ghostfolio/api/app/endpoints/asset-profiles/asset-profiles.service'; import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor'; import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor'; import type { AssetResponse } from '@ghostfolio/common/interfaces'; @@ -9,7 +9,9 @@ import { pick } from 'lodash'; @Controller('asset') export class AssetController { - public constructor(private readonly adminService: AdminService) {} + public constructor( + private readonly assetProfilesService: AssetProfilesService + ) {} @Get(':dataSource/:symbol') @UseInterceptors(TransformDataSourceInRequestInterceptor) @@ -19,7 +21,10 @@ export class AssetController { @Param('symbol') symbol: string ): Promise { const { assetProfile, marketData } = - await this.adminService.getMarketDataBySymbol({ dataSource, symbol }); + await this.assetProfilesService.getAssetProfile({ + dataSource, + symbol + }); return { marketData, diff --git a/apps/api/src/app/asset/asset.module.ts b/apps/api/src/app/asset/asset.module.ts index 168585ed8..311c8694f 100644 --- a/apps/api/src/app/asset/asset.module.ts +++ b/apps/api/src/app/asset/asset.module.ts @@ -1,4 +1,4 @@ -import { AdminModule } from '@ghostfolio/api/app/admin/admin.module'; +import { AssetProfilesModule } from '@ghostfolio/api/app/endpoints/asset-profiles/asset-profiles.module'; import { TransformDataSourceInRequestModule } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.module'; import { TransformDataSourceInResponseModule } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.module'; @@ -9,7 +9,7 @@ import { AssetController } from './asset.controller'; @Module({ controllers: [AssetController], imports: [ - AdminModule, + AssetProfilesModule, TransformDataSourceInRequestModule, TransformDataSourceInResponseModule ] diff --git a/apps/api/src/app/endpoints/asset-profiles/asset-profiles.controller.ts b/apps/api/src/app/endpoints/asset-profiles/asset-profiles.controller.ts index 2606d8075..fcddb0bc3 100644 --- a/apps/api/src/app/endpoints/asset-profiles/asset-profiles.controller.ts +++ b/apps/api/src/app/endpoints/asset-profiles/asset-profiles.controller.ts @@ -1,11 +1,17 @@ import { HasPermission } from '@ghostfolio/api/decorators/has-permission.decorator'; import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'; +import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor'; +import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor'; import { ApiService } from '@ghostfolio/api/services/api/api.service'; +import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { UpdateAssetProfileDataDto } from '@ghostfolio/common/dtos'; +import { getCurrencyFromSymbol, isCurrency } from '@ghostfolio/common/helper'; +import { AssetProfileResponse } from '@ghostfolio/common/interfaces'; import { AssetProfilesResponse, EnhancedSymbolProfile } from '@ghostfolio/common/interfaces'; +import { hasPermission } from '@ghostfolio/common/permissions'; import { permissions } from '@ghostfolio/common/permissions'; import { MarketDataPreset, RequestWithUser } from '@ghostfolio/common/types'; @@ -18,7 +24,8 @@ import { Param, Patch, Query, - UseGuards + UseGuards, + UseInterceptors } from '@nestjs/common'; import { REQUEST } from '@nestjs/core'; import { AuthGuard } from '@nestjs/passport'; @@ -32,7 +39,8 @@ export class AssetProfilesController { public constructor( private readonly apiService: ApiService, private readonly assetProfilesService: AssetProfilesService, - @Inject(REQUEST) private readonly request: RequestWithUser + @Inject(REQUEST) private readonly request: RequestWithUser, + private readonly symbolProfileService: SymbolProfileService ) {} @Get() @@ -64,6 +72,52 @@ export class AssetProfilesController { }); } + @Get(':dataSource/:symbol') + @UseGuards(AuthGuard('jwt')) + @UseInterceptors(TransformDataSourceInRequestInterceptor) + @UseInterceptors(TransformDataSourceInResponseInterceptor) + public async getAssetProfile( + @Param('dataSource') dataSource: DataSource, + @Param('symbol') symbol: string + ): Promise { + const [assetProfile] = await this.symbolProfileService.getSymbolProfiles([ + { dataSource, symbol } + ]); + + if (!assetProfile && !isCurrency(getCurrencyFromSymbol(symbol))) { + throw new HttpException( + getReasonPhrase(StatusCodes.NOT_FOUND), + StatusCodes.NOT_FOUND + ); + } + + const canReadAllAssetProfiles = hasPermission( + this.request.user.permissions, + permissions.readMarketData + ); + + const canReadOwnAssetProfile = + assetProfile?.userId === this.request.user.id && + hasPermission( + this.request.user.permissions, + permissions.readMarketDataOfOwnAssetProfile + ); + + if (!canReadAllAssetProfiles && !canReadOwnAssetProfile) { + throw new HttpException( + assetProfile?.userId + ? getReasonPhrase(StatusCodes.NOT_FOUND) + : getReasonPhrase(StatusCodes.FORBIDDEN), + assetProfile?.userId ? StatusCodes.NOT_FOUND : StatusCodes.FORBIDDEN + ); + } + + return this.assetProfilesService.getAssetProfile({ + dataSource, + symbol + }); + } + @HasPermission(permissions.accessAdminControl) @Patch(':dataSource/:symbol') @UseGuards(AuthGuard('jwt'), HasPermissionGuard) diff --git a/apps/api/src/app/endpoints/asset-profiles/asset-profiles.module.ts b/apps/api/src/app/endpoints/asset-profiles/asset-profiles.module.ts index 98463ce5d..f96d92eb3 100644 --- a/apps/api/src/app/endpoints/asset-profiles/asset-profiles.module.ts +++ b/apps/api/src/app/endpoints/asset-profiles/asset-profiles.module.ts @@ -1,7 +1,11 @@ import { ActivitiesModule } from '@ghostfolio/api/app/activities/activities.module'; +import { TransformDataSourceInRequestModule } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.module'; +import { TransformDataSourceInResponseModule } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.module'; import { ApiModule } from '@ghostfolio/api/services/api/api.module'; import { BenchmarkModule } from '@ghostfolio/api/services/benchmark/benchmark.module'; +import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; +import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; @@ -12,13 +16,18 @@ import { AssetProfilesService } from './asset-profiles.service'; @Module({ controllers: [AssetProfilesController], + exports: [AssetProfilesService], imports: [ ActivitiesModule, ApiModule, BenchmarkModule, + DataProviderModule, ExchangeRateDataModule, + MarketDataModule, PrismaModule, - SymbolProfileModule + SymbolProfileModule, + TransformDataSourceInRequestModule, + TransformDataSourceInResponseModule ], providers: [AssetProfilesService] }) diff --git a/apps/api/src/app/endpoints/asset-profiles/asset-profiles.service.ts b/apps/api/src/app/endpoints/asset-profiles/asset-profiles.service.ts index 39eaa1642..3b0828868 100644 --- a/apps/api/src/app/endpoints/asset-profiles/asset-profiles.service.ts +++ b/apps/api/src/app/endpoints/asset-profiles/asset-profiles.service.ts @@ -1,6 +1,8 @@ import { ActivitiesService } from '@ghostfolio/api/app/activities/activities.service'; import { BenchmarkService } from '@ghostfolio/api/services/benchmark/benchmark.service'; +import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { UpdateAssetProfileDataDto } from '@ghostfolio/common/dtos'; @@ -11,8 +13,9 @@ import { isCurrency } from '@ghostfolio/common/helper'; import { - AssetProfileItem, + AdminMarketDataDetails, AssetProfileIdentifier, + AssetProfileItem, AssetProfilesResponse, EnhancedSymbolProfile, Filter @@ -28,11 +31,67 @@ export class AssetProfilesService { public constructor( private readonly activitiesService: ActivitiesService, private readonly benchmarkService: BenchmarkService, + private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, + private readonly marketDataService: MarketDataService, private readonly prismaService: PrismaService, private readonly symbolProfileService: SymbolProfileService ) {} + public async getAssetProfile({ + dataSource, + symbol + }: AssetProfileIdentifier): Promise { + let activitiesCount: EnhancedSymbolProfile['activitiesCount'] = 0; + let currency: EnhancedSymbolProfile['currency'] = '-'; + let dateOfFirstActivity: EnhancedSymbolProfile['dateOfFirstActivity']; + + const isCurrencyAssetProfile = isCurrency(getCurrencyFromSymbol(symbol)); + + if (isCurrencyAssetProfile) { + currency = getCurrencyFromSymbol(symbol); + ({ activitiesCount, dateOfFirstActivity } = + await this.activitiesService.getStatisticsByCurrency(currency)); + } + + const [[assetProfile], marketData] = await Promise.all([ + this.symbolProfileService.getSymbolProfiles([ + { + dataSource, + symbol + } + ]), + this.marketDataService.marketDataItems({ + orderBy: { + date: 'asc' + }, + where: { + dataSource, + symbol + } + }) + ]); + + if (assetProfile) { + assetProfile.dataProviderInfo = this.dataProviderService + .getDataProvider(assetProfile.dataSource) + .getDataProviderInfo(); + } + + return { + marketData, + assetProfile: assetProfile ?? { + activitiesCount, + currency, + dataSource, + dateOfFirstActivity, + symbol, + assetClass: isCurrencyAssetProfile ? AssetClass.LIQUIDITY : undefined, + assetSubClass: isCurrencyAssetProfile ? AssetSubClass.CASH : undefined, + isActive: true + } + }; + } public async getAssetProfiles({ filters = [], presetId, diff --git a/apps/api/src/app/endpoints/benchmarks/benchmarks.service.ts b/apps/api/src/app/endpoints/benchmarks/benchmarks.service.ts index 0b95880d4..1fe42ab0d 100644 --- a/apps/api/src/app/endpoints/benchmarks/benchmarks.service.ts +++ b/apps/api/src/app/endpoints/benchmarks/benchmarks.service.ts @@ -81,6 +81,14 @@ export class BenchmarksService { }) ]); + if (!currentSymbolItem) { + this.logger.error( + `No current market price is available for ${symbol} (${dataSource})` + ); + + return { marketData }; + } + const exchangeRates = await this.exchangeRateDataService.getExchangeRatesByCurrency({ startDate, diff --git a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts index b84ca881f..caff86ae0 100644 --- a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts +++ b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts @@ -16,6 +16,7 @@ import { DERIVED_CURRENCIES } from '@ghostfolio/common/config'; import { PROPERTY_DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER_MAX_REQUESTS } from '@ghostfolio/common/config'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { DataProviderGhostfolioAssetProfileResponse, DataProviderHistoricalResponse, @@ -60,7 +61,13 @@ export class GhostfolioService { } ]) .then(async (assetProfiles) => { - const assetProfile = assetProfiles[symbol]; + const assetProfile = + assetProfiles[ + getAssetProfileIdentifier({ + symbol, + dataSource: dataProviderService.getName() + }) + ]; const dataSourceOrigin = DataSource.GHOSTFOLIO; if (assetProfile) { diff --git a/apps/api/src/app/endpoints/market-data/market-data.controller.ts b/apps/api/src/app/endpoints/market-data/market-data.controller.ts index f6857283b..0905a50bb 100644 --- a/apps/api/src/app/endpoints/market-data/market-data.controller.ts +++ b/apps/api/src/app/endpoints/market-data/market-data.controller.ts @@ -1,9 +1,6 @@ -import { AdminService } from '@ghostfolio/api/app/admin/admin.service'; import { SymbolService } from '@ghostfolio/api/app/symbol/symbol.service'; import { HasPermission } from '@ghostfolio/api/decorators/has-permission.decorator'; import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'; -import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor'; -import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { @@ -14,10 +11,7 @@ import { } from '@ghostfolio/common/config'; import { UpdateBulkMarketDataDto } from '@ghostfolio/common/dtos'; import { getCurrencyFromSymbol, isCurrency } from '@ghostfolio/common/helper'; -import { - MarketDataDetailsResponse, - MarketDataOfMarketsResponse -} from '@ghostfolio/common/interfaces'; +import { MarketDataOfMarketsResponse } from '@ghostfolio/common/interfaces'; import { hasPermission, permissions } from '@ghostfolio/common/permissions'; import { RequestWithUser } from '@ghostfolio/common/types'; @@ -30,8 +24,7 @@ import { Param, Post, Query, - UseGuards, - UseInterceptors + UseGuards } from '@nestjs/common'; import { REQUEST } from '@nestjs/core'; import { AuthGuard } from '@nestjs/passport'; @@ -42,7 +35,6 @@ import { getReasonPhrase, StatusCodes } from 'http-status-codes'; @Controller('market-data') export class MarketDataController { public constructor( - private readonly adminService: AdminService, private readonly marketDataService: MarketDataService, @Inject(REQUEST) private readonly request: RequestWithUser, private readonly symbolProfileService: SymbolProfileService, @@ -64,14 +56,16 @@ export class MarketDataController { dataGatheringItem: { dataSource: ghostfolioFearAndGreedIndexDataSourceCryptocurrencies, symbol: ghostfolioFearAndGreedIndexSymbolCryptocurrencies - } + }, + useIntradayData: true }), this.symbolService.get({ includeHistoricalData, dataGatheringItem: { dataSource: ghostfolioFearAndGreedIndexDataSourceStocks, symbol: ghostfolioFearAndGreedIndexSymbolStocks - } + }, + useIntradayData: true }) ]); @@ -87,49 +81,6 @@ export class MarketDataController { }; } - @Get(':dataSource/:symbol') - @UseGuards(AuthGuard('jwt')) - @UseInterceptors(TransformDataSourceInRequestInterceptor) - @UseInterceptors(TransformDataSourceInResponseInterceptor) - public async getMarketDataBySymbol( - @Param('dataSource') dataSource: DataSource, - @Param('symbol') symbol: string - ): Promise { - const [assetProfile] = await this.symbolProfileService.getSymbolProfiles([ - { dataSource, symbol } - ]); - - if (!assetProfile && !isCurrency(getCurrencyFromSymbol(symbol))) { - throw new HttpException( - getReasonPhrase(StatusCodes.NOT_FOUND), - StatusCodes.NOT_FOUND - ); - } - - const canReadAllAssetProfiles = hasPermission( - this.request.user.permissions, - permissions.readMarketData - ); - - const canReadOwnAssetProfile = - assetProfile?.userId === this.request.user.id && - hasPermission( - this.request.user.permissions, - permissions.readMarketDataOfOwnAssetProfile - ); - - if (!canReadAllAssetProfiles && !canReadOwnAssetProfile) { - throw new HttpException( - assetProfile?.userId - ? getReasonPhrase(StatusCodes.NOT_FOUND) - : getReasonPhrase(StatusCodes.FORBIDDEN), - assetProfile?.userId ? StatusCodes.NOT_FOUND : StatusCodes.FORBIDDEN - ); - } - - return this.adminService.getMarketDataBySymbol({ dataSource, symbol }); - } - @Post(':dataSource/:symbol') @UseGuards(AuthGuard('jwt')) public async updateMarketData( diff --git a/apps/api/src/app/endpoints/market-data/market-data.module.ts b/apps/api/src/app/endpoints/market-data/market-data.module.ts index d5d64673d..1de10907b 100644 --- a/apps/api/src/app/endpoints/market-data/market-data.module.ts +++ b/apps/api/src/app/endpoints/market-data/market-data.module.ts @@ -1,7 +1,4 @@ -import { AdminModule } from '@ghostfolio/api/app/admin/admin.module'; import { SymbolModule } from '@ghostfolio/api/app/symbol/symbol.module'; -import { TransformDataSourceInRequestModule } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.module'; -import { TransformDataSourceInResponseModule } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.module'; import { MarketDataModule as MarketDataServiceModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; @@ -11,13 +8,6 @@ import { MarketDataController } from './market-data.controller'; @Module({ controllers: [MarketDataController], - imports: [ - AdminModule, - MarketDataServiceModule, - SymbolModule, - SymbolProfileModule, - TransformDataSourceInRequestModule, - TransformDataSourceInResponseModule - ] + imports: [MarketDataServiceModule, SymbolModule, SymbolProfileModule] }) export class MarketDataModule {} diff --git a/apps/api/src/app/endpoints/watchlist/watchlist.service.ts b/apps/api/src/app/endpoints/watchlist/watchlist.service.ts index 666023dbf..88702da00 100644 --- a/apps/api/src/app/endpoints/watchlist/watchlist.service.ts +++ b/apps/api/src/app/endpoints/watchlist/watchlist.service.ts @@ -4,10 +4,14 @@ import { MarketDataService } from '@ghostfolio/api/services/market-data/market-d import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; -import { WatchlistResponse } from '@ghostfolio/common/interfaces'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; +import { + AssetProfileIdentifier, + WatchlistResponse +} from '@ghostfolio/common/interfaces'; import { BadRequestException, Injectable } from '@nestjs/common'; -import { DataSource, Prisma } from '@prisma/client'; +import { Prisma } from '@prisma/client'; @Injectable() export class WatchlistService { @@ -24,11 +28,7 @@ export class WatchlistService { dataSource, symbol, userId - }: { - dataSource: DataSource; - symbol: string; - userId: string; - }): Promise { + }: { userId: string } & AssetProfileIdentifier): Promise { const symbolProfile = await this.prismaService.symbolProfile.findUnique({ where: { dataSource_symbol: { dataSource, symbol } @@ -40,14 +40,17 @@ export class WatchlistService { { dataSource, symbol } ]); - if (!assetProfiles[symbol]?.currency) { + const assetProfile = + assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (!assetProfile?.currency) { throw new BadRequestException( `Asset profile not found for ${symbol} (${dataSource})` ); } await this.symbolProfileService.add( - assetProfiles[symbol] as Prisma.SymbolProfileCreateInput + assetProfile as Prisma.SymbolProfileCreateInput ); } @@ -72,11 +75,7 @@ export class WatchlistService { dataSource, symbol, userId - }: { - dataSource: DataSource; - symbol: string; - userId: string; - }) { + }: { userId: string } & AssetProfileIdentifier) { await this.prismaService.user.update({ data: { watchlist: { @@ -127,7 +126,8 @@ export class WatchlistService { const performancePercent = this.benchmarkService.calculateChangeInPercentage( allTimeHigh?.marketPrice, - quotes[symbol]?.marketPrice + quotes[getAssetProfileIdentifier({ dataSource, symbol })] + ?.marketPrice ); return { diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index b82f763a0..ba704d5ad 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -536,6 +536,8 @@ export class ImportService { url, comment: assetProfile.comment, currency: assetProfile.currency, + dataGatheringFrequency: + assetProfile.dataGatheringFrequency ?? 'DAILY', userId: dataSource === 'MANUAL' ? user.id : undefined }, symbolProfileId: undefined, @@ -590,18 +592,19 @@ export class ImportService { const value = new Big(quantity).mul(unitPrice).toNumber(); - const valueInBaseCurrency = this.exchangeRateDataService.toCurrencyAtDate( - value, - currency ?? assetProfile.currency, - userCurrency, - date - ); + const valueInBaseCurrency = + (await this.exchangeRateDataService.toCurrencyAtDate( + value, + currency ?? assetProfile.currency, + userCurrency, + date + )) ?? 0; activities.push({ ...order, error, value, - valueInBaseCurrency: await valueInBaseCurrency, + valueInBaseCurrency, // @ts-ignore SymbolProfile: assetProfile }); diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.ts index 86630db53..4441036a2 100644 --- a/apps/api/src/app/info/info.service.ts +++ b/apps/api/src/app/info/info.service.ts @@ -1,6 +1,7 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service'; import { SubscriptionService } from '@ghostfolio/api/app/subscription/subscription.service'; import { UserService } from '@ghostfolio/api/app/user/user.service'; +import { encodeDataSource } from '@ghostfolio/api/helper/data-source.helper'; import { BenchmarkService } from '@ghostfolio/api/services/benchmark/benchmark.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; @@ -17,7 +18,6 @@ import { PROPERTY_UPTIME, ghostfolioFearAndGreedIndexDataSourceStocks } from '@ghostfolio/common/config'; -import { encodeDataSource } from '@ghostfolio/common/helper'; import { InfoItem, Statistics } from '@ghostfolio/common/interfaces'; import { permissions } from '@ghostfolio/common/permissions'; diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index f0a451975..9cfeda3bd 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -51,13 +51,13 @@ export class CurrentRateService { const values: GetValueObject[] = []; if (includesToday) { - const quotesBySymbol = await this.dataProviderService.getQuotes({ + const quotes = await this.dataProviderService.getQuotes({ items: dataGatheringItems, user: this.request?.user }); for (const { dataSource, symbol } of dataGatheringItems) { - const quote = quotesBySymbol[symbol]; + const quote = quotes[getAssetProfileIdentifier({ dataSource, symbol })]; if (quote?.dataProviderInfo) { dataProviderInfos.push(quote.dataProviderInfo); diff --git a/apps/api/src/app/portfolio/portfolio.service.spec.ts b/apps/api/src/app/portfolio/portfolio.service.spec.ts index e0e7a8255..04142c92b 100644 --- a/apps/api/src/app/portfolio/portfolio.service.spec.ts +++ b/apps/api/src/app/portfolio/portfolio.service.spec.ts @@ -11,6 +11,7 @@ import { ImpersonationService } from '@ghostfolio/api/services/impersonation/imp import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { UNKNOWN_KEY } from '@ghostfolio/common/config'; import { parseDate } from '@ghostfolio/common/helper'; +import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { Account, DataSource } from '@prisma/client'; import { Big } from 'big.js'; @@ -198,7 +199,7 @@ describe('PortfolioService', () => { portfolioService as unknown as { getCashSymbolProfiles: ( aCashDetails: CashDetails - ) => { dataSource: DataSource; symbol: string }[]; + ) => AssetProfileIdentifier[]; } ).getCashSymbolProfiles(cashDetails); diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index c15353521..41697b346 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -46,6 +46,7 @@ import { import { AccountsResponse, Activity, + AssetProfileIdentifier, EnhancedSymbolProfile, Filter, HistoricalDataItem, @@ -204,21 +205,21 @@ export class PortfolioService { switch (type) { case ActivityType.DIVIDEND: dividendInBaseCurrency += - await this.exchangeRateDataService.toCurrencyAtDate( + (await this.exchangeRateDataService.toCurrencyAtDate( new Big(quantity).mul(unitPrice).toNumber(), currency ?? SymbolProfile.currency, userCurrency, date - ); + )) ?? 0; break; case ActivityType.INTEREST: interestInBaseCurrency += - await this.exchangeRateDataService.toCurrencyAtDate( + (await this.exchangeRateDataService.toCurrencyAtDate( unitPrice, currency ?? SymbolProfile.currency, userCurrency, date - ); + )) ?? 0; break; } @@ -776,11 +777,9 @@ export class PortfolioService { symbol, userId }: { - dataSource: DataSource; impersonationId: string; - symbol: string; userId: string; - }): Promise { + } & AssetProfileIdentifier): Promise { userId = await this.getUserId(impersonationId, userId); const user = await this.userService.user({ id: userId }); const userCurrency = this.getUserCurrency(user); @@ -890,10 +889,13 @@ export class PortfolioService { marketPrice ); - if (historicalData[symbol]) { + const historicalDataItems = + historicalData[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (historicalDataItems) { let j = -1; for (const [date, { marketPrice }] of Object.entries( - historicalData[symbol] + historicalDataItems )) { while ( j + 1 < transactionPoints.length && @@ -955,7 +957,6 @@ export class PortfolioService { marketPrice, marketPriceMax, marketPriceMin, - SymbolProfile, tags, assetProfile: { assetClass: SymbolProfile.assetClass, @@ -1381,12 +1382,10 @@ export class PortfolioService { tags, userId }: { - dataSource: DataSource; impersonationId: string; - symbol: string; tags: Tag[]; userId: string; - }) { + } & AssetProfileIdentifier) { userId = await this.getUserId(impersonationId, userId); await this.activitiesService.assignTags({ diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index fdbc7f84c..39d279de1 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -1,7 +1,10 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; -import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { + DATE_FORMAT, + getAssetProfileIdentifier +} from '@ghostfolio/common/helper'; import { DataProviderHistoricalResponse, HistoricalDataItem, @@ -24,15 +27,31 @@ export class SymbolService { public async get({ dataGatheringItem, - includeHistoricalData + includeHistoricalData, + useIntradayData = false }: { dataGatheringItem: DataGatheringItem; includeHistoricalData?: number; + useIntradayData?: boolean; }): Promise { - const quotes = await this.dataProviderService.getQuotes({ - items: [dataGatheringItem] - }); - const { currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {}; + let currency: string; + let marketPrice: number; + + if (useIntradayData) { + const latestMarketData = await this.marketDataService.getLatest({ + dataSource: dataGatheringItem.dataSource, + symbol: dataGatheringItem.symbol + }); + + marketPrice = latestMarketData?.marketPrice; + } else { + const quotes = await this.dataProviderService.getQuotes({ + items: [dataGatheringItem] + }); + + ({ currency, marketPrice } = + quotes[getAssetProfileIdentifier(dataGatheringItem)] ?? {}); + } if (dataGatheringItem.dataSource && marketPrice >= 0) { let historicalData: HistoricalDataItem[] = []; @@ -75,12 +94,17 @@ export class SymbolService { date = new Date(), symbol }: DataGatheringItem): Promise { + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + let historicalData: { - [symbol: string]: { + [assetProfileIdentifier: string]: { [date: string]: DataProviderHistoricalResponse; }; } = { - [symbol]: {} + [assetProfileIdentifier]: {} }; try { @@ -93,7 +117,8 @@ export class SymbolService { return { marketPrice: - historicalData?.[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice + historicalData?.[assetProfileIdentifier]?.[format(date, DATE_FORMAT)] + ?.marketPrice }; } diff --git a/apps/api/src/events/asset-profile-changed.listener.ts b/apps/api/src/events/asset-profile-changed.listener.ts index e2aea382e..15c19ff0a 100644 --- a/apps/api/src/events/asset-profile-changed.listener.ts +++ b/apps/api/src/events/asset-profile-changed.listener.ts @@ -5,10 +5,10 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate- import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { DEFAULT_CURRENCY } from '@ghostfolio/common/config'; import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; +import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; -import { DataSource } from '@prisma/client'; import ms from 'ms'; import { AssetProfileChangedEvent } from './asset-profile-changed.event'; @@ -64,11 +64,7 @@ export class AssetProfileChangedListener { currency, dataSource, symbol - }: { - currency: string; - dataSource: DataSource; - symbol: string; - }) { + }: { currency: string } & AssetProfileIdentifier) { this.logger.log(`Asset profile of ${symbol} (${dataSource}) has changed`); if ( diff --git a/apps/api/src/helper/country.helper.ts b/apps/api/src/helper/country.helper.ts index 9d14d8778..1d9f8f99a 100644 --- a/apps/api/src/helper/country.helper.ts +++ b/apps/api/src/helper/country.helper.ts @@ -7,8 +7,12 @@ export function getCountryCodeByName({ aliases?: Record; name: string; }): string { + if (aliases[name]) { + return aliases[name]; + } + for (const [code, country] of Object.entries(countries)) { - if (country.name === name || country.name === aliases[name]) { + if (country.name === name) { return code; } } diff --git a/apps/api/src/helper/data-source.helper.ts b/apps/api/src/helper/data-source.helper.ts new file mode 100644 index 000000000..f3ed75229 --- /dev/null +++ b/apps/api/src/helper/data-source.helper.ts @@ -0,0 +1,67 @@ +import { DataSource } from '@prisma/client'; +import { createHash } from 'node:crypto'; + +const encodedDataSourceByDataSource = new Map( + Object.values(DataSource).map((dataSource) => { + return [dataSource, hashDataSource(dataSource)]; + }) +); + +const dataSourceByEncodedDataSource = new Map( + [...encodedDataSourceByDataSource].map(([dataSource, encodedDataSource]) => { + return [encodedDataSource, dataSource]; + }) +); + +/** + * @deprecated Backward compatibility to support importing data that was + * exported using the previous data source encoding + */ +const dataSourceByDeprecatedEncodedDataSource = new Map( + Object.values(DataSource).map((dataSource) => { + return [deprecatedHashDataSource(dataSource), dataSource]; + }) +); + +/** + * @deprecated Backward compatibility (see above) + */ +function deprecatedHashDataSource(dataSource: DataSource) { + return Buffer.from(dataSource, 'utf-8').toString('hex'); +} + +function hashDataSource(dataSource: DataSource) { + return createHash('sha256').update(dataSource).digest('hex').slice(0, 8); +} + +export function decodeDataSource(encodedDataSource: string) { + if (!encodedDataSource) { + return undefined; + } + + return ( + dataSourceByEncodedDataSource.get(encodedDataSource) ?? + dataSourceByDeprecatedEncodedDataSource.get(encodedDataSource) ?? + encodedDataSource + ); +} + +export function encodeDataSource(dataSource: DataSource) { + if (!dataSource) { + return undefined; + } + + return encodedDataSourceByDataSource.get(dataSource); +} + +export function getMaskedGhostfolioDataSource({ + dataSource, + ghostfolioDataSources +}: { + dataSource: DataSource; + ghostfolioDataSources: string[]; +}) { + return ghostfolioDataSources.includes(dataSource) + ? DataSource.GHOSTFOLIO + : dataSource; +} diff --git a/apps/api/src/interceptors/performance-logging/performance-logging.service.ts b/apps/api/src/interceptors/performance-logging/performance-logging.service.ts index a07783cd9..b2ea03c37 100644 --- a/apps/api/src/interceptors/performance-logging/performance-logging.service.ts +++ b/apps/api/src/interceptors/performance-logging/performance-logging.service.ts @@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; @Injectable() export class PerformanceLoggingService { - private readonly logger = new Logger(PerformanceLoggingService.name); + private readonly logger = new Logger(); public logPerformance({ className, diff --git a/apps/api/src/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor.ts b/apps/api/src/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor.ts index 17c5ebe57..4ab0794ec 100644 --- a/apps/api/src/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor.ts +++ b/apps/api/src/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor.ts @@ -1,5 +1,5 @@ +import { decodeDataSource } from '@ghostfolio/api/helper/data-source.helper'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; -import { decodeDataSource } from '@ghostfolio/common/helper'; import { CallHandler, diff --git a/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts b/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts index fb8ff5dc5..56d260d4d 100644 --- a/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts +++ b/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts @@ -1,6 +1,10 @@ +import { + encodeDataSource, + getMaskedGhostfolioDataSource +} from '@ghostfolio/api/helper/data-source.helper'; import { redactPaths } from '@ghostfolio/api/helper/object.helper'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; -import { encodeDataSource } from '@ghostfolio/common/helper'; +import { hasRole } from '@ghostfolio/common/permissions'; import { CallHandler, @@ -44,20 +48,32 @@ export class TransformDataSourceInResponseInterceptor< next: CallHandler ): Observable { const isExportMode = context.getClass().name === 'ExportController'; + const { user } = context.switchToHttp().getRequest(); return next.handle().pipe( map((data: any) => { if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) { - const valueMap = this.encodedDataSourceMap; + const valueMap = hasRole(user, 'ADMIN') + ? {} + : { ...this.encodedDataSourceMap }; if (isExportMode) { - for (const dataSource of this.configurationService.get( + const ghostfolioDataSources = this.configurationService.get( 'DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER' - )) { - valueMap[dataSource] = 'GHOSTFOLIO'; + ) as DataSource[]; + + for (const dataSource of ghostfolioDataSources) { + valueMap[dataSource] = getMaskedGhostfolioDataSource({ + dataSource, + ghostfolioDataSources + }); } } + if (Object.keys(valueMap).length === 0) { + return data; + } + data = redactPaths({ valueMap, object: data, diff --git a/apps/api/src/services/benchmark/benchmark.service.ts b/apps/api/src/services/benchmark/benchmark.service.ts index 022a0e928..56a629163 100644 --- a/apps/api/src/services/benchmark/benchmark.service.ts +++ b/apps/api/src/services/benchmark/benchmark.service.ts @@ -8,7 +8,10 @@ import { CACHE_TTL_INFINITE, PROPERTY_BENCHMARKS } from '@ghostfolio/common/config'; -import { calculateBenchmarkTrend } from '@ghostfolio/common/helper'; +import { + calculateBenchmarkTrend, + getAssetProfileIdentifier +} from '@ghostfolio/common/helper'; import { AssetProfileIdentifier, Benchmark, @@ -266,8 +269,9 @@ export class BenchmarkService { let storeInCache = true; const benchmarks = allTimeHighs.map((allTimeHigh, index) => { + const { dataSource, symbol } = benchmarkAssetProfiles[index]; const { marketPrice } = - quotes[benchmarkAssetProfiles[index].symbol] ?? {}; + quotes[getAssetProfileIdentifier({ dataSource, symbol })] ?? {}; let performancePercentFromAllTimeHigh = 0; diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts index 5f9d1055d..703a90c3a 100644 --- a/apps/api/src/services/configuration/configuration.service.ts +++ b/apps/api/src/services/configuration/configuration.service.ts @@ -6,6 +6,7 @@ import { DEFAULT_PORT, DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY, DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY, + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY, DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY, DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT } from '@ghostfolio/common/config'; @@ -43,6 +44,7 @@ export class ConfigurationService { ENABLE_FEATURE_AUTH_GOOGLE: bool({ default: false }), ENABLE_FEATURE_AUTH_OIDC: bool({ default: false }), ENABLE_FEATURE_AUTH_TOKEN: bool({ default: true }), + ENABLE_FEATURE_CRON: bool({ default: true }), ENABLE_FEATURE_FEAR_AND_GREED_INDEX: bool({ default: false }), ENABLE_FEATURE_GATHER_NEW_EXCHANGE_RATES: bool({ default: true }), ENABLE_FEATURE_READ_ONLY_MODE: bool({ default: false }), @@ -88,6 +90,9 @@ export class ConfigurationService { PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: num({ default: DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY }), + PROCESSOR_GATHER_STATISTICS_CONCURRENCY: num({ + default: DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY + }), PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: num({ default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY }), diff --git a/apps/api/src/services/cron/cron.module.ts b/apps/api/src/services/cron/cron.module.ts index bcc9d3360..bc14990c1 100644 --- a/apps/api/src/services/cron/cron.module.ts +++ b/apps/api/src/services/cron/cron.module.ts @@ -1,12 +1,17 @@ import { UserModule } from '@ghostfolio/api/app/user/user.module'; +import { UserService } from '@ghostfolio/api/app/user/user.service'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; -import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; +import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; +import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { StatisticsGatheringQueueModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module'; +import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service'; import { TwitterBotModule } from '@ghostfolio/api/services/twitter-bot/twitter-bot.module'; +import { TwitterBotService } from '@ghostfolio/api/services/twitter-bot/twitter-bot.service'; -import { Module } from '@nestjs/common'; +import { Logger, Module } from '@nestjs/common'; import { CronService } from './cron.service'; @@ -14,12 +19,46 @@ import { CronService } from './cron.service'; imports: [ ConfigurationModule, DataGatheringQueueModule, - ExchangeRateDataModule, PropertyModule, StatisticsGatheringQueueModule, TwitterBotModule, UserModule ], - providers: [CronService] + providers: [ + { + inject: [ + ConfigurationService, + DataGatheringService, + PropertyService, + StatisticsGatheringService, + TwitterBotService, + UserService + ], + provide: CronService, + useFactory: ( + configurationService: ConfigurationService, + dataGatheringService: DataGatheringService, + propertyService: PropertyService, + statisticsGatheringService: StatisticsGatheringService, + twitterBotService: TwitterBotService, + userService: UserService + ) => { + if (!configurationService.get('ENABLE_FEATURE_CRON')) { + Logger.log('Scheduled cron jobs are disabled', 'CronService'); + + return null; + } + + return new CronService( + configurationService, + dataGatheringService, + propertyService, + statisticsGatheringService, + twitterBotService, + userService + ); + } + } + ] }) export class CronModule {} diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts index e680f0063..b3e60ae59 100644 --- a/apps/api/src/services/cron/cron.service.ts +++ b/apps/api/src/services/cron/cron.service.ts @@ -1,6 +1,5 @@ import { UserService } from '@ghostfolio/api/app/user/user.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; -import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service'; @@ -24,7 +23,6 @@ export class CronService { public constructor( private readonly configurationService: ConfigurationService, private readonly dataGatheringService: DataGatheringService, - private readonly exchangeRateDataService: ExchangeRateDataService, private readonly propertyService: PropertyService, private readonly statisticsGatheringService: StatisticsGatheringService, private readonly twitterBotService: TwitterBotService, @@ -41,15 +39,11 @@ export class CronService { @Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE) public async runEveryHourAtRandomMinute() { if (await this.isDataGatheringEnabled()) { - await this.dataGatheringService.gather7Days(); + await this.dataGatheringService.gatherHourlyMarketData(); + await this.dataGatheringService.gatherRecentMarketData(); } } - @Cron(CronExpression.EVERY_12_HOURS) - public async runEveryTwelveHours() { - await this.exchangeRateDataService.loadCurrencies(); - } - @Cron(CronExpression.EVERY_DAY_AT_5PM) public async runEveryDayAtFivePm() { if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) { diff --git a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts index c8291a901..9f69dc39b 100644 --- a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts @@ -15,8 +15,11 @@ import { SymbolProfile } from '@prisma/client'; export class TrackinsightDataEnhancerService implements DataEnhancerInterface { private static baseUrl = 'https://www.trackinsight.com/data-api'; private static countriesMapping = { - 'Russian Federation': 'Russia', - USA: 'United States' + 'Republic of Korea': 'KR', + 'Russian Federation': 'RU', + Turkey: 'TR', + USA: 'US', + 'Virgin Islands, British': 'VG' }; private static holdingsWeightTreshold = 0.85; private static sectorsMapping: Record = { diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 5b54afb0b..c6e9c83c1 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -1,5 +1,6 @@ import { ImportDataDto } from '@ghostfolio/api/app/import/import-data.dto'; import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service'; +import { getMaskedGhostfolioDataSource } from '@ghostfolio/api/helper/data-source.helper'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; @@ -77,19 +78,20 @@ export class DataProviderService implements OnModuleInit { useCache: false }); - if (quotes[symbol]?.marketPrice > 0) { + if ( + quotes[getAssetProfileIdentifier({ dataSource, symbol })]?.marketPrice > 0 + ) { return true; } return false; } - // TODO: Change symbol in response to assetProfileIdentifier public async getAssetProfiles(items: AssetProfileIdentifier[]): Promise<{ - [symbol: string]: Partial; + [assetProfileIdentifier: string]: Partial; }> { const response: { - [symbol: string]: Partial; + [assetProfileIdentifier: string]: Partial; } = {}; const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => { @@ -115,7 +117,12 @@ export class DataProviderService implements OnModuleInit { promises.push( promise.then((assetProfile) => { if (isCurrency(assetProfile?.currency)) { - response[symbol] = assetProfile; + response[ + getAssetProfileIdentifier({ + symbol, + dataSource: DataSource[dataSource] + }) + ] = { ...assetProfile, symbol }; } }) ); @@ -215,6 +222,9 @@ export class DataProviderService implements OnModuleInit { } = {}; const dataSources = await this.getDataSources(); + const ghostfolioDataSources = this.configurationService.get( + 'DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER' + ); for (const [ index, @@ -222,6 +232,10 @@ export class DataProviderService implements OnModuleInit { ] of activitiesDto.entries()) { const activityPath = maxActivitiesToImport === 1 ? 'activity' : `activities.${index}`; + const maskedDataSource = getMaskedGhostfolioDataSource({ + dataSource, + ghostfolioDataSources + }); if (!dataSources.includes(dataSource)) { throw new Error( @@ -237,7 +251,7 @@ export class DataProviderService implements OnModuleInit { if (dataProvider.getDataProviderInfo().isPremium) { throw new Error( - `${activityPath}.dataSource ("${dataSource}") is not valid` + `${activityPath}.dataSource ("${maskedDataSource}") requires Ghostfolio Premium` ); } } @@ -281,7 +295,7 @@ export class DataProviderService implements OnModuleInit { symbol } ]) - )?.[symbol]; + )?.[assetProfileIdentifier]; } catch {} if (!assetProfile?.name) { @@ -300,7 +314,7 @@ export class DataProviderService implements OnModuleInit { if (!assetProfile?.name) { throw new Error( - `activities.${index}.symbol ("${symbol}") is not valid for the specified data source ("${dataSource}")` + `activities.${index}.symbol ("${symbol}") is not valid for the specified data source ("${maskedDataSource}")` ); } @@ -318,12 +332,10 @@ export class DataProviderService implements OnModuleInit { symbol, to }: { - dataSource: DataSource; from: Date; granularity: Granularity; - symbol: string; to: Date; - }) { + } & AssetProfileIdentifier) { return this.getDataProvider(DataSource[dataSource]).getDividends({ from, granularity, @@ -333,17 +345,20 @@ export class DataProviderService implements OnModuleInit { }); } - // TODO: Change symbol in response to assetProfileIdentifier public async getHistorical( aItems: AssetProfileIdentifier[], aGranularity: Granularity = 'month', from: Date, to: Date ): Promise<{ - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; }> { let response: { - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; } = {}; if (isEmpty(aItems) || !isValid(from) || !isValid(to)) { @@ -383,13 +398,20 @@ export class DataProviderService implements OnModuleInit { ORDER BY date;`; response = marketDataByGranularity.reduce((r, marketData) => { - const { date, marketPrice, symbol } = marketData; + const { dataSource, date, marketPrice, symbol } = marketData; + + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); - if (!r[symbol]) { - r[symbol] = {}; + if (!r[assetProfileIdentifier]) { + r[assetProfileIdentifier] = {}; } - r[symbol][format(new Date(date), DATE_FORMAT)] = { marketPrice }; + r[assetProfileIdentifier][format(new Date(date), DATE_FORMAT)] = { + marketPrice + }; return r; }, {}); @@ -400,7 +422,6 @@ export class DataProviderService implements OnModuleInit { } } - // TODO: Change symbol in response to assetProfileIdentifier public async getHistoricalRaw({ assetProfileIdentifiers, from, @@ -410,7 +431,9 @@ export class DataProviderService implements OnModuleInit { from: Date; to: Date; }): Promise<{ - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; }> { for (const { currency, rootCurrency } of DERIVED_CURRENCIES) { if ( @@ -443,11 +466,14 @@ export class DataProviderService implements OnModuleInit { ); const result: { - [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + [assetProfileIdentifier: string]: { + [date: string]: DataProviderHistoricalResponse; + }; } = {}; const promises: Promise<{ data: { [date: string]: DataProviderHistoricalResponse }; + dataSource: DataSource; symbol: string; }>[] = []; for (const { dataSource, symbol } of assetProfileIdentifiers) { @@ -465,6 +491,7 @@ export class DataProviderService implements OnModuleInit { promises.push( Promise.resolve({ data, + dataSource, symbol }) ); @@ -478,7 +505,7 @@ export class DataProviderService implements OnModuleInit { requestTimeout: ms('30 seconds') }) .then((data) => { - return { symbol, data: data?.[symbol] }; + return { dataSource, symbol, data: data?.[symbol] }; }) ); } @@ -488,22 +515,26 @@ export class DataProviderService implements OnModuleInit { try { const allData = await Promise.all(promises); - for (const { data, symbol } of allData) { + for (const { data, dataSource, symbol } of allData) { const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => { return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol; }); if (currency) { // Add derived currency - result[`${DEFAULT_CURRENCY}${currency.currency}`] = - this.transformHistoricalData({ - allData, - currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`, - factor: currency.factor - }); + result[ + getAssetProfileIdentifier({ + dataSource, + symbol: `${DEFAULT_CURRENCY}${currency.currency}` + }) + ] = this.transformHistoricalData({ + allData, + currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`, + factor: currency.factor + }); } - result[symbol] = data; + result[getAssetProfileIdentifier({ dataSource, symbol })] = data; } } catch (error) { this.logger.error(error); @@ -514,7 +545,6 @@ export class DataProviderService implements OnModuleInit { return result; } - // TODO: Change symbol in response to assetProfileIdentifier public async getQuotes({ items, requestTimeout, @@ -526,10 +556,12 @@ export class DataProviderService implements OnModuleInit { useCache?: boolean; user?: UserWithSettings; }): Promise<{ - [symbol: string]: DataProviderResponse; + [assetProfileIdentifier: string]: DataProviderResponse; }> { const response: { - [symbol: string]: DataProviderResponse; + [assetProfileIdentifier: string]: DataProviderResponse & { + symbol: string; + }; } = {}; const startTimeTotal = performance.now(); @@ -538,11 +570,17 @@ export class DataProviderService implements OnModuleInit { return symbol === `${DEFAULT_CURRENCY}USX`; }) ) { - response[`${DEFAULT_CURRENCY}USX`] = { + response[ + getAssetProfileIdentifier({ + dataSource: this.getDataSourceForExchangeRates(), + symbol: `${DEFAULT_CURRENCY}USX` + }) + ] = { currency: 'USX', dataSource: this.getDataSourceForExchangeRates(), marketPrice: 100, - marketState: 'open' + marketState: 'open', + symbol: `${DEFAULT_CURRENCY}USX` }; } @@ -557,8 +595,13 @@ export class DataProviderService implements OnModuleInit { if (quoteString) { try { - const cachedDataProviderResponse = JSON.parse(quoteString); - response[symbol] = cachedDataProviderResponse; + const cachedDataProviderResponse = JSON.parse( + quoteString + ) as DataProviderResponse; + response[getAssetProfileIdentifier({ dataSource, symbol })] = { + ...cachedDataProviderResponse, + symbol + }; continue; } catch {} } @@ -627,7 +670,11 @@ export class DataProviderService implements OnModuleInit { ); const promise = Promise.resolve( - dataProvider.getQuotes({ requestTimeout, symbols: symbolsChunk }) + dataProvider.getQuotes({ + requestTimeout, + useCache, + symbols: symbolsChunk + }) ); promises.push( @@ -646,14 +693,19 @@ export class DataProviderService implements OnModuleInit { continue; } - response[symbol] = dataProviderResponse; + response[ + getAssetProfileIdentifier({ + symbol, + dataSource: DataSource[dataSource] + }) + ] = { ...dataProviderResponse, symbol }; this.redisCacheService.set( this.redisCacheService.getQuoteKey({ symbol, dataSource: DataSource[dataSource] }), - JSON.stringify(response[symbol]), + JSON.stringify(dataProviderResponse), this.configurationService.get('CACHE_QUOTES_TTL') ); @@ -663,7 +715,7 @@ export class DataProviderService implements OnModuleInit { rootCurrency } of DERIVED_CURRENCIES) { if (symbol === `${DEFAULT_CURRENCY}${rootCurrency}`) { - response[`${DEFAULT_CURRENCY}${currency}`] = { + const derivedDataProviderResponse: DataProviderResponse = { ...dataProviderResponse, currency, marketPrice: new Big( @@ -674,12 +726,22 @@ export class DataProviderService implements OnModuleInit { marketState: 'open' }; + response[ + getAssetProfileIdentifier({ + dataSource: DataSource[dataSource], + symbol: `${DEFAULT_CURRENCY}${currency}` + }) + ] = { + ...derivedDataProviderResponse, + symbol: `${DEFAULT_CURRENCY}${currency}` + }; + this.redisCacheService.set( this.redisCacheService.getQuoteKey({ dataSource: DataSource[dataSource], symbol: `${DEFAULT_CURRENCY}${currency}` }), - JSON.stringify(response[`${DEFAULT_CURRENCY}${currency}`]), + JSON.stringify(derivedDataProviderResponse), this.configurationService.get('CACHE_QUOTES_TTL') ); } @@ -697,21 +759,21 @@ export class DataProviderService implements OnModuleInit { try { await this.marketDataService.updateMany({ - data: Object.keys(response) - .filter((symbol) => { + data: Object.values(response) + .filter(({ marketPrice, marketState }) => { return ( - isNumber(response[symbol].marketPrice) && - response[symbol].marketPrice > 0 && - response[symbol].marketState === 'open' + isNumber(marketPrice) && + marketPrice > 0 && + marketState === 'open' ); }) - .map((symbol) => { + .map((dataProviderResponse) => { return { - symbol, - dataSource: response[symbol].dataSource, + dataSource: dataProviderResponse.dataSource, date: getStartOfUtcDate(new Date()), - marketPrice: response[symbol].marketPrice, - state: 'INTRADAY' + marketPrice: dataProviderResponse.marketPrice, + state: 'INTRADAY', + symbol: dataProviderResponse.symbol }; }) }); diff --git a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts index ca48bb247..ec4aa39c0 100644 --- a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts +++ b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts @@ -54,9 +54,9 @@ export class FinancialModelingPrepService implements DataProviderInterface, OnModuleInit { private static countriesMapping = { - 'Korea (the Republic of)': 'South Korea', - 'Russian Federation': 'Russia', - 'Taiwan (Province of China)': 'Taiwan' + 'Korea (the Republic of)': 'KR', + 'Russian Federation': 'RU', + 'Taiwan (Province of China)': 'TW' }; private readonly logger = new Logger(FinancialModelingPrepService.name); diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts index a55c9f328..5002fa87e 100644 --- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts @@ -75,6 +75,7 @@ export interface GetHistoricalParams { export interface GetQuotesParams { requestTimeout?: number; symbols: string[]; + useCache?: boolean; } export interface GetSearchParams { diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 87e116dda..66571c239 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -136,7 +136,8 @@ export class ManualService implements DataProviderInterface { } public async getQuotes({ - symbols + symbols, + useCache = true }: GetQuotesParams): Promise<{ [symbol: string]: DataProviderResponse }> { const response: { [symbol: string]: DataProviderResponse } = {}; @@ -164,32 +165,32 @@ export class ManualService implements DataProviderInterface { } }); - const symbolProfilesWithScraperConfigurationAndInstantMode = - symbolProfiles.filter(({ scraperConfiguration }) => { + const symbolProfilesToScrape = symbolProfiles.filter( + ({ scraperConfiguration }) => { return ( - scraperConfiguration?.mode === 'instant' && + (scraperConfiguration?.mode === 'instant' || !useCache) && scraperConfiguration?.selector && scraperConfiguration?.url ); - }); - - const scraperResultPromises = - symbolProfilesWithScraperConfigurationAndInstantMode.map( - async ({ scraperConfiguration, symbol }) => { - try { - const marketPrice = await this.scrape({ - scraperConfiguration, - symbol - }); - return { marketPrice, symbol }; - } catch (error) { - this.logger.error( - `Could not get quote for ${symbol} (${this.getName()}): [${error.name}] ${error.message}` - ); - return { symbol, marketPrice: undefined }; - } + } + ); + + const scraperResultPromises = symbolProfilesToScrape.map( + async ({ scraperConfiguration, symbol }) => { + try { + const marketPrice = await this.scrape({ + scraperConfiguration, + symbol + }); + return { marketPrice, symbol }; + } catch (error) { + this.logger.error( + `Could not get quote for ${symbol} (${this.getName()}): [${error.name}] ${error.message}` + ); + return { symbol, marketPrice: undefined }; } - ); + } + ); // Wait for all scraping requests to complete concurrently const scraperResults = await Promise.all(scraperResultPromises); diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts index 9941ae9eb..d8ba1ec67 100644 --- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts @@ -8,10 +8,7 @@ import { GetSearchParams } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service'; -import { - ghostfolioFearAndGreedIndexSymbol, - ghostfolioFearAndGreedIndexSymbolStocks -} from '@ghostfolio/common/config'; +import { ghostfolioFearAndGreedIndexSymbolStocks } from '@ghostfolio/common/config'; import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper'; import { DataProviderHistoricalResponse, @@ -64,12 +61,7 @@ export class RapidApiService implements DataProviderInterface { [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; }> { try { - if ( - [ - ghostfolioFearAndGreedIndexSymbol, - ghostfolioFearAndGreedIndexSymbolStocks - ].includes(symbol) - ) { + if (symbol === ghostfolioFearAndGreedIndexSymbolStocks) { const fgi = await this.getFearAndGreedIndex(); return { @@ -106,12 +98,7 @@ export class RapidApiService implements DataProviderInterface { try { const symbol = symbols[0]; - if ( - [ - ghostfolioFearAndGreedIndexSymbol, - ghostfolioFearAndGreedIndexSymbolStocks - ].includes(symbol) - ) { + if (symbol === ghostfolioFearAndGreedIndexSymbolStocks) { const fgi = await this.getFearAndGreedIndex(); return { diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts index 708bfa591..3b48ce292 100644 --- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts @@ -11,9 +11,11 @@ import { } from '@ghostfolio/common/config'; import { DATE_FORMAT, + getAssetProfileIdentifier, getYesterday, resetHours } from '@ghostfolio/common/helper'; +import { DataProviderHistoricalResponse } from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { @@ -162,7 +164,7 @@ export class ExchangeRateDataService { } public async loadCurrencies() { - const result = await this.dataProviderService.getHistorical( + const historicalData = await this.dataProviderService.getHistorical( this.currencyPairs, 'day', getYesterday(), @@ -176,11 +178,26 @@ export class ExchangeRateDataService { requestTimeout: ms('30 seconds') }); - for (const symbol of Object.keys(quotes)) { - if (isNumber(quotes[symbol].marketPrice)) { + const result: { + [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + } = {}; + + for (const { dataSource, symbol } of this.currencyPairs) { + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + + if (historicalData[assetProfileIdentifier]) { + result[symbol] = historicalData[assetProfileIdentifier]; + } + + const quote = quotes[assetProfileIdentifier]; + + if (isNumber(quote?.marketPrice)) { result[symbol] = { [format(getYesterday(), DATE_FORMAT)]: { - marketPrice: quotes[symbol].marketPrice + marketPrice: quote.marketPrice } }; } diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index 57c58898e..11bdeabf6 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -19,6 +19,7 @@ export interface Environment extends CleanedEnvAccessors { ENABLE_FEATURE_AUTH_GOOGLE: boolean; ENABLE_FEATURE_AUTH_OIDC: boolean; ENABLE_FEATURE_AUTH_TOKEN: boolean; + ENABLE_FEATURE_CRON: boolean; ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean; ENABLE_FEATURE_GATHER_NEW_EXCHANGE_RATES: boolean; ENABLE_FEATURE_READ_ONLY_MODE: boolean; @@ -44,6 +45,7 @@ export interface Environment extends CleanedEnvAccessors { PORT: number; PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY: number; PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: number; + PROCESSOR_GATHER_STATISTICS_CONCURRENCY: number; PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: number; PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: number; REDIS_DB: number; diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 87b08e1bd..086434724 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -1,6 +1,7 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.interface'; import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; +import { DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_TIMEOUT } from '@ghostfolio/common/config'; import { UpdateMarketDataDto } from '@ghostfolio/common/dtos'; import { resetHours } from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; @@ -40,6 +41,19 @@ export class MarketDataService { }); } + public async getLatest({ + dataSource, + symbol + }: AssetProfileIdentifier): Promise { + return this.prismaService.marketData.findFirst({ + orderBy: [{ date: 'desc' }], + where: { + dataSource, + symbol + } + }); + } + public async getMax({ dataSource, symbol }: AssetProfileIdentifier) { return this.prismaService.marketData.findFirst({ select: { @@ -142,49 +156,52 @@ export class MarketDataService { dataSource, symbol }: AssetProfileIdentifier & { data: Prisma.MarketDataUpdateInput[] }) { - await this.prismaService.$transaction(async (prisma) => { - if (data.length > 0) { - let minTime = Infinity; - let maxTime = -Infinity; + await this.prismaService.$transaction( + async (prisma) => { + if (data.length > 0) { + let minTime = Infinity; + let maxTime = -Infinity; - for (const { date } of data) { - const time = (date as Date).getTime(); + for (const { date } of data) { + const time = (date as Date).getTime(); - if (time < minTime) { - minTime = time; - } + if (time < minTime) { + minTime = time; + } - if (time > maxTime) { - maxTime = time; + if (time > maxTime) { + maxTime = time; + } } - } - const minDate = new Date(minTime); - const maxDate = new Date(maxTime); + const minDate = new Date(minTime); + const maxDate = new Date(maxTime); - await prisma.marketData.deleteMany({ - where: { - dataSource, - symbol, - date: { - gte: minDate, - lte: maxDate + await prisma.marketData.deleteMany({ + where: { + dataSource, + symbol, + date: { + gte: minDate, + lte: maxDate + } } - } - }); + }); - await prisma.marketData.createMany({ - data: data.map(({ date, marketPrice, state }) => ({ - dataSource, - symbol, - date: date as Date, - marketPrice: marketPrice as number, - state: state as MarketDataState - })), - skipDuplicates: true - }); - } - }); + await prisma.marketData.createMany({ + data: data.map(({ date, marketPrice, state }) => ({ + dataSource, + symbol, + date: date as Date, + marketPrice: marketPrice as number, + state: state as MarketDataState + })), + skipDuplicates: true + }); + } + }, + { timeout: DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_TIMEOUT } + ); } public async updateAssetProfileIdentifier( diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.module.ts b/apps/api/src/services/queues/data-gathering/data-gathering.module.ts index 5ac6c40c0..d66411797 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.module.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.module.ts @@ -28,7 +28,8 @@ import { DataGatheringProcessor } from './data-gathering.processor'; }), BullModule.registerQueue({ limiter: { - duration: ms('4 seconds'), + duration: ms('3 seconds'), + groupKey: 'dataSource', max: 1 }, name: DATA_GATHERING_QUEUE diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts index ee5cb838a..8b7e3489f 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts @@ -10,7 +10,11 @@ import { GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME } from '@ghostfolio/common/config'; -import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; +import { + DATE_FORMAT, + getAssetProfileIdentifier, + getStartOfUtcDate +} from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { Process, Processor } from '@nestjs/bull'; @@ -114,6 +118,11 @@ export class DataGatheringProcessor { to: new Date() }); + const assetProfileIdentifier = getAssetProfileIdentifier({ + dataSource, + symbol + }); + const data: Prisma.MarketDataUpdateInput[] = []; let lastMarketPrice: number; @@ -131,12 +140,14 @@ export class DataGatheringProcessor { ) ) { if ( - historicalData[symbol]?.[format(currentDate, DATE_FORMAT)] - ?.marketPrice + historicalData[assetProfileIdentifier]?.[ + format(currentDate, DATE_FORMAT) + ]?.marketPrice ) { lastMarketPrice = - historicalData[symbol]?.[format(currentDate, DATE_FORMAT)] - ?.marketPrice; + historicalData[assetProfileIdentifier]?.[ + format(currentDate, DATE_FORMAT) + ]?.marketPrice; } if (lastMarketPrice) { diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index b5b701fe4..1a2fa720e 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -2,6 +2,7 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; +import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; @@ -17,6 +18,7 @@ import { import { DATE_FORMAT, getAssetProfileIdentifier, + getStartOfUtcDate, resetHours } from '@ghostfolio/common/helper'; import { @@ -26,7 +28,7 @@ import { import { InjectQueue } from '@nestjs/bull'; import { Inject, Injectable, Logger } from '@nestjs/common'; -import { DataSource } from '@prisma/client'; +import { Prisma } from '@prisma/client'; import { JobOptions, Queue } from 'bull'; import { format, min, subDays, subMilliseconds, subYears } from 'date-fns'; import { isEmpty } from 'lodash'; @@ -43,6 +45,7 @@ export class DataGatheringService { private readonly dataGatheringQueue: Queue, private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, + private readonly marketDataService: MarketDataService, private readonly prismaService: PrismaService, private readonly propertyService: PropertyService, private readonly symbolProfileService: SymbolProfileService @@ -66,93 +69,6 @@ export class DataGatheringService { return this.dataGatheringQueue.addBulk(jobs); } - public async gather7Days() { - await this.gatherSymbols({ - dataGatheringItems: await this.getCurrencies7D(), - priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH - }); - - await this.gatherSymbols({ - dataGatheringItems: await this.getSymbols7D({ - withUserSubscription: true - }), - priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM - }); - - await this.gatherSymbols({ - dataGatheringItems: await this.getSymbols7D({ - withUserSubscription: false - }), - priority: DATA_GATHERING_QUEUE_PRIORITY_LOW - }); - } - - public async gatherMax() { - const dataGatheringItems = await this.getSymbolsMax(); - await this.gatherSymbols({ - dataGatheringItems, - priority: DATA_GATHERING_QUEUE_PRIORITY_LOW - }); - } - - public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) { - const dataGatheringItems = (await this.getSymbolsMax()) - .filter((dataGatheringItem) => { - return ( - dataGatheringItem.dataSource === dataSource && - dataGatheringItem.symbol === symbol - ); - }) - .map((item) => ({ - ...item, - date: date ?? item.date - })); - - await this.gatherSymbols({ - dataGatheringItems, - force: true, - priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH - }); - } - - public async gatherSymbolForDate({ - dataSource, - date, - symbol - }: { - dataSource: DataSource; - date: Date; - symbol: string; - }) { - try { - const historicalData = await this.dataProviderService.getHistoricalRaw({ - assetProfileIdentifiers: [{ dataSource, symbol }], - from: date, - to: date - }); - - const marketPrice = - historicalData[symbol][format(date, DATE_FORMAT)].marketPrice; - - if (marketPrice) { - return await this.prismaService.marketData.upsert({ - create: { - dataSource, - date, - marketPrice, - symbol - }, - update: { marketPrice }, - where: { dataSource_date_symbol: { dataSource, date, symbol } } - }); - } - } catch (error) { - this.logger.error(error); - } finally { - return undefined; - } - } - public async gatherAssetProfiles( aAssetProfileIdentifiers?: AssetProfileIdentifier[] ) { @@ -177,7 +93,9 @@ export class DataGatheringService { assetProfileIdentifiers ); - for (const [symbol, assetProfile] of Object.entries(assetProfiles)) { + for (const assetProfile of Object.values(assetProfiles)) { + const { symbol } = assetProfile; + const symbolProfile = symbolProfiles.find( ({ symbol: symbolProfileSymbol }) => { return symbolProfileSymbol === symbol; @@ -279,6 +197,137 @@ export class DataGatheringService { } } + public async gatherHourlyMarketData() { + try { + await this.exchangeRateDataService.loadCurrencies(); + } catch (error) { + this.logger.error('Could not gather exchange rates', error); + } + + const assetProfileIdentifiers = + await this.getHourlyAssetProfileIdentifiers(); + + if (assetProfileIdentifiers.length <= 0) { + return; + } + + const date = getStartOfUtcDate(new Date()); + + try { + const quotes = await this.dataProviderService.getQuotes({ + items: assetProfileIdentifiers, + useCache: false + }); + + const data: Prisma.MarketDataUpdateInput[] = []; + + for (const { dataSource, symbol } of assetProfileIdentifiers) { + const quote = quotes[getAssetProfileIdentifier({ dataSource, symbol })]; + + if (!quote?.marketPrice) { + continue; + } + + data.push({ + dataSource, + date, + symbol, + marketPrice: quote.marketPrice, + state: 'INTRADAY' + }); + } + + await this.marketDataService.updateMany({ data }); + } catch (error) { + this.logger.error('Could not gather hourly market data', error); + } + } + + public async gatherMax() { + const dataGatheringItems = await this.getSymbolsMax(); + await this.gatherSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW + }); + } + + public async gatherRecentMarketData() { + await this.gatherSymbols({ + dataGatheringItems: await this.getCurrencies7D(), + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); + + await this.gatherSymbols({ + dataGatheringItems: await this.getSymbols7D({ + withUserSubscription: true + }), + priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM + }); + + await this.gatherSymbols({ + dataGatheringItems: await this.getSymbols7D({ + withUserSubscription: false + }), + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW + }); + } + + public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) { + const dataGatheringItems = (await this.getSymbolsMax()) + .filter((dataGatheringItem) => { + return ( + dataGatheringItem.dataSource === dataSource && + dataGatheringItem.symbol === symbol + ); + }) + .map((item) => ({ + ...item, + date: date ?? item.date + })); + + await this.gatherSymbols({ + dataGatheringItems, + force: true, + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); + } + + public async gatherSymbolForDate({ + dataSource, + date, + symbol + }: { date: Date } & AssetProfileIdentifier) { + try { + const historicalData = await this.dataProviderService.getHistoricalRaw({ + assetProfileIdentifiers: [{ dataSource, symbol }], + from: date, + to: date + }); + + const marketPrice = + historicalData[getAssetProfileIdentifier({ dataSource, symbol })][ + format(date, DATE_FORMAT) + ].marketPrice; + + if (marketPrice) { + return await this.prismaService.marketData.upsert({ + create: { + dataSource, + date, + marketPrice, + symbol + }, + update: { marketPrice }, + where: { dataSource_date_symbol: { dataSource, date, symbol } } + }); + } + } catch (error) { + this.logger.error(error); + } finally { + return undefined; + } + } + public async gatherSymbols({ dataGatheringItems, force = false, @@ -389,6 +438,36 @@ export class DataGatheringService { return min([aStartDate, subYears(new Date(), 10)]); } + private async getHourlyAssetProfileIdentifiers(): Promise< + AssetProfileIdentifier[] + > { + const symbolProfiles = await this.prismaService.symbolProfile.findMany({ + orderBy: [{ symbol: 'asc' }, { dataSource: 'asc' }], + select: { + dataSource: true, + scraperConfiguration: true, + symbol: true + }, + where: { + dataGatheringFrequency: 'HOURLY', + isActive: true + } + }); + + return symbolProfiles + .filter(({ dataSource, scraperConfiguration }) => { + const manualDataSourceWithScraperConfiguration = + dataSource === 'MANUAL' && !isEmpty(scraperConfiguration); + + return ( + dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration + ); + }) + .map(({ dataSource, symbol }) => { + return { dataSource, symbol }; + }); + } + private async getSymbols7D({ withUserSubscription = false }: { @@ -469,14 +548,12 @@ export class DataGatheringService { } }) ) - .filter((symbolProfile) => { + .filter(({ dataSource, scraperConfiguration }) => { const manualDataSourceWithScraperConfiguration = - symbolProfile.dataSource === 'MANUAL' && - !isEmpty(symbolProfile.scraperConfiguration); + dataSource === 'MANUAL' && !isEmpty(scraperConfiguration); return ( - symbolProfile.dataSource !== 'MANUAL' || - manualDataSourceWithScraperConfiguration + dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration ); }) .map((symbolProfile) => { diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts index 7eefc101f..82f362d25 100644 --- a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts +++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts @@ -2,6 +2,7 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY, GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME, GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME, GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME, @@ -35,7 +36,14 @@ export class StatisticsGatheringProcessor { private readonly propertyService: PropertyService ) {} - @Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME) + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ?? + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(), + 10 + ), + name: GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME + }) public async gatherDockerHubPullsStatistics() { this.logger.log('Docker Hub pulls statistics gathering has been started'); @@ -49,7 +57,14 @@ export class StatisticsGatheringProcessor { this.logger.log('Docker Hub pulls statistics gathering has been completed'); } - @Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME) + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ?? + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(), + 10 + ), + name: GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME + }) public async gatherGitHubContributorsStatistics() { this.logger.log( 'GitHub contributors statistics gathering has been started' @@ -67,7 +82,14 @@ export class StatisticsGatheringProcessor { ); } - @Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME) + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ?? + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(), + 10 + ), + name: GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME + }) public async gatherGitHubStargazersStatistics() { this.logger.log('GitHub stargazers statistics gathering has been started'); @@ -83,7 +105,14 @@ export class StatisticsGatheringProcessor { ); } - @Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME) + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ?? + DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(), + 10 + ), + name: GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME + }) public async gatherUptimeStatistics() { const monitorId = await this.propertyService.getByKey( PROPERTY_BETTER_UPTIME_MONITOR_ID diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.ts index 2d5116274..5d0968c5c 100644 --- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts +++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts @@ -178,6 +178,7 @@ export class SymbolProfileService { comment, countries, currency, + dataGatheringFrequency, holdings, isActive, name, @@ -195,6 +196,7 @@ export class SymbolProfileService { comment, countries, currency, + dataGatheringFrequency, holdings, isActive, name, diff --git a/apps/api/src/services/twitter-bot/twitter-bot.service.ts b/apps/api/src/services/twitter-bot/twitter-bot.service.ts index ffd0c5452..4e6fd5bb2 100644 --- a/apps/api/src/services/twitter-bot/twitter-bot.service.ts +++ b/apps/api/src/services/twitter-bot/twitter-bot.service.ts @@ -3,7 +3,7 @@ import { BenchmarkService } from '@ghostfolio/api/services/benchmark/benchmark.s import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ghostfolioFearAndGreedIndexDataSourceStocks, - ghostfolioFearAndGreedIndexSymbol + ghostfolioFearAndGreedIndexSymbolStocks } from '@ghostfolio/common/config'; import { resolveFearAndGreedIndex, @@ -49,7 +49,7 @@ export class TwitterBotService implements OnModuleInit { const symbolItem = await this.symbolService.get({ dataGatheringItem: { dataSource: ghostfolioFearAndGreedIndexDataSourceStocks, - symbol: ghostfolioFearAndGreedIndexSymbol + symbol: ghostfolioFearAndGreedIndexSymbolStocks } }); diff --git a/apps/client/src/app/app.component.ts b/apps/client/src/app/app.component.ts index e1967970d..90ff2f1bc 100644 --- a/apps/client/src/app/app.component.ts +++ b/apps/client/src/app/app.component.ts @@ -1,5 +1,9 @@ import { getCssVariable } from '@ghostfolio/common/helper'; -import { InfoItem, User } from '@ghostfolio/common/interfaces'; +import { + AssetProfileIdentifier, + InfoItem, + User +} from '@ghostfolio/common/interfaces'; import { hasPermission, permissions } from '@ghostfolio/common/permissions'; import { internalRoutes, publicRoutes } from '@ghostfolio/common/routes/routes'; import { ColorScheme } from '@ghostfolio/common/types'; @@ -27,7 +31,6 @@ import { RouterLink, RouterOutlet } from '@angular/router'; -import { DataSource } from '@prisma/client'; import { Chart } from 'chart.js'; import { addIcons } from 'ionicons'; import { openOutline } from 'ionicons/icons'; @@ -269,10 +272,7 @@ export class GfAppComponent implements OnInit { private openHoldingDetailDialog({ dataSource, symbol - }: { - dataSource: DataSource; - symbol: string; - }) { + }: AssetProfileIdentifier) { this.userService .get() .pipe(takeUntilDestroyed(this.destroyRef)) diff --git a/apps/client/src/app/components/access-table/access-table.component.html b/apps/client/src/app/components/access-table/access-table.component.html index 8ba906e0f..506190cfb 100644 --- a/apps/client/src/app/components/access-table/access-table.component.html +++ b/apps/client/src/app/components/access-table/access-table.component.html @@ -9,7 +9,7 @@ Grantee - + {{ element.grantee }} diff --git a/apps/client/src/app/components/account-detail-dialog/account-detail-dialog.component.ts b/apps/client/src/app/components/account-detail-dialog/account-detail-dialog.component.ts index 7cdf3e671..5af8c9d00 100644 --- a/apps/client/src/app/components/account-detail-dialog/account-detail-dialog.component.ts +++ b/apps/client/src/app/components/account-detail-dialog/account-detail-dialog.component.ts @@ -3,7 +3,7 @@ import { UserService } from '@ghostfolio/client/services/user/user.service'; import { DEFAULT_DATE_RANGE, DEFAULT_PAGE_SIZE, - NUMERICAL_PRECISION_THRESHOLD_5_FIGURES + NUMERICAL_PRECISION_THRESHOLD_4_FIGURES } from '@ghostfolio/common/config'; import { CreateAccountBalanceDto } from '@ghostfolio/common/dtos'; import { DATE_FORMAT, downloadAsFile } from '@ghostfolio/common/helper'; @@ -245,7 +245,7 @@ export class GfAccountDetailDialogComponent implements OnInit { this.balance = balance; if ( - this.balance >= NUMERICAL_PRECISION_THRESHOLD_5_FIGURES && + this.balance >= NUMERICAL_PRECISION_THRESHOLD_4_FIGURES && this.data.deviceType === 'mobile' ) { this.balancePrecision = 0; @@ -257,7 +257,7 @@ export class GfAccountDetailDialogComponent implements OnInit { if ( this.data.deviceType === 'mobile' && this.dividendInBaseCurrency >= - NUMERICAL_PRECISION_THRESHOLD_5_FIGURES + NUMERICAL_PRECISION_THRESHOLD_4_FIGURES ) { this.dividendInBaseCurrencyPrecision = 0; } @@ -267,7 +267,7 @@ export class GfAccountDetailDialogComponent implements OnInit { if ( this.data.deviceType === 'mobile' && - this.equity >= NUMERICAL_PRECISION_THRESHOLD_5_FIGURES + this.equity >= NUMERICAL_PRECISION_THRESHOLD_4_FIGURES ) { this.equityPrecision = 0; } @@ -280,7 +280,7 @@ export class GfAccountDetailDialogComponent implements OnInit { if ( this.data.deviceType === 'mobile' && this.interestInBaseCurrency >= - NUMERICAL_PRECISION_THRESHOLD_5_FIGURES + NUMERICAL_PRECISION_THRESHOLD_4_FIGURES ) { this.interestInBaseCurrencyPrecision = 0; } diff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.html b/apps/client/src/app/components/admin-jobs/admin-jobs.html index e615db31b..fe893690b 100644 --- a/apps/client/src/app/components/admin-jobs/admin-jobs.html +++ b/apps/client/src/app/components/admin-jobs/admin-jobs.html @@ -27,7 +27,7 @@ diff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts b/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts index f7396eb1d..ae0f8fda2 100644 --- a/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts +++ b/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts @@ -305,17 +305,6 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit { ); } - protected onGather7Days() { - this.adminService - .gather7Days() - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe(() => { - setTimeout(() => { - window.location.reload(); - }, 300); - }); - } - protected onGatherMax() { this.adminService .gatherMax() @@ -334,6 +323,17 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit { .subscribe(); } + protected onGatherRecentMarketData() { + this.adminService + .gatherRecentMarketData() + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(() => { + setTimeout(() => { + window.location.reload(); + }, 300); + }); + } + protected onOpenAssetProfileDialog({ dataSource, symbol @@ -408,10 +408,7 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit { private openAssetProfileDialog({ dataSource, symbol - }: { - dataSource: DataSource; - symbol: string; - }) { + }: AssetProfileIdentifier) { this.userService .get() .pipe(takeUntilDestroyed(this.destroyRef)) diff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.html b/apps/client/src/app/components/admin-market-data/admin-market-data.html index e86e3e631..83703b548 100644 --- a/apps/client/src/app/components/admin-market-data/admin-market-data.html +++ b/apps/client/src/app/components/admin-market-data/admin-market-data.html @@ -220,7 +220,7 @@ class="no-max-width" xPosition="before" > -