diff --git a/CHANGELOG.md b/CHANGELOG.md index ab2f94cfc..8cbac09ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- Added the name to the tooltip of the chart of the holdings tab on the home page (experimental) + +### Changed + +- Exposed the timeout of the portfolio snapshot computation as an environment variable (`PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT`) +- Improved the portfolio unit tests to work with exported activity files + +### Fixed + +- Considered the language of the user settings on login with _Security Token_ + ## 2.114.0 - 2024-10-10 ### Added diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 0cf8d78bd..2c469612e 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -158,7 +158,22 @@ export class AdminController { @Param('dataSource') dataSource: DataSource, @Param('symbol') symbol: string ): Promise { - this.dataGatheringService.gatherSymbol({ dataSource, symbol }); + await this.dataGatheringService.gatherSymbol({ dataSource, symbol }); + + return; + } + + @Post('gatherMissing/:dataSource/:symbol') + @UseGuards(AuthGuard('jwt'), HasPermissionGuard) + @HasPermission(permissions.accessAdminControl) + public async gatherSymbolMissingOnly( + @Param('dataSource') dataSource: DataSource, + @Param('symbol') symbol: string + ): Promise { + await this.dataGatheringService.gatherSymbolMissingOnly({ + dataSource, + symbol + }); return; } diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts index 324087a0c..8a11d430d 100644 --- a/apps/api/src/app/admin/admin.service.ts +++ b/apps/api/src/app/admin/admin.service.ts @@ -41,9 +41,7 @@ import { PrismaClient, Property, SymbolProfile, - DataSource, - Tag, - SymbolProfileOverrides + DataSource } from '@prisma/client'; import { differenceInDays } from 'date-fns'; import { groupBy } from 'lodash'; diff --git a/apps/api/src/app/portfolio/calculator/constantPortfolioReturn/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/constantPortfolioReturn/portfolio-calculator.ts index 6dacc3875..988e01425 100644 --- a/apps/api/src/app/portfolio/calculator/constantPortfolioReturn/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/constantPortfolioReturn/portfolio-calculator.ts @@ -7,16 +7,13 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; -import { getIntervalFromDateRange } from '@ghostfolio/common/calculation-helper'; import { DATE_FORMAT, parseDate, resetHours } from '@ghostfolio/common/helper'; import { Filter, HistoricalDataItem } from '@ghostfolio/common/interfaces'; -import { DateRange } from '@ghostfolio/common/types'; import { Inject, Logger } from '@nestjs/common'; import { Big } from 'big.js'; import { addDays, - differenceInDays, eachDayOfInterval, endOfDay, format, @@ -83,19 +80,19 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator { start: Date; end: Date; }): Promise<{ chart: HistoricalDataItem[] }> { - let item = await super.getPerformance({ + const item = await super.getPerformance({ end, start }); - let itemResult = item.chart; - let dates = itemResult.map((item) => parseDate(item.date)); - let timeWeighted = await this.getTimeWeightedChartData({ + const itemResult = item.chart; + const dates = itemResult.map((item) => parseDate(item.date)); + const timeWeighted = await this.getTimeWeightedChartData({ dates }); item.chart = itemResult.map((itemInt) => { - let timeWeightedItem = timeWeighted.find( + const timeWeightedItem = timeWeighted.find( (timeWeightedItem) => timeWeightedItem.date === itemInt.date ); if (timeWeightedItem) { @@ -135,20 +132,20 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator { dateQuery: { in: [end] } }); const endString = format(end, DATE_FORMAT); - let exchangeRates = await Promise.all( + const exchangeRates = await Promise.all( Object.keys(holdings[endString]).map(async (holding) => { - let symbol = marketMap.values.find((m) => m.symbol === holding); - let symbolCurrency = this.getCurrencyFromActivities(orders, holding); - let exchangeRate = await this.exchangeRateDataService.toCurrencyAtDate( - 1, - symbolCurrency, - this.currency, - end - ); + const symbolCurrency = this.getCurrencyFromActivities(orders, holding); + const exchangeRate = + await this.exchangeRateDataService.toCurrencyAtDate( + 1, + symbolCurrency, + this.currency, + end + ); return { symbolCurrency, exchangeRate }; }) ); - let currencyRates = exchangeRates.reduce<{ [currency: string]: number }>( + const currencyRates = exchangeRates.reduce<{ [currency: string]: number }>( (all, currency): { [currency: string]: number } => { all[currency.symbolCurrency] ??= currency.exchangeRate; return all; @@ -156,12 +153,12 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator { {} ); - let totalInvestment = await Object.keys(holdings[endString]).reduce( + const totalInvestment = await Object.keys(holdings[endString]).reduce( (sum, holding) => { if (!holdings[endString][holding].toNumber()) { return sum; } - let symbol = marketMap.values.find((m) => m.symbol === holding); + const symbol = marketMap.values.find((m) => m.symbol === holding); if (symbol?.marketPrice === undefined) { Logger.warn( @@ -170,8 +167,8 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator { ); return sum; } else { - let symbolCurrency = this.getCurrency(holding); - let price = new Big(currencyRates[symbolCurrency]).mul( + const symbolCurrency = this.getCurrency(holding); + const price = new Big(currencyRates[symbolCurrency]).mul( symbol.marketPrice ); return sum.plus(new Big(price).mul(holdings[endString][holding])); @@ -191,7 +188,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator { dates = dates.sort((a, b) => a.getTime() - b.getTime()); const start = dates[0]; const end = dates[dates.length - 1]; - let marketMapTask = this.computeMarketMap({ + const marketMapTask = this.computeMarketMap({ gte: start, lt: addDays(end, 1) }); @@ -201,7 +198,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator { end ); - let data: HistoricalDataItem[] = []; + const data: HistoricalDataItem[] = []; const startString = format(start, DATE_FORMAT); data.push({ @@ -435,12 +432,12 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator { end: Date ) { const transactionDates = Object.keys(investmentByDate).sort(); - let dates = eachDayOfInterval({ start, end }, { step: 1 }) + const dates = eachDayOfInterval({ start, end }, { step: 1 }) .map((date) => { return resetHours(date); }) .sort((a, b) => a.getTime() - b.getTime()); - let currentHoldings: { [date: string]: { [symbol: string]: Big } } = {}; + const currentHoldings: { [date: string]: { [symbol: string]: Big } } = {}; this.calculateInitialHoldings(investmentByDate, start, currentHoldings); @@ -448,7 +445,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator { const dateString = format(dates[i], DATE_FORMAT); const previousDateString = format(dates[i - 1], DATE_FORMAT); if (transactionDates.some((d) => d === dateString)) { - let holdings = { ...currentHoldings[previousDateString] }; + const holdings = { ...currentHoldings[previousDateString] }; investmentByDate[dateString].forEach((trade) => { holdings[trade.SymbolProfile.symbol] ??= new Big(0); holdings[trade.SymbolProfile.symbol] = holdings[ @@ -488,7 +485,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator { for (const symbol of Object.keys(preRangeTrades)) { const trades: PortfolioOrder[] = preRangeTrades[symbol]; - let startQuantity = trades.reduce((sum, trade) => { + const startQuantity = trades.reduce((sum, trade) => { return sum.plus(trade.quantity.mul(getFactor(trade.type))); }, new Big(0)); currentHoldings[format(start, DATE_FORMAT)][symbol] = startQuantity; diff --git a/apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts b/apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts index d458be708..217ec499b 100644 --- a/apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts +++ b/apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts @@ -1,3 +1,5 @@ +import { readFileSync } from 'fs'; + export const activityDummyData = { accountId: undefined, accountUserId: undefined, @@ -29,3 +31,7 @@ export const symbolProfileDummyData = { export const userDummyData = { id: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' }; + +export function loadActivityExportFile(filePath: string) { + return JSON.parse(readFileSync(filePath, 'utf8')).activities; +} diff --git a/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts b/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts index 1361ca6cf..2f028d84d 100644 --- a/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts +++ b/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts @@ -13,7 +13,6 @@ import { OrderService } from '../../order/order.service'; import { CPRPortfolioCalculator } from './constantPortfolioReturn/portfolio-calculator'; import { MWRPortfolioCalculator } from './mwr/portfolio-calculator'; import { PortfolioCalculator } from './portfolio-calculator'; -import { TWRPortfolioCalculator } from './twr/portfolio-calculator'; export enum PerformanceCalculationType { MWR = 'MWR', // Money-Weighted Rate of Return diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts index e4531c8ae..3cb84ca14 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts @@ -156,25 +156,25 @@ describe('PortfolioCalculator', () => { dividendInBaseCurrency: new Big('0.62'), fee: new Big('19'), firstBuyDate: '2021-09-16', - grossPerformance: new Big('33.25'), - grossPerformancePercentage: new Big('0.11136043941322258691'), + grossPerformance: new Big('33.87'), + grossPerformancePercentage: new Big('0.11343693482483756447'), grossPerformancePercentageWithCurrencyEffect: new Big( - '0.11136043941322258691' + '0.11343693482483756447' ), - grossPerformanceWithCurrencyEffect: new Big('33.25'), + grossPerformanceWithCurrencyEffect: new Big('33.87'), investment: new Big('298.58'), investmentWithCurrencyEffect: new Big('298.58'), marketPrice: 331.83, marketPriceInBaseCurrency: 331.83, - netPerformance: new Big('14.25'), - netPerformancePercentage: new Big('0.04772590260566682296'), + netPerformance: new Big('14.87'), + netPerformancePercentage: new Big('0.04980239801728180052'), netPerformancePercentageWithCurrencyEffectMap: { - max: new Big('0.04772590260566682296') + max: new Big('0.04980239801728180052') }, netPerformanceWithCurrencyEffectMap: { '1d': new Big('-5.39'), - '5y': new Big('14.25'), - max: new Big('14.25'), + '5y': new Big('14.87'), + max: new Big('14.87'), wtd: new Big('-5.39') }, quantity: new Big('1'), diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts index 30b17dcd4..575d01dc6 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts @@ -1,6 +1,8 @@ +import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto'; import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface'; import { activityDummyData, + loadActivityExportFile, symbolProfileDummyData, userDummyData } from '@ghostfolio/api/app/portfolio/calculator/portfolio-calculator-test-utils'; @@ -20,6 +22,7 @@ import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; import { last } from 'lodash'; +import { join } from 'path'; jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { return { @@ -52,6 +55,8 @@ jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { }); describe('PortfolioCalculator', () => { + let activityDtos: CreateOrderDto[]; + let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; @@ -59,6 +64,15 @@ describe('PortfolioCalculator', () => { let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; + beforeAll(() => { + activityDtos = loadActivityExportFile( + join( + __dirname, + '../../../../../../../test/import/ok-novn-buy-and-sell.json' + ) + ); + }); + beforeEach(() => { configurationService = new ConfigurationService(); @@ -89,38 +103,18 @@ describe('PortfolioCalculator', () => { it.only('with NOVN.SW buy and sell', async () => { jest.useFakeTimers().setSystemTime(parseDate('2022-04-11').getTime()); - const activities: Activity[] = [ - { - ...activityDummyData, - date: new Date('2022-03-07'), - fee: 0, - quantity: 2, - SymbolProfile: { - ...symbolProfileDummyData, - currency: 'CHF', - dataSource: 'YAHOO', - name: 'Novartis AG', - symbol: 'NOVN.SW' - }, - type: 'BUY', - unitPrice: 75.8 - }, - { - ...activityDummyData, - date: new Date('2022-04-08'), - fee: 0, - quantity: 2, - SymbolProfile: { - ...symbolProfileDummyData, - currency: 'CHF', - dataSource: 'YAHOO', - name: 'Novartis AG', - symbol: 'NOVN.SW' - }, - type: 'SELL', - unitPrice: 85.73 + const activities: Activity[] = activityDtos.map((activity) => ({ + ...activityDummyData, + ...activity, + date: parseDate(activity.date), + SymbolProfile: { + ...symbolProfileDummyData, + currency: activity.currency, + dataSource: activity.dataSource, + name: 'Novartis AG', + symbol: activity.symbol } - ]; + })); const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, diff --git a/apps/api/src/app/portfolio/portfolio.controller.ts b/apps/api/src/app/portfolio/portfolio.controller.ts index 2a9e62112..fe42c9838 100644 --- a/apps/api/src/app/portfolio/portfolio.controller.ts +++ b/apps/api/src/app/portfolio/portfolio.controller.ts @@ -84,7 +84,6 @@ export class PortfolioController { @Query('assetClasses') filterByAssetClasses?: string, @Query('range') dateRange: DateRange = 'max', @Query('tags') filterByTags?: string, - @Query('isAllocation') isAllocation: boolean = false, @Query('withMarkets') withMarketsParam = 'false' ): Promise { const withMarkets = withMarketsParam === 'true'; @@ -483,8 +482,7 @@ export class PortfolioController { @Query('range') dateRange: DateRange = 'max', @Query('tags') filterByTags?: string, @Query('withExcludedAccounts') withExcludedAccounts = false, - @Query('timeWeightedPerformance') calculateTimeWeightedPerformance = false, - @Query('withItems') withItems = false + @Query('timeWeightedPerformance') calculateTimeWeightedPerformance = false ): Promise { const filters = this.apiService.buildFiltersFromQueryParams({ filterByAccounts, diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index 3054cce47..d05ac4033 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -15,7 +15,6 @@ import { FeeRatioInitialInvestment } from '@ghostfolio/api/models/rules/fees/fee import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service'; -import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { getAnnualizedPerformancePercent, @@ -818,23 +817,6 @@ export class PortfolioService { }) ); - // Convert investment, gross and net performance to currency of user - const investment = this.exchangeRateDataService.toCurrency( - position.investment?.toNumber(), - currency, - userCurrency - ); - const grossPerformance = this.exchangeRateDataService.toCurrency( - position.grossPerformance?.toNumber(), - currency, - userCurrency - ); - const netPerformance = this.exchangeRateDataService.toCurrency( - position.netPerformance?.toNumber(), - currency, - userCurrency - ); - const historicalData = await this.dataProviderService.getHistorical( [{ dataSource, symbol: aSymbol }], 'day', @@ -1899,7 +1881,7 @@ export class PortfolioService { cashDetailsWithExcludedAccounts.balanceInBaseCurrency ).minus(balanceInBaseCurrency); - let excludedAccountsAndActivities = excludedBalanceInBaseCurrency + const excludedAccountsAndActivities = excludedBalanceInBaseCurrency .plus(totalOfExcludedActivities) .toNumber(); diff --git a/apps/api/src/helper/dateQueryHelper.ts b/apps/api/src/helper/dateQueryHelper.ts index 6269dce9c..a2b312a12 100644 --- a/apps/api/src/helper/dateQueryHelper.ts +++ b/apps/api/src/helper/dateQueryHelper.ts @@ -13,8 +13,8 @@ export class DateQueryHelper { let query = dateQuery; if (dateQuery.in?.length > 0) { dates = dateQuery.in; - let end = Math.max(...dates.map((d) => d.getTime())); - let start = Math.min(...dates.map((d) => d.getTime())); + const end = Math.max(...dates.map((d) => d.getTime())); + const start = Math.min(...dates.map((d) => d.getTime())); query = { gte: resetHours(new Date(start)), lt: resetHours(addDays(end, 1)) diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts index cca393a2a..dafd4803c 100644 --- a/apps/api/src/services/configuration/configuration.service.ts +++ b/apps/api/src/services/configuration/configuration.service.ts @@ -4,6 +4,7 @@ import { DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE, DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA, DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT, + DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT, DEFAULT_ROOT_URL } from '@ghostfolio/common/config'; @@ -59,6 +60,9 @@ export class ConfigurationService { PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT: num({ default: DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT }), + PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: num({ + default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT + }), REDIS_DB: num({ default: 0 }), REDIS_HOST: str({ default: 'localhost' }), REDIS_PASSWORD: str({ default: '' }), diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 9512b6530..9853b0186 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -475,7 +475,7 @@ export class DataProviderService { } response[symbol] = dataProviderResponse; - let quotesCacheTTL = + const quotesCacheTTL = this.getAppropriateCacheTTL(dataProviderResponse); this.redisCacheService.set( @@ -573,8 +573,8 @@ export class DataProviderService { if (dataProviderResponse.dataSource === 'MANUAL') { quotesCacheTTL = 14400; // 4h Cache for Manual Service } else if (dataProviderResponse.marketState === 'closed') { - let date = new Date(); - let dayOfWeek = date.getDay(); + const date = new Date(); + const dayOfWeek = date.getDay(); if (dayOfWeek === 0 || dayOfWeek === 6) { quotesCacheTTL = 14400; } else if (date.getHours() > 16) { diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 88936d5c4..6b77e373c 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -3,7 +3,6 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.i import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper'; import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; -import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper'; import { resetHours } from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts index 2745aa288..d469671ea 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts @@ -1,20 +1,25 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; -import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; +import { + IDataGatheringItem, + IDataProviderHistoricalResponse +} from '@ghostfolio/api/services/interfaces/interfaces'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { DATA_GATHERING_QUEUE, DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE, DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA, GATHER_ASSET_PROFILE_PROCESS, - GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME + GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, + GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME } from '@ghostfolio/common/config'; import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { Process, Processor } from '@nestjs/bull'; import { Injectable, Logger } from '@nestjs/common'; -import { Prisma } from '@prisma/client'; +import { DataSource, Prisma } from '@prisma/client'; import { Job } from 'bull'; +import { isNumber } from 'class-validator'; import { addDays, format, @@ -22,7 +27,9 @@ import { getMonth, getYear, isBefore, - parseISO + parseISO, + eachDayOfInterval, + isEqual } from 'date-fns'; import { DataGatheringService } from './data-gathering.service'; @@ -150,4 +157,148 @@ export class DataGatheringProcessor { throw new Error(error); } } + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA ?? + DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA.toString(), + 10 + ), + name: GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME + }) + public async gatherMissingHistoricalMarketData(job: Job) { + try { + const { dataSource, date, symbol } = job.data; + + Logger.log( + `Historical market data gathering for missing values has been started for ${symbol} (${dataSource}) at ${format( + date, + DATE_FORMAT + )}`, + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` + ); + const entries = await this.marketDataService.marketDataItems({ + where: { + AND: { + symbol: { + equals: symbol + }, + dataSource: { + equals: dataSource + } + } + }, + orderBy: { + date: 'asc' + }, + take: 1 + }); + const firstEntry = entries[0]; + const marketData = await this.marketDataService + .getRange({ + assetProfileIdentifiers: [{ dataSource, symbol }], + dateQuery: { + gte: addDays(firstEntry.date, -10) + } + }) + .then((md) => md.map((m) => m.date)); + + let dates = eachDayOfInterval( + { + start: firstEntry.date, + end: new Date() + }, + { + step: 1 + } + ); + dates = dates.filter((d) => !marketData.some((md) => isEqual(md, d))); + + const historicalData = await this.dataProviderService.getHistoricalRaw({ + dataGatheringItems: [{ dataSource, symbol }], + from: firstEntry.date, + to: new Date() + }); + + const data: Prisma.MarketDataUpdateInput[] = + this.mapToMarketUpsertDataInputs( + dates, + historicalData, + symbol, + dataSource + ); + + await this.marketDataService.updateMany({ data }); + + Logger.log( + `Historical market data gathering for missing values has been completed for ${symbol} (${dataSource}) at ${format( + date, + DATE_FORMAT + )}`, + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` + ); + } catch (error) { + Logger.error( + error, + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` + ); + + throw new Error(error); + } + } + + private mapToMarketUpsertDataInputs( + missingMarketData: Date[], + historicalData: Record< + string, + Record + >, + symbol: string, + dataSource: DataSource + ): Prisma.MarketDataUpdateInput[] { + return missingMarketData.map((date) => { + if ( + isNumber( + historicalData[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice + ) + ) { + return { + date, + symbol, + dataSource, + marketPrice: + historicalData[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice + }; + } else { + let earlierDate = date; + let index = 0; + while ( + !isNumber( + historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)] + ?.marketPrice + ) + ) { + earlierDate = addDays(earlierDate, -1); + index++; + if (index > 10) { + break; + } + } + if ( + isNumber( + historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)] + ?.marketPrice + ) + ) { + return { + date, + symbol, + dataSource, + marketPrice: + historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)] + ?.marketPrice + }; + } + } + }); + } } diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index a66e05b72..24b174785 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -13,6 +13,8 @@ import { DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS, + GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, + GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS, PROPERTY_BENCHMARKS } from '@ghostfolio/common/config'; import { @@ -28,7 +30,6 @@ import { import { InjectQueue } from '@nestjs/bull'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { DataSource } from '@prisma/client'; -import AwaitLock from 'await-lock'; import { JobOptions, Queue } from 'bull'; import { format, min, subDays, subYears } from 'date-fns'; import { isEmpty } from 'lodash'; @@ -48,8 +49,6 @@ export class DataGatheringService { private readonly symbolProfileService: SymbolProfileService ) {} - lock = new AwaitLock(); - public async addJobToQueue({ data, name, @@ -114,6 +113,24 @@ export class DataGatheringService { }); } + public async gatherSymbolMissingOnly({ + dataSource, + symbol + }: AssetProfileIdentifier) { + const dataGatheringItems = (await this.getSymbolsMax()).filter( + (dataGatheringItem) => { + return ( + dataGatheringItem.dataSource === dataSource && + dataGatheringItem.symbol === symbol + ); + } + ); + await this.gatherMissingDataSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); + } + public async gatherSymbolForDate({ dataSource, date, @@ -296,6 +313,35 @@ export class DataGatheringService { ); } + public async gatherMissingDataSymbols({ + dataGatheringItems, + priority + }: { + dataGatheringItems: IDataGatheringItem[]; + priority: number; + }) { + await this.addJobsToQueue( + dataGatheringItems.map(({ dataSource, date, symbol }) => { + return { + data: { + dataSource, + date, + symbol + }, + name: GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, + opts: { + ...GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS, + priority, + jobId: `${getAssetProfileIdentifier({ + dataSource, + symbol + })}-missing-${format(date, DATE_FORMAT)}` + } + }; + }) + ); + } + public async getAllAssetProfileIdentifiers(): Promise< AssetProfileIdentifier[] > { diff --git a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts index 620feda53..058d971d8 100644 --- a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts +++ b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts @@ -8,7 +8,10 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data- import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; -import { PORTFOLIO_SNAPSHOT_QUEUE } from '@ghostfolio/common/config'; +import { + DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT, + PORTFOLIO_SNAPSHOT_QUEUE +} from '@ghostfolio/common/config'; import { BullModule } from '@nestjs/bull'; import { Module } from '@nestjs/common'; @@ -20,7 +23,14 @@ import { PortfolioSnapshotProcessor } from './portfolio-snapshot.processor'; imports: [ AccountBalanceModule, BullModule.registerQueue({ - name: PORTFOLIO_SNAPSHOT_QUEUE + name: PORTFOLIO_SNAPSHOT_QUEUE, + settings: { + lockDuration: parseInt( + process.env.PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT ?? + DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT.toString(), + 10 + ) + } }), ConfigurationModule, DataProviderModule, diff --git a/apps/api/src/services/symbol-profile/symbol-profile-overwrite.service.ts b/apps/api/src/services/symbol-profile/symbol-profile-overwrite.service.ts index ed454ae9f..bb43a0756 100644 --- a/apps/api/src/services/symbol-profile/symbol-profile-overwrite.service.ts +++ b/apps/api/src/services/symbol-profile/symbol-profile-overwrite.service.ts @@ -47,7 +47,7 @@ export class SymbolProfileOverwriteService { Symbol: string, datasource: DataSource ): Promise { - let SymbolProfileId = await this.prismaService.symbolProfile + const SymbolProfileId = await this.prismaService.symbolProfile .findFirst({ where: { symbol: Symbol, @@ -56,7 +56,7 @@ export class SymbolProfileOverwriteService { }) .then((s) => s.id); - let symbolProfileIdSaved = await this.prismaService.symbolProfileOverrides + const symbolProfileIdSaved = await this.prismaService.symbolProfileOverrides .findFirst({ where: { symbolProfileId: SymbolProfileId diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts index d79ba44f3..4eb65a70d 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts @@ -223,6 +223,16 @@ export class AssetProfileDialog implements OnDestroy, OnInit { .subscribe(() => {}); } + public onGatherSymbolMissingOnly({ + dataSource, + symbol + }: AssetProfileIdentifier) { + this.adminService + .gatherSymbolMissingOnly({ dataSource, symbol }) + .pipe(takeUntil(this.unsubscribeSubject)) + .subscribe(() => {}); + } + public onImportHistoricalData() { try { const marketData = csvToJson( diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html index 76bce400b..14b2f423c 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html @@ -31,6 +31,19 @@ > Gather Historical Data +