diff --git a/apps/api/src/app/activities/activities.service.ts b/apps/api/src/app/activities/activities.service.ts index 821185e11..77642f4f8 100644 --- a/apps/api/src/app/activities/activities.service.ts +++ b/apps/api/src/app/activities/activities.service.ts @@ -11,12 +11,16 @@ import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathe import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DATA_GATHERING_QUEUE_PRIORITY_HIGH, + DEFAULT_CURRENCY, GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS, - ghostfolioPrefix, - TAG_ID_EXCLUDE_FROM_ANALYSIS + ghostfolioPrefix } from '@ghostfolio/common/config'; -import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; +import { + DATE_FORMAT, + getAssetProfileIdentifier, + resetHours +} from '@ghostfolio/common/helper'; import { ActivitiesResponse, Activity, @@ -39,8 +43,8 @@ import { } from '@prisma/client'; import { Big } from 'big.js'; import { isUUID } from 'class-validator'; -import { endOfToday, isAfter } from 'date-fns'; -import { groupBy, uniqBy } from 'lodash'; +import { endOfToday, format, isAfter, subDays } from 'date-fns'; +import { uniqBy } from 'lodash'; import { randomUUID } from 'node:crypto'; @Injectable() @@ -469,7 +473,7 @@ export class ActivitiesService { sortColumn, sortDirection = 'asc', startDate, - take = Number.MAX_SAFE_INTEGER, + take, types, userCurrency, userId, @@ -488,166 +492,112 @@ export class ActivitiesService { userId: string; withExcludedAccountsAndActivities?: boolean; }): Promise { - let orderBy: Prisma.Enumerable = [ - { date: 'asc' } - ]; - - const where: Prisma.OrderWhereInput = { userId }; - - if (endDate || startDate) { - where.AND = []; + const where: Prisma.OrderWhereInput = { + ...(includeDrafts === false ? { isDraft: false } : {}), + ...(types?.length > 0 ? { type: { in: types } } : {}), + ...(userId ? { userId } : {}) + }; - if (endDate) { - where.AND.push({ date: { lte: endDate } }); - } + if (startDate) { + where.date = { gte: resetHours(startDate) }; + } - if (startDate) { - where.AND.push({ date: { gt: startDate } }); - } + if (endDate) { + where.date = { + ...(where.date as Prisma.DateTimeFilter), + lte: resetHours(endDate) + }; } - const { - ACCOUNT: filtersByAccount, - ASSET_CLASS: filtersByAssetClass, - TAG: filtersByTag - } = groupBy(filters, ({ type }) => { - return type; - }); + if (withExcludedAccountsAndActivities === false) { + where.account = { isExcluded: false }; + } - const filterByDataSource = filters?.find(({ type }) => { - return type === 'DATA_SOURCE'; - })?.id; + if (filters?.length > 0) { + where.OR = []; - const filterBySymbol = filters?.find(({ type }) => { - return type === 'SYMBOL'; - })?.id; + const accountIds = filters + .filter(({ type }) => type === 'ACCOUNT') + .map(({ id }) => id); + if (accountIds.length > 0) { + where.OR.push({ accountId: { in: accountIds } }); + } - const searchQuery = filters?.find(({ type }) => { - return type === 'SEARCH_QUERY'; - })?.id; + const assetClasses = filters + .filter(({ type }) => type === 'ASSET_CLASS') + .map(({ id }) => id) as AssetClass[]; + if (assetClasses.length > 0) { + where.OR.push({ SymbolProfile: { assetClass: { in: assetClasses } } }); + } - if (filtersByAccount?.length > 0) { - where.accountId = { - in: filtersByAccount.map(({ id }) => { - return id; - }) - }; + const tags = filters + .filter(({ type }) => type === 'TAG') + .map(({ id }) => id); + if (tags.length > 0) { + where.OR.push({ tags: { some: { id: { in: tags } } } }); + } } - if (includeDrafts === false) { - where.isDraft = false; - } + let orderBy: Prisma.OrderOrderByWithRelationInput[] = [ + { date: sortDirection } + ]; - if (filtersByAssetClass?.length > 0) { - where.SymbolProfile = { - OR: [ - { - AND: [ - { - OR: filtersByAssetClass.map(({ id }) => { - return { assetClass: AssetClass[id] }; - }) - }, - { - OR: [ - { SymbolProfileOverrides: { is: null } }, - { SymbolProfileOverrides: { assetClass: null } } - ] - } - ] - }, - { - SymbolProfileOverrides: { - OR: filtersByAssetClass.map(({ id }) => { - return { assetClass: AssetClass[id] }; - }) - } - } - ] - }; - } + if (sortColumn) { + orderBy = []; - if (filterByDataSource && filterBySymbol) { - if (where.SymbolProfile) { - where.SymbolProfile = { - AND: [ - where.SymbolProfile, - { - AND: [ - { dataSource: filterByDataSource as DataSource }, - { symbol: filterBySymbol } - ] - } - ] - }; + if ( + ['currency', 'fee', 'quantity', 'type', 'unitPrice'].includes( + sortColumn + ) + ) { + orderBy.push({ [sortColumn]: sortDirection }); } else { - where.SymbolProfile = { - AND: [ - { dataSource: filterByDataSource as DataSource }, - { symbol: filterBySymbol } - ] - }; + if (sortColumn === 'SymbolProfile.name') { + orderBy.push({ SymbolProfile: { name: sortDirection } }); + } else if (sortColumn === 'account.name') { + orderBy.push({ account: { name: sortDirection } }); + } } } - if (searchQuery) { - const searchQueryWhereInput: Prisma.SymbolProfileWhereInput[] = [ - { id: { mode: 'insensitive', startsWith: searchQuery } }, - { isin: { mode: 'insensitive', startsWith: searchQuery } }, - { name: { mode: 'insensitive', startsWith: searchQuery } }, - { symbol: { mode: 'insensitive', startsWith: searchQuery } } - ]; - - if (where.SymbolProfile) { - where.SymbolProfile = { - AND: [ - where.SymbolProfile, - { - OR: searchQueryWhereInput - } - ] - }; - } else { - where.SymbolProfile = { - OR: searchQueryWhereInput - }; - } - } + const count = await this.prismaService.order.count({ where }); - if (filtersByTag?.length > 0) { - where.tags = { - some: { - OR: filtersByTag.map(({ id }) => { - return { id }; - }) - } - }; - } + let orders: OrderWithAccount[] = []; - if (sortColumn) { - orderBy = [{ [sortColumn]: sortDirection }]; - } + // If take is undefined and count is extremely large, batch fetch to prevent Prisma P2029 limits + const BATCH_SIZE = 5000; - if (types?.length > 0) { - where.type = { in: types }; - } + if (take === undefined && count > BATCH_SIZE) { + let currentSkip = skip || 0; + const totalToFetch = count - currentSkip; - if (withExcludedAccountsAndActivities === false) { - where.OR = [ - { account: null }, - { account: { NOT: { isExcluded: true } } } - ]; - - where.tags = { - ...where.tags, - none: { - id: TAG_ID_EXCLUDE_FROM_ANALYSIS + while (orders.length < totalToFetch) { + const batch = await this.orders({ + skip: currentSkip, + take: BATCH_SIZE, + where, + include: { + account: { + include: { + platform: true + } + }, + // eslint-disable-next-line @typescript-eslint/naming-convention + SymbolProfile: true, + tags: true + }, + orderBy: [...orderBy, { id: sortDirection }] + }); + + if (batch.length === 0) { + break; } - }; - } - const [orders, count] = await Promise.all([ - this.orders({ + orders = orders.concat(batch); + currentSkip += batch.length; + } + } else { + orders = await this.orders({ skip, take, where, @@ -662,9 +612,8 @@ export class ActivitiesService { tags: true }, orderBy: [...orderBy, { id: sortDirection }] - }), - this.prismaService.order.count({ where }) - ]); + }); + } const assetProfileIdentifiers = uniqBy( orders.map(({ SymbolProfile }) => { @@ -685,60 +634,178 @@ export class ActivitiesService { assetProfileIdentifiers ); - const activities = await Promise.all( - orders.map(async (order) => { - const assetProfile = assetProfiles.find(({ dataSource, symbol }) => { - return ( - dataSource === order.SymbolProfile.dataSource && - symbol === order.SymbolProfile.symbol - ); - }); + let exchangeRatesToUser: any = {}; + let exchangeRatesToDefault: any = {}; + + if (orders.length > 0) { + let minDate = orders[0].date; + let maxDate = orders[0].date; + const uniqueCurrencies = new Set(); + uniqueCurrencies.add(userCurrency); + uniqueCurrencies.add(DEFAULT_CURRENCY); + + const uniqueDatesSet = new Set(); + + for (const order of orders) { + if (order.date < minDate) { + minDate = order.date; + } + if (order.date > maxDate) { + maxDate = order.date; + } + uniqueDatesSet.add(resetHours(order.date).getTime()); + + if (order.currency) { + uniqueCurrencies.add(order.currency); + } + if (order.SymbolProfile?.currency) { + uniqueCurrencies.add(order.SymbolProfile.currency); + } + } + + const currenciesList = Array.from(uniqueCurrencies).filter(Boolean); + const uniqueDates = Array.from(uniqueDatesSet).map( + (time) => new Date(time) + ); + const startDatePreload = subDays(resetHours(minDate), 1); + const endDatePreload = resetHours(maxDate); + + const [ratesUser, ratesDefault] = await Promise.all([ + this.exchangeRateDataService.getExchangeRatesByCurrency({ + currencies: currenciesList, + dates: uniqueDates, + endDate: endDatePreload, + startDate: startDatePreload, + targetCurrency: userCurrency + }), + this.exchangeRateDataService.getExchangeRatesByCurrency({ + currencies: currenciesList, + dates: uniqueDates, + endDate: endDatePreload, + startDate: startDatePreload, + targetCurrency: DEFAULT_CURRENCY + }) + ]); + + exchangeRatesToUser = ratesUser; + exchangeRatesToDefault = ratesDefault; + } + + const getPreloadedRate = ( + from: string, + to: string, + dateStr: string + ): number | undefined => { + if (from === to) { + return 1; + } - const value = new Big(order.quantity).mul(order.unitPrice).toNumber(); - - const [ - feeInAssetProfileCurrency, - feeInBaseCurrency, - unitPriceInAssetProfileCurrency, - valueInBaseCurrency - ] = await Promise.all([ - this.exchangeRateDataService.toCurrencyAtDate( - order.fee, - order.currency ?? order.SymbolProfile.currency, - order.SymbolProfile.currency, - order.date - ), - this.exchangeRateDataService.toCurrencyAtDate( - order.fee, - order.currency ?? order.SymbolProfile.currency, - userCurrency, - order.date - ), - this.exchangeRateDataService.toCurrencyAtDate( - order.unitPrice, - order.currency ?? order.SymbolProfile.currency, - order.SymbolProfile.currency, - order.date - ), - this.exchangeRateDataService.toCurrencyAtDate( + if (to === userCurrency) { + return exchangeRatesToUser[`${from}${userCurrency}`]?.[dateStr]; + } + + if (to === DEFAULT_CURRENCY) { + return exchangeRatesToDefault[`${from}${DEFAULT_CURRENCY}`]?.[dateStr]; + } + + if (from === DEFAULT_CURRENCY) { + const rateToDefault = + exchangeRatesToDefault[`${to}${DEFAULT_CURRENCY}`]?.[dateStr]; + return rateToDefault ? 1 / rateToDefault : undefined; + } + + const rateFromToDefault = + exchangeRatesToDefault[`${from}${DEFAULT_CURRENCY}`]?.[dateStr]; + const rateToToDefault = + exchangeRatesToDefault[`${to}${DEFAULT_CURRENCY}`]?.[dateStr]; + if (rateFromToDefault !== undefined && rateToToDefault) { + return rateFromToDefault / rateToToDefault; + } + + return undefined; + }; + + const convertValue = async ( + val: number, + from: string, + to: string, + date: Date + ): Promise => { + if (val === 0) { + return 0; + } + const dateStr = format(date, DATE_FORMAT); + const rate = getPreloadedRate(from, to, dateStr); + if (rate !== undefined && !isNaN(rate)) { + return rate * val; + } + return this.exchangeRateDataService.toCurrencyAtDate(val, from, to, date); + }; + + const activities = []; + const chunkSize = 500; + + for (let i = 0; i < orders.length; i += chunkSize) { + const chunk = orders.slice(i, i + chunkSize); + + const processedChunk = await Promise.all( + chunk.map(async (order) => { + const assetProfile = assetProfiles.find(({ dataSource, symbol }) => { + return ( + dataSource === order.SymbolProfile.dataSource && + symbol === order.SymbolProfile.symbol + ); + }); + + const value = new Big(order.quantity).mul(order.unitPrice).toNumber(); + + const [ + feeInAssetProfileCurrency, + feeInBaseCurrency, + unitPriceInAssetProfileCurrency, + valueInBaseCurrency + ] = await Promise.all([ + convertValue( + order.fee, + order.currency ?? order.SymbolProfile.currency, + order.SymbolProfile.currency, + order.date + ), + convertValue( + order.fee, + order.currency ?? order.SymbolProfile.currency, + userCurrency, + order.date + ), + convertValue( + order.unitPrice, + order.currency ?? order.SymbolProfile.currency, + order.SymbolProfile.currency, + order.date + ), + convertValue( + value, + order.currency ?? order.SymbolProfile.currency, + userCurrency, + order.date + ) + ]); + + return { + ...order, + feeInAssetProfileCurrency: feeInAssetProfileCurrency ?? order.fee, + feeInBaseCurrency: feeInBaseCurrency ?? order.fee, + unitPriceInAssetProfileCurrency: + unitPriceInAssetProfileCurrency ?? order.unitPrice, value, - order.currency ?? order.SymbolProfile.currency, - userCurrency, - order.date - ) - ]); + valueInBaseCurrency: valueInBaseCurrency ?? value, + SymbolProfile: assetProfile + }; + }) + ); - return { - ...order, - feeInAssetProfileCurrency, - feeInBaseCurrency, - unitPriceInAssetProfileCurrency, - value, - valueInBaseCurrency, - SymbolProfile: assetProfile - }; - }) - ); + activities.push(...processedChunk); + } return { activities, count }; } diff --git a/apps/api/src/app/portfolio/calculator/mwr/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/mwr/portfolio-calculator.ts index 1460892fa..5835b814a 100644 --- a/apps/api/src/app/portfolio/calculator/mwr/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/mwr/portfolio-calculator.ts @@ -23,7 +23,7 @@ export class MwrPortfolioCalculator extends PortfolioCalculator { }; start: Date; step?: number; - } & AssetProfileIdentifier): SymbolMetrics { + } & AssetProfileIdentifier): Promise { throw new Error('Method not implemented.'); } } diff --git a/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts b/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts index 7b5ab1a0d..04eab8fcb 100644 --- a/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts +++ b/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts @@ -34,6 +34,7 @@ export class PortfolioCalculatorFactory { calculationType, currency, filters = [], + skipInitialize = false, userId }: { accountBalanceItems?: HistoricalDataItem[]; @@ -41,6 +42,7 @@ export class PortfolioCalculatorFactory { calculationType: PerformanceCalculationType; currency: string; filters?: Filter[]; + skipInitialize?: boolean; userId: string; }): PortfolioCalculator { switch (calculationType) { @@ -55,7 +57,8 @@ export class PortfolioCalculatorFactory { currentRateService: this.currentRateService, exchangeRateDataService: this.exchangeRateDataService, portfolioSnapshotService: this.portfolioSnapshotService, - redisCacheService: this.redisCacheService + redisCacheService: this.redisCacheService, + skipInitialize }); case PerformanceCalculationType.ROAI: @@ -69,7 +72,8 @@ export class PortfolioCalculatorFactory { currentRateService: this.currentRateService, exchangeRateDataService: this.exchangeRateDataService, portfolioSnapshotService: this.portfolioSnapshotService, - redisCacheService: this.redisCacheService + redisCacheService: this.redisCacheService, + skipInitialize }); case PerformanceCalculationType.ROI: @@ -83,7 +87,8 @@ export class PortfolioCalculatorFactory { currentRateService: this.currentRateService, exchangeRateDataService: this.exchangeRateDataService, portfolioSnapshotService: this.portfolioSnapshotService, - redisCacheService: this.redisCacheService + redisCacheService: this.redisCacheService, + skipInitialize }); case PerformanceCalculationType.TWR: @@ -97,7 +102,8 @@ export class PortfolioCalculatorFactory { currentRateService: this.currentRateService, exchangeRateDataService: this.exchangeRateDataService, portfolioSnapshotService: this.portfolioSnapshotService, - redisCacheService: this.redisCacheService + redisCacheService: this.redisCacheService, + skipInitialize }); default: diff --git a/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts index d57b85d8c..9bd422fc2 100644 --- a/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts @@ -59,6 +59,13 @@ import { } from 'date-fns'; import { isNumber, sortBy, sum, uniqBy } from 'lodash'; +const yieldToEventLoop = async () => { + if (process.env.NODE_ENV === 'test') { + return; + } + await new Promise((resolve) => setImmediate(resolve)); +}; + export abstract class PortfolioCalculator { protected static readonly ENABLE_LOGGING = false; @@ -90,6 +97,7 @@ export abstract class PortfolioCalculator { filters, portfolioSnapshotService, redisCacheService, + skipInitialize = false, userId }: { accountBalanceItems: HistoricalDataItem[]; @@ -101,6 +109,7 @@ export abstract class PortfolioCalculator { filters: Filter[]; portfolioSnapshotService: PortfolioSnapshotService; redisCacheService: RedisCacheService; + skipInitialize?: boolean; userId: string; }) { this.accountBalanceItems = accountBalanceItems; @@ -166,9 +175,9 @@ export abstract class PortfolioCalculator { this.endDate = endOfDay(endDate); this.startDate = startOfDay(startDate); - this.computeTransactionPoints(); - - this.snapshotPromise = this.initialize(); + if (!skipInitialize) { + this.snapshotPromise = this.initialize(); + } } protected abstract calculateOverallPerformance( @@ -177,6 +186,11 @@ export abstract class PortfolioCalculator { @LogPerformance public async computeSnapshot(): Promise { + console.log('[Trace] computeSnapshot started'); + if (!this.transactionPoints) { + await this.computeTransactionPoints(); + } + const lastTransactionPoint = this.transactionPoints.at(-1); const transactionPoints = this.transactionPoints?.filter(({ date }) => { @@ -234,6 +248,8 @@ export abstract class PortfolioCalculator { } } + Logger.log('Fetching exchange rates...', 'Trace'); + const t1 = Date.now(); const exchangeRatesByCurrency = await this.exchangeRateDataService.getExchangeRatesByCurrency({ currencies: Array.from(new Set(Object.values(currencies))), @@ -242,6 +258,13 @@ export abstract class PortfolioCalculator { targetCurrency: this.currency }); + Logger.log( + 'Exchange rates fetched in ' + + (Date.now() - t1) + + 'ms. Fetching market data...', + 'Trace' + ); + const t2 = Date.now(); const { dataProviderInfos, errors: currentRateErrors, @@ -256,6 +279,13 @@ export abstract class PortfolioCalculator { this.dataProviderInfos = dataProviderInfos; + Logger.log( + 'Market data fetched in ' + + (Date.now() - t2) + + 'ms. Processing symbols...', + 'Trace' + ); + const t3 = Date.now(); const marketSymbolMap: { [date: string]: { [symbol: string]: Big }; } = {}; @@ -294,6 +324,13 @@ export abstract class PortfolioCalculator { chartDateMap[accountBalanceItem.date] = true; } + Logger.log( + 'Symbols processed in ' + + (Date.now() - t3) + + 'ms. Processing positions...', + 'Trace' + ); + console.log('t4', Date.now()); const chartDates = sortBy(Object.keys(chartDateMap), (chartDate) => { return chartDate; }); @@ -338,7 +375,13 @@ export abstract class PortfolioCalculator { }; } = {}; - for (const item of lastTransactionPoint.items) { + Logger.log('Starting symbol metrics loop...', 'Trace'); + console.log('t5', Date.now()); + for (let i = 0; i < lastTransactionPoint.items.length; i++) { + if (i % 5 === 0) { + await yieldToEventLoop(); + } + const item = lastTransactionPoint.items[i]; const marketPriceInBaseCurrency = ( marketSymbolMap[endDateString]?.[item.symbol] ?? item.averagePrice ).mul( @@ -374,7 +417,7 @@ export abstract class PortfolioCalculator { totalInvestment, totalInvestmentWithCurrencyEffect, totalLiabilitiesInBaseCurrency - } = this.getSymbolMetrics({ + } = await this.getSymbolMetrics({ chartDateMap, marketSymbolMap, dataSource: item.dataSource, @@ -483,7 +526,11 @@ export abstract class PortfolioCalculator { let lastKnownBalance = new Big(0); - for (const dateString of chartDates) { + for (let c = 0; c < chartDates.length; c++) { + if (c % 100 === 0) { + await yieldToEventLoop(); + } + const dateString = chartDates[c]; if (accountBalanceItemsMap[dateString] !== undefined) { // If there's an exact balance for this date, update lastKnownBalance lastKnownBalance = accountBalanceItemsMap[dateString]; @@ -831,7 +878,7 @@ export abstract class PortfolioCalculator { [date: string]: { [symbol: string]: Big }; }; start: Date; - } & AssetProfileIdentifier): SymbolMetrics; + } & AssetProfileIdentifier): Promise; public getTransactionPoints() { return this.transactionPoints; @@ -924,23 +971,38 @@ export abstract class PortfolioCalculator { } @LogPerformance - private computeTransactionPoints() { + protected async computeTransactionPoints() { + console.log( + '[Trace] computeTransactionPoints started, activities count: ' + + this.activities.length + ); this.transactionPoints = []; const symbols: { [symbol: string]: TransactionPointSymbol } = {}; let lastDate: string = null; let lastTransactionPoint: TransactionPoint = null; - for (const { - date, - fee, - feeInBaseCurrency, - quantity, - SymbolProfile, - tags, - type, - unitPrice - } of this.activities) { + for (let i = 0; i < this.activities.length; i++) { + if (i % 500 === 0) { + console.log( + '[Trace] computeTransactionPoints progress: ' + + i + + '/' + + this.activities.length + ); + await yieldToEventLoop(); + } + + const { + date, + fee, + feeInBaseCurrency, + quantity, + SymbolProfile, + tags, + type, + unitPrice + } = this.activities[i]; let currentTransactionPointItem: TransactionPointSymbol; const assetSubClass = SymbolProfile.assetSubClass; diff --git a/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator-msft-buy-and-sell.spec.ts b/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator-msft-buy-and-sell.spec.ts index baa6ae1ed..6f560129e 100644 --- a/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator-msft-buy-and-sell.spec.ts +++ b/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator-msft-buy-and-sell.spec.ts @@ -72,7 +72,7 @@ describe('PortfolioCalculator', () => { }); describe('get transaction point', () => { - it('with MSFT buy and sell with fractional quantities (multiples of 1/3)', () => { + it('with MSFT buy and sell with fractional quantities (multiples of 1/3)', async () => { jest.useFakeTimers().setSystemTime(parseDate('2024-04-01').getTime()); const activities: Activity[] = [ @@ -133,6 +133,7 @@ describe('PortfolioCalculator', () => { userId: userDummyData.id }); + await portfolioCalculator.computeSnapshot(); const transactionPoints = portfolioCalculator.getTransactionPoints(); const lastTransactionPoint = transactionPoints[transactionPoints.length - 1]; diff --git a/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator.ts index 2841e9975..009f9d2e4 100644 --- a/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/roai/portfolio-calculator.ts @@ -23,6 +23,13 @@ import { } from 'date-fns'; import { cloneDeep, sortBy } from 'lodash'; +const yieldToEventLoop = async () => { + if (process.env.NODE_ENV === 'test') { + return; + } + await new Promise((resolve) => setImmediate(resolve)); +}; + export class RoaiPortfolioCalculator extends PortfolioCalculator { private chartDates: string[]; @@ -127,7 +134,7 @@ export class RoaiPortfolioCalculator extends PortfolioCalculator { return PerformanceCalculationType.ROAI; } - protected getSymbolMetrics({ + protected async getSymbolMetrics({ chartDateMap, dataSource, end, @@ -143,7 +150,7 @@ export class RoaiPortfolioCalculator extends PortfolioCalculator { [date: string]: { [symbol: string]: Big }; }; start: Date; - } & AssetProfileIdentifier): SymbolMetrics { + } & AssetProfileIdentifier): Promise { const currentExchangeRate = exchangeRates[format(new Date(), DATE_FORMAT)]; const currentValues: { [date: string]: Big } = {}; const currentValuesWithCurrencyEffect: { [date: string]: Big } = {}; @@ -345,7 +352,11 @@ export class RoaiPortfolioCalculator extends PortfolioCalculator { this.chartDates = Object.keys(chartDateMap).sort(); } - for (const dateString of this.chartDates) { + for (let d = 0; d < this.chartDates.length; d++) { + if (d % 500 === 0) { + await yieldToEventLoop(); + } + const dateString = this.chartDates[d]; if (dateString < startDateString) { continue; } else if (dateString > endDateString) { @@ -408,6 +419,10 @@ export class RoaiPortfolioCalculator extends PortfolioCalculator { let sumOfTimeWeightedInvestmentsWithCurrencyEffect = new Big(0); for (let i = 0; i < orders.length; i += 1) { + if (i % 500 === 0) { + await yieldToEventLoop(); + } + const order = orders[i]; if (PortfolioCalculator.ENABLE_LOGGING) { @@ -887,6 +902,10 @@ export class RoaiPortfolioCalculator extends PortfolioCalculator { let dayCount = 0; for (let i = this.chartDates.length - 1; i >= 0; i -= 1) { + if (i % 500 === 0) { + await yieldToEventLoop(); + } + const date = this.chartDates[i]; if (date > rangeEndDateString) { diff --git a/apps/api/src/app/portfolio/calculator/roi/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/roi/portfolio-calculator.ts index b4929c570..685bc604d 100644 --- a/apps/api/src/app/portfolio/calculator/roi/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/roi/portfolio-calculator.ts @@ -23,7 +23,7 @@ export class RoiPortfolioCalculator extends PortfolioCalculator { }; start: Date; step?: number; - } & AssetProfileIdentifier): SymbolMetrics { + } & AssetProfileIdentifier): Promise { throw new Error('Method not implemented.'); } } diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts index 8a58f816a..e95a5737a 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.ts @@ -23,7 +23,7 @@ export class TwrPortfolioCalculator extends PortfolioCalculator { }; start: Date; step?: number; - } & AssetProfileIdentifier): SymbolMetrics { + } & AssetProfileIdentifier): Promise { throw new Error('Method not implemented.'); } } diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index f0a451975..8f9da8077 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -24,8 +24,6 @@ import { GetValuesParams } from './interfaces/get-values-params.interface'; @Injectable() export class CurrentRateService { - private static readonly MARKET_DATA_PAGE_SIZE = 50000; - public constructor( private readonly activitiesService: ActivitiesService, private readonly dataProviderService: DataProviderService, @@ -84,32 +82,27 @@ export class CurrentRateService { return { dataSource, symbol }; }); - const marketDataCount = await this.marketDataService.getRangeCount({ - assetProfileIdentifiers, - dateQuery - }); - - for ( - let i = 0; - i < marketDataCount; - i += CurrentRateService.MARKET_DATA_PAGE_SIZE - ) { - // Use page size to limit the number of records fetched at once - const data = await this.marketDataService.getRange({ - assetProfileIdentifiers, - dateQuery, - skip: i, - take: CurrentRateService.MARKET_DATA_PAGE_SIZE - }); - - values.push( - ...data.map(({ dataSource, date, marketPrice, symbol }) => ({ + // Fetch each asset profile individually to use the composite index efficiently + // Process in batches of 10 to avoid overwhelming the database + const batchSize = 10; + for (let i = 0; i < assetProfileIdentifiers.length; i += batchSize) { + const batch = assetProfileIdentifiers.slice(i, i + batchSize); + const promises = batch.map(async (assetProfile) => { + const data = await this.marketDataService.getRange({ + assetProfileIdentifiers: [assetProfile], + dateQuery + }); + return data.map(({ dataSource, date, marketPrice, symbol }) => ({ dataSource, date, marketPrice, symbol - })) - ); + })); + }); + const results = await Promise.all(promises); + for (const result of results) { + values.push(...result); + } } const response: GetValuesObject = { diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts index 024bdf4e1..b06e3a77b 100644 --- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts @@ -53,16 +53,18 @@ export class ExchangeRateDataService { @LogPerformance public async getExchangeRatesByCurrency({ currencies, + dates, endDate = new Date(), startDate, targetCurrency }: { currencies: string[]; + dates?: Date[]; endDate?: Date; - startDate: Date; + startDate?: Date; targetCurrency: string; }): Promise { - if (!startDate) { + if (!startDate && !dates?.length) { return {}; } @@ -73,6 +75,8 @@ export class ExchangeRateDataService { for (const currency of currencies) { exchangeRatesByCurrency[`${currency}${targetCurrency}`] = await this.getExchangeRates({ + dates, + endDate, startDate, currencyFrom: currency, currencyTo: targetCurrency @@ -90,11 +94,14 @@ export class ExchangeRateDataService { lastDateString ] ?? 1; + const loopStartDate = + startDate || dates?.reduce((min, d) => (d < min ? d : min), dates[0]); + // Start from the most recent date and fill in missing exchange rates // using the latest available rate for ( let date = endDate; - !isBefore(date, startDate); + loopStartDate && !isBefore(date, loopStartDate); date = subDays(resetHours(date), 1) ) { const dateString = format(date, DATE_FORMAT); @@ -110,7 +117,7 @@ export class ExchangeRateDataService { previousExchangeRate; if (currency === DEFAULT_CURRENCY && isBefore(date, new Date())) { - Logger.error( + Logger.debug( `No exchange rate has been found for ${currency}${targetCurrency} at ${dateString}`, 'ExchangeRateDataService' ); @@ -355,19 +362,22 @@ export class ExchangeRateDataService { private async getExchangeRates({ currencyFrom, currencyTo, + dates, endDate = new Date(), startDate }: { currencyFrom: string; currencyTo: string; + dates?: Date[]; endDate?: Date; - startDate: Date; + startDate?: Date; }) { - const dates = eachDayOfInterval({ end: endDate, start: startDate }); + const datesToProcess = + dates ?? eachDayOfInterval({ end: endDate, start: startDate }); const factors: { [dateString: string]: number } = {}; if (currencyFrom === currencyTo) { - for (const date of dates) { + for (const date of datesToProcess) { factors[format(date, DATE_FORMAT)] = 1; } @@ -378,7 +388,7 @@ export class ExchangeRateDataService { this.derivedCurrencyFactors[`${currencyFrom}${currencyTo}`]; if (derivedCurrencyFactor) { - for (const date of dates) { + for (const date of datesToProcess) { factors[format(date, DATE_FORMAT)] = derivedCurrencyFactor; } @@ -395,7 +405,8 @@ export class ExchangeRateDataService { symbol } ], - dateQuery: { gte: startDate, lt: endDate } + dateQuery: + dates?.length > 0 ? { in: dates } : { gte: startDate, lt: endDate } }); if (marketData?.length > 0) { @@ -414,7 +425,7 @@ export class ExchangeRateDataService { try { if (currencyFrom === DEFAULT_CURRENCY) { - for (const date of dates) { + for (const date of datesToProcess) { marketPriceBaseCurrencyFromCurrency[format(date, DATE_FORMAT)] = 1; } } else { @@ -425,7 +436,10 @@ export class ExchangeRateDataService { symbol: `${DEFAULT_CURRENCY}${currencyFrom}` } ], - dateQuery: { gte: startDate, lt: endDate } + dateQuery: + dates?.length > 0 + ? { in: dates } + : { gte: startDate, lt: endDate } }); for (const { date, marketPrice } of marketData) { @@ -437,7 +451,7 @@ export class ExchangeRateDataService { try { if (currencyTo === DEFAULT_CURRENCY) { - for (const date of dates) { + for (const date of datesToProcess) { marketPriceBaseCurrencyToCurrency[format(date, DATE_FORMAT)] = 1; } } else { @@ -448,10 +462,10 @@ export class ExchangeRateDataService { symbol: `${DEFAULT_CURRENCY}${currencyTo}` } ], - dateQuery: { - gte: startDate, - lt: endDate - } + dateQuery: + dates?.length > 0 + ? { in: dates } + : { gte: startDate, lt: endDate } }); for (const { date, marketPrice } of marketData) { @@ -461,30 +475,32 @@ export class ExchangeRateDataService { } } catch {} - for (const date of dates) { - try { - const factor = - (1 / - marketPriceBaseCurrencyFromCurrency[format(date, DATE_FORMAT)]) * - marketPriceBaseCurrencyToCurrency[format(date, DATE_FORMAT)]; + for (let i = 0; i < datesToProcess.length; i++) { + if (i % 500 === 0 && process.env.NODE_ENV !== 'test') { + await new Promise((resolve) => setImmediate(resolve)); + } - if (isNaN(factor)) { - throw new Error('Exchange rate is not a number'); - } else { - factors[format(date, DATE_FORMAT)] = factor; - } - } catch { - let errorMessage = `No exchange rate has been found for ${currencyFrom}${currencyTo} at ${format( - date, - DATE_FORMAT - )}. Please complement market data for ${DEFAULT_CURRENCY}${currencyFrom}`; - - if (DEFAULT_CURRENCY !== currencyTo) { - errorMessage = `${errorMessage} and ${DEFAULT_CURRENCY}${currencyTo}`; + const date = datesToProcess[i]; + const dateString = format(date, DATE_FORMAT); + + const priceFrom = marketPriceBaseCurrencyFromCurrency[dateString]; + const priceTo = marketPriceBaseCurrencyToCurrency[dateString]; + + if (priceFrom && priceTo) { + const factor = (1 / priceFrom) * priceTo; + if (!isNaN(factor) && isFinite(factor)) { + factors[dateString] = factor; + continue; } + } - Logger.error(`${errorMessage}.`, 'ExchangeRateDataService'); + let errorMessage = `No exchange rate has been found for ${currencyFrom}${currencyTo} at ${dateString}. Please complement market data for ${DEFAULT_CURRENCY}${currencyFrom}`; + + if (DEFAULT_CURRENCY !== currencyTo) { + errorMessage = `${errorMessage} and ${DEFAULT_CURRENCY}${currencyTo}`; } + + Logger.debug(`${errorMessage}.`, 'ExchangeRateDataService'); } } diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 87b08e1bd..ffa5c650d 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -92,26 +92,6 @@ export class MarketDataService { }); } - public async getRangeCount({ - assetProfileIdentifiers, - dateQuery - }: { - assetProfileIdentifiers: AssetProfileIdentifier[]; - dateQuery: DateQuery; - }): Promise { - return this.prismaService.marketData.count({ - where: { - date: dateQuery, - OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => { - return { - dataSource, - symbol - }; - }) - } - }); - } - public async marketDataItems(params: { select?: Prisma.MarketDataSelectScalar; skip?: number; diff --git a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.processor.ts b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.processor.ts index f3aa6e77e..02cbb3734 100644 --- a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.processor.ts +++ b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.processor.ts @@ -67,6 +67,7 @@ export class PortfolioSnapshotProcessor { calculationType: job.data.calculationType, currency: job.data.userCurrency, filters: job.data.filters, + skipInitialize: true, userId: job.data.userId });