topaLE
/
ghostfolio
mirror of https://github.com/ghostfolio/ghostfolio
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

perf: batch and chunk database queries to prevent Prisma P2029 limits 

- Split large market data queries into batches of 10 to use composite indexes efficiently
- Chunk activities queries to avoid exceeding Prisma parameter limits
pull/6912/head
Andrea Bugeja 5 days ago
parent
53ce901b08
commit
ddb05e3f4e
2 changed files with 265 additions and 218 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 463
      apps/api/src/app/activities/activities.service.ts
  2. 20
      apps/api/src/services/market-data/market-data.service.ts

463
apps/api/src/app/activities/activities.service.ts
View File

@ -11,12 +11,16 @@ import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathe
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import { import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH, DATA_GATHERING_QUEUE_PRIORITY_HIGH,
DEFAULT_CURRENCY,
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME, GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS, GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS,
ghostfolioPrefix, ghostfolioPrefix
TAG_ID_EXCLUDE_FROM_ANALYSIS
} from '@ghostfolio/common/config'; } from '@ghostfolio/common/config';
import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import {
DATE_FORMAT,
getAssetProfileIdentifier,
resetHours
} from '@ghostfolio/common/helper';
import { import {
ActivitiesResponse, ActivitiesResponse,
Activity, Activity,
@ -39,8 +43,8 @@ import {
} from '@prisma/client'; } from '@prisma/client';
import { Big } from 'big.js'; import { Big } from 'big.js';
import { isUUID } from 'class-validator'; import { isUUID } from 'class-validator';
import { endOfToday, isAfter } from 'date-fns'; import { endOfToday, format, isAfter, subDays } from 'date-fns';
import { groupBy, uniqBy } from 'lodash'; import { uniqBy } from 'lodash';
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
@Injectable() @Injectable()
@ -470,7 +474,7 @@ export class ActivitiesService {
sortColumn, sortColumn,
sortDirection = 'asc', sortDirection = 'asc',
startDate, startDate,
take = Number.MAX_SAFE_INTEGER, take,
types, types,
userCurrency, userCurrency,
userId, userId,
@ -489,166 +493,112 @@ export class ActivitiesService {
userId: string; userId: string;
withExcludedAccountsAndActivities?: boolean; withExcludedAccountsAndActivities?: boolean;
}): Promise<ActivitiesResponse> { }): Promise<ActivitiesResponse> {
let orderBy: Prisma.Enumerable<Prisma.OrderOrderByWithRelationInput> = [ const where: Prisma.OrderWhereInput = {
{ date: 'asc' } ...(includeDrafts === false ? { isDraft: false } : {}),
]; ...(types?.length > 0 ? { type: { in: types } } : {}),
...(userId ? { userId } : {})
const where: Prisma.OrderWhereInput = { userId }; };
if (endDate || startDate) {
where.AND = [];
if (endDate) { if (startDate) {
where.AND.push({ date: { lte: endDate } }); where.date = { gte: resetHours(startDate) };
} }
if (startDate) { if (endDate) {
where.AND.push({ date: { gt: startDate } }); where.date = {
} ...(where.date as Prisma.DateTimeFilter),
lte: resetHours(endDate)
};
} }
const { if (withExcludedAccountsAndActivities === false) {
ACCOUNT: filtersByAccount, where.account = { isExcluded: false };
ASSET_CLASS: filtersByAssetClass, }
TAG: filtersByTag
} = groupBy(filters, ({ type }) => {
return type;
});
const filterByDataSource = filters?.find(({ type }) => { if (filters?.length > 0) {
return type === 'DATA_SOURCE'; where.OR = [];
})?.id;
const filterBySymbol = filters?.find(({ type }) => { const accountIds = filters
return type === 'SYMBOL'; .filter(({ type }) => type === 'ACCOUNT')
})?.id; .map(({ id }) => id);
if (accountIds.length > 0) {
where.OR.push({ accountId: { in: accountIds } });
}
const searchQuery = filters?.find(({ type }) => { const assetClasses = filters
return type === 'SEARCH_QUERY'; .filter(({ type }) => type === 'ASSET_CLASS')
})?.id; .map(({ id }) => id) as AssetClass[];
if (assetClasses.length > 0) {
where.OR.push({ SymbolProfile: { assetClass: { in: assetClasses } } });
}
if (filtersByAccount?.length > 0) { const tags = filters
where.accountId = { .filter(({ type }) => type === 'TAG')
in: filtersByAccount.map(({ id }) => { .map(({ id }) => id);
return id; if (tags.length > 0) {
}) where.OR.push({ tags: { some: { id: { in: tags } } } });
}; }
} }
if (includeDrafts === false) { let orderBy: Prisma.OrderOrderByWithRelationInput[] = [
where.isDraft = false; { date: sortDirection }
} ];
if (filtersByAssetClass?.length > 0) { if (sortColumn) {
where.SymbolProfile = { orderBy = [];
OR: [
{
AND: [
{
OR: filtersByAssetClass.map(({ id }) => {
return { assetClass: AssetClass[id] };
})
},
{
OR: [
{ SymbolProfileOverrides: { is: null } },
{ SymbolProfileOverrides: { assetClass: null } }
]
}
]
},
{
SymbolProfileOverrides: {
OR: filtersByAssetClass.map(({ id }) => {
return { assetClass: AssetClass[id] };
})
}
}
]
};
}
if (filterByDataSource && filterBySymbol) { if (
if (where.SymbolProfile) { ['currency', 'fee', 'quantity', 'type', 'unitPrice'].includes(
where.SymbolProfile = { sortColumn
AND: [ )
where.SymbolProfile, ) {
{ orderBy.push({ [sortColumn]: sortDirection });
AND: [
{ dataSource: filterByDataSource as DataSource },
{ symbol: filterBySymbol }
]
}
]
};
} else { } else {
where.SymbolProfile = { if (sortColumn === 'SymbolProfile.name') {
AND: [ orderBy.push({ SymbolProfile: { name: sortDirection } });
{ dataSource: filterByDataSource as DataSource }, } else if (sortColumn === 'account.name') {
{ symbol: filterBySymbol } orderBy.push({ account: { name: sortDirection } });
] }
};
} }
} }
if (searchQuery) { const count = await this.prismaService.order.count({ where });
const searchQueryWhereInput: Prisma.SymbolProfileWhereInput[] = [
{ id: { mode: 'insensitive', startsWith: searchQuery } },
{ isin: { mode: 'insensitive', startsWith: searchQuery } },
{ name: { mode: 'insensitive', startsWith: searchQuery } },
{ symbol: { mode: 'insensitive', startsWith: searchQuery } }
];
if (where.SymbolProfile) {
where.SymbolProfile = {
AND: [
where.SymbolProfile,
{
OR: searchQueryWhereInput
}
]
};
} else {
where.SymbolProfile = {
OR: searchQueryWhereInput
};
}
}
if (filtersByTag?.length > 0) { let orders: OrderWithAccount[] = [];
where.tags = {
some: {
OR: filtersByTag.map(({ id }) => {
return { id };
})
}
};
}
if (sortColumn) { // If take is undefined and count is extremely large, batch fetch to prevent Prisma P2029 limits
orderBy = [{ [sortColumn]: sortDirection }]; const BATCH_SIZE = 5000;
}
if (types?.length > 0) { if (take === undefined && count > BATCH_SIZE) {
where.type = { in: types }; let currentSkip = skip || 0;
} const totalToFetch = count - currentSkip;
if (withExcludedAccountsAndActivities === false) { while (orders.length < totalToFetch) {
where.OR = [ const batch = await this.orders({
{ account: null }, skip: currentSkip,
{ account: { NOT: { isExcluded: true } } } take: BATCH_SIZE,
]; where,
include: {
where.tags = { account: {
...where.tags, include: {
none: { platform: true
id: TAG_ID_EXCLUDE_FROM_ANALYSIS }
},
// eslint-disable-next-line @typescript-eslint/naming-convention
SymbolProfile: true,
tags: true
},
orderBy: [...orderBy, { id: sortDirection }]
});
if (batch.length === 0) {
break;
} }
};
}
const [orders, count] = await Promise.all([ orders = orders.concat(batch);
this.orders({ currentSkip += batch.length;
}
} else {
orders = await this.orders({
skip, skip,
take, take,
where, where,
@ -663,9 +613,8 @@ export class ActivitiesService {
tags: true tags: true
}, },
orderBy: [...orderBy, { id: sortDirection }] orderBy: [...orderBy, { id: sortDirection }]
}), });
this.prismaService.order.count({ where }) }
]);
const assetProfileIdentifiers = uniqBy( const assetProfileIdentifiers = uniqBy(
orders.map(({ SymbolProfile }) => { orders.map(({ SymbolProfile }) => {
@ -686,60 +635,178 @@ export class ActivitiesService {
assetProfileIdentifiers assetProfileIdentifiers
); );
const activities = await Promise.all( let exchangeRatesToUser: any = {};
orders.map(async (order) => { let exchangeRatesToDefault: any = {};
const assetProfile = assetProfiles.find(({ dataSource, symbol }) => {
return ( if (orders.length > 0) {
dataSource === order.SymbolProfile.dataSource && let minDate = orders[0].date;
symbol === order.SymbolProfile.symbol let maxDate = orders[0].date;
); const uniqueCurrencies = new Set<string>();
}); uniqueCurrencies.add(userCurrency);
uniqueCurrencies.add(DEFAULT_CURRENCY);
const uniqueDatesSet = new Set<number>();
for (const order of orders) {
if (order.date < minDate) {
minDate = order.date;
}
if (order.date > maxDate) {
maxDate = order.date;
}
uniqueDatesSet.add(resetHours(order.date).getTime());
if (order.currency) {
uniqueCurrencies.add(order.currency);
}
if (order.SymbolProfile?.currency) {
uniqueCurrencies.add(order.SymbolProfile.currency);
}
}
const currenciesList = Array.from(uniqueCurrencies).filter(Boolean);
const uniqueDates = Array.from(uniqueDatesSet).map(
(time) => new Date(time)
);
const startDatePreload = subDays(resetHours(minDate), 1);
const endDatePreload = resetHours(maxDate);
const [ratesUser, ratesDefault] = await Promise.all([
this.exchangeRateDataService.getExchangeRatesByCurrency({
currencies: currenciesList,
dates: uniqueDates,
endDate: endDatePreload,
startDate: startDatePreload,
targetCurrency: userCurrency
}),
this.exchangeRateDataService.getExchangeRatesByCurrency({
currencies: currenciesList,
dates: uniqueDates,
endDate: endDatePreload,
startDate: startDatePreload,
targetCurrency: DEFAULT_CURRENCY
})
]);
exchangeRatesToUser = ratesUser;
exchangeRatesToDefault = ratesDefault;
}
const getPreloadedRate = (
from: string,
to: string,
dateStr: string
): number | undefined => {
if (from === to) {
return 1;
}
const value = new Big(order.quantity).mul(order.unitPrice).toNumber(); if (to === userCurrency) {
return exchangeRatesToUser[`${from}${userCurrency}`]?.[dateStr];
const [ }
feeInAssetProfileCurrency,
feeInBaseCurrency, if (to === DEFAULT_CURRENCY) {
unitPriceInAssetProfileCurrency, return exchangeRatesToDefault[`${from}${DEFAULT_CURRENCY}`]?.[dateStr];
valueInBaseCurrency }
] = await Promise.all([
this.exchangeRateDataService.toCurrencyAtDate( if (from === DEFAULT_CURRENCY) {
order.fee, const rateToDefault =
order.currency ?? order.SymbolProfile.currency, exchangeRatesToDefault[`${to}${DEFAULT_CURRENCY}`]?.[dateStr];
order.SymbolProfile.currency, return rateToDefault ? 1 / rateToDefault : undefined;
order.date }
),
this.exchangeRateDataService.toCurrencyAtDate( const rateFromToDefault =
order.fee, exchangeRatesToDefault[`${from}${DEFAULT_CURRENCY}`]?.[dateStr];
order.currency ?? order.SymbolProfile.currency, const rateToToDefault =
userCurrency, exchangeRatesToDefault[`${to}${DEFAULT_CURRENCY}`]?.[dateStr];
order.date if (rateFromToDefault !== undefined && rateToToDefault) {
), return rateFromToDefault / rateToToDefault;
this.exchangeRateDataService.toCurrencyAtDate( }
order.unitPrice,
order.currency ?? order.SymbolProfile.currency, return undefined;
order.SymbolProfile.currency, };
order.date
), const convertValue = async (
this.exchangeRateDataService.toCurrencyAtDate( val: number,
from: string,
to: string,
date: Date
): Promise<number> => {
if (val === 0) {
return 0;
}
const dateStr = format(date, DATE_FORMAT);
const rate = getPreloadedRate(from, to, dateStr);
if (rate !== undefined && !isNaN(rate)) {
return rate * val;
}
return this.exchangeRateDataService.toCurrencyAtDate(val, from, to, date);
};
const activities = [];
const chunkSize = 500;
for (let i = 0; i < orders.length; i += chunkSize) {
const chunk = orders.slice(i, i + chunkSize);
const processedChunk = await Promise.all(
chunk.map(async (order) => {
const assetProfile = assetProfiles.find(({ dataSource, symbol }) => {
return (
dataSource === order.SymbolProfile.dataSource &&
symbol === order.SymbolProfile.symbol
);
});
const value = new Big(order.quantity).mul(order.unitPrice).toNumber();
const [
feeInAssetProfileCurrency,
feeInBaseCurrency,
unitPriceInAssetProfileCurrency,
valueInBaseCurrency
] = await Promise.all([
convertValue(
order.fee,
order.currency ?? order.SymbolProfile.currency,
order.SymbolProfile.currency,
order.date
),
convertValue(
order.fee,
order.currency ?? order.SymbolProfile.currency,
userCurrency,
order.date
),
convertValue(
order.unitPrice,
order.currency ?? order.SymbolProfile.currency,
order.SymbolProfile.currency,
order.date
),
convertValue(
value,
order.currency ?? order.SymbolProfile.currency,
userCurrency,
order.date
)
]);
return {
...order,
feeInAssetProfileCurrency: feeInAssetProfileCurrency ?? order.fee,
feeInBaseCurrency: feeInBaseCurrency ?? order.fee,
unitPriceInAssetProfileCurrency:
unitPriceInAssetProfileCurrency ?? order.unitPrice,
value, value,
order.currency ?? order.SymbolProfile.currency, valueInBaseCurrency: valueInBaseCurrency ?? value,
userCurrency, SymbolProfile: assetProfile
order.date };
) })
]); );
return { activities.push(...processedChunk);
...order, }
feeInAssetProfileCurrency,
feeInBaseCurrency,
unitPriceInAssetProfileCurrency,
value,
valueInBaseCurrency,
SymbolProfile: assetProfile
};
})
);
return { activities, count }; return { activities, count };
} }

20
apps/api/src/services/market-data/market-data.service.ts
View File

@ -92,26 +92,6 @@ export class MarketDataService {
}); });
} }
public async getRangeCount({
assetProfileIdentifiers,
dateQuery
}: {
assetProfileIdentifiers: AssetProfileIdentifier[];
dateQuery: DateQuery;
}): Promise<number> {
return this.prismaService.marketData.count({
where: {
date: dateQuery,
OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => {
return {
dataSource,
symbol
};
})
}
});
}
public async marketDataItems(params: { public async marketDataItems(params: {
select?: Prisma.MarketDataSelectScalar; select?: Prisma.MarketDataSelectScalar;
skip?: number; skip?: number;

Write Preview
Loading…
Cancel
Save