From 321602928f9216f3233c2e155f9205a7e608d4c4 Mon Sep 17 00:00:00 2001 From: Daniel Devaud Date: Tue, 14 Nov 2023 20:40:55 +0100 Subject: [PATCH 1/4] Added Chunk Todos --- apps/api/src/app/order/order.service.ts | 5 +++++ apps/api/src/services/data-provider/manual/manual.service.ts | 1 + apps/api/src/services/market-data/market-data.service.ts | 1 + .../src/services/symbol-profile/symbol-profile.service.ts | 2 ++ 4 files changed, 9 insertions(+) diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 3c228bc79..54bce79b8 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -274,6 +274,7 @@ export class OrderService { { AND: [ { + // TODO Chunk? OR: filtersByAssetClass.map(({ id }) => { return { assetClass: AssetClass[id] }; }) @@ -288,6 +289,7 @@ export class OrderService { }, { SymbolProfileOverrides: { + // TODO Chunk? OR: filtersByAssetClass.map(({ id }) => { return { assetClass: AssetClass[id] }; }) @@ -304,6 +306,7 @@ export class OrderService { { tags: { some: { + // TODO Chunk? OR: filtersByTag.map(({ id }) => { return { id: id @@ -316,6 +319,7 @@ export class OrderService { SymbolProfile: { tags: { some: { + // TODO Chunk? OR: filtersByTag.map(({ id }) => { return { id }; }) @@ -329,6 +333,7 @@ export class OrderService { } if (types) { + // TODO Chunk? where.OR = types.map((type) => { return { type: { diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 1464a526d..fed4169ca 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -161,6 +161,7 @@ export class ManualService implements DataProviderInterface { take: symbols.length, where: { symbol: { + // TODO Chunk! in: symbols } } diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 52c833784..3b75f99a3 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -74,6 +74,7 @@ export class MarketDataService { } ], where: { + //TODO Chunk! OR: uniqueAssets.map(({ dataSource, symbol }) => { return { AND: [ diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.ts index 3747b07f3..d8fe26971 100644 --- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts +++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts @@ -58,6 +58,7 @@ export class SymbolProfileService { SymbolProfileOverrides: true }, where: { + // TODO: CHUNK ! OR: aUniqueAssets.map(({ dataSource, symbol }) => { return { dataSource, @@ -83,6 +84,7 @@ export class SymbolProfileService { }, where: { id: { + //TODO CHUNK!!!! in: symbolProfileIds.map((symbolProfileId) => { return symbolProfileId; }) From 9ad0cfae17b9ffa9c51f63e17d998d93365032f0 Mon Sep 17 00:00:00 2001 From: Daniel Devaud Date: Tue, 14 Nov 2023 20:48:36 +0100 Subject: [PATCH 2/4] Added ToDo for date in queries --- apps/api/src/app/portfolio/portfolio-calculator.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/api/src/app/portfolio/portfolio-calculator.ts b/apps/api/src/app/portfolio/portfolio-calculator.ts index 626faf0c9..e2258cae5 100644 --- a/apps/api/src/app/portfolio/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/portfolio-calculator.ts @@ -215,6 +215,7 @@ export class PortfolioCalculator { await this.currentRateService.getValues({ currencies, dataGatheringItems, + // TODO Refactor in to lte & gte dateQuery: { in: dates }, @@ -402,6 +403,7 @@ export class PortfolioCalculator { } = await this.currentRateService.getValues({ currencies, dataGatheringItems, + // TODO Refactor to lte & gte dateQuery: { in: dates }, From a5e4315ea3895c90cef00dcb4624044fc990e305 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 15 Nov 2023 14:29:45 +0100 Subject: [PATCH 3/4] Fix Asset Class Editor --- apps/api/src/app/admin/admin.service.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts index 58e5469a0..a87f2ad10 100644 --- a/apps/api/src/app/admin/admin.service.ts +++ b/apps/api/src/app/admin/admin.service.ts @@ -369,13 +369,13 @@ export class AdminService { symbolProfileId }); } else { - symbolProfileId = await this.symbolProfileService.getSymbolProfiles([ + let profiles = await this.symbolProfileService.getSymbolProfiles([ { dataSource, symbol } - ])[0]; - + ]); + symbolProfileId = profiles[0].id; await this.symbolProfileOverwriteService.add({ SymbolProfile: { connect: { From 21955f3c094ab03ab6c9bdb003df15ca423e59d2 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 15 Nov 2023 14:31:20 +0100 Subject: [PATCH 4/4] Fix too many bind Parameters --- apps/api/src/app/order/order.service.ts | 5 -- .../src/app/portfolio/current-rate.service.ts | 14 +++-- .../src/app/portfolio/portfolio-calculator.ts | 2 - apps/api/src/helper/dateQueryHelper.ts | 23 +++++++ .../data-provider/manual/manual.service.ts | 33 ++++++---- .../market-data/market-data.service.ts | 62 ++++++++++++------- .../symbol-profile/symbol-profile.service.ts | 2 - libs/common/src/lib/chunkhelper.ts | 46 ++++++++++++++ 8 files changed, 137 insertions(+), 50 deletions(-) create mode 100644 apps/api/src/helper/dateQueryHelper.ts create mode 100644 libs/common/src/lib/chunkhelper.ts diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 54bce79b8..3c228bc79 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -274,7 +274,6 @@ export class OrderService { { AND: [ { - // TODO Chunk? OR: filtersByAssetClass.map(({ id }) => { return { assetClass: AssetClass[id] }; }) @@ -289,7 +288,6 @@ export class OrderService { }, { SymbolProfileOverrides: { - // TODO Chunk? OR: filtersByAssetClass.map(({ id }) => { return { assetClass: AssetClass[id] }; }) @@ -306,7 +304,6 @@ export class OrderService { { tags: { some: { - // TODO Chunk? OR: filtersByTag.map(({ id }) => { return { id: id @@ -319,7 +316,6 @@ export class OrderService { SymbolProfile: { tags: { some: { - // TODO Chunk? OR: filtersByTag.map(({ id }) => { return { id }; }) @@ -333,7 +329,6 @@ export class OrderService { } if (types) { - // TODO Chunk? where.OR = types.map((type) => { return { type: { diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index 718ec6095..3aeedff9a 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -14,9 +14,12 @@ import { flatten, isEmpty, uniqBy } from 'lodash'; import { GetValueObject } from './interfaces/get-value-object.interface'; import { GetValuesObject } from './interfaces/get-values-object.interface'; import { GetValuesParams } from './interfaces/get-values-params.interface'; +import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper'; @Injectable() export class CurrentRateService { + private dateQueryHelper = new DateQueryHelper(); + public constructor( private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, @@ -34,7 +37,7 @@ export class CurrentRateService { (!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) && (!dateQuery.gte || isBefore(dateQuery.gte, new Date())) && (!dateQuery.in || this.containsToday(dateQuery.in)); - + let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery); const promises: Promise[] = []; const quoteErrors: ResponseError['errors'] = []; const today = resetHours(new Date()); @@ -89,7 +92,7 @@ export class CurrentRateService { promises.push( this.marketDataService .getRange({ - dateQuery, + dateQuery: query, uniqueAssets }) .then((data) => { @@ -116,9 +119,12 @@ export class CurrentRateService { errors: quoteErrors.map(({ dataSource, symbol }) => { return { dataSource, symbol }; }), - values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`) + values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`).filter( + (v) => + dates?.length === 0 || + dates.some((d: Date) => d.getTime() === v.date.getTime()) + ) }; - if (!isEmpty(quoteErrors)) { for (const { dataSource, symbol } of quoteErrors) { try { diff --git a/apps/api/src/app/portfolio/portfolio-calculator.ts b/apps/api/src/app/portfolio/portfolio-calculator.ts index e2258cae5..626faf0c9 100644 --- a/apps/api/src/app/portfolio/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/portfolio-calculator.ts @@ -215,7 +215,6 @@ export class PortfolioCalculator { await this.currentRateService.getValues({ currencies, dataGatheringItems, - // TODO Refactor in to lte & gte dateQuery: { in: dates }, @@ -403,7 +402,6 @@ export class PortfolioCalculator { } = await this.currentRateService.getValues({ currencies, dataGatheringItems, - // TODO Refactor to lte & gte dateQuery: { in: dates }, diff --git a/apps/api/src/helper/dateQueryHelper.ts b/apps/api/src/helper/dateQueryHelper.ts new file mode 100644 index 000000000..2016de74a --- /dev/null +++ b/apps/api/src/helper/dateQueryHelper.ts @@ -0,0 +1,23 @@ +import { resetHours } from '@ghostfolio/common/helper'; +import { DateQuery } from '../app/portfolio/interfaces/date-query.interface'; +import { addDays } from 'date-fns'; + +export class DateQueryHelper { + public handleDateQueryIn(dateQuery: DateQuery): { + query: DateQuery; + dates: Date[]; + } { + let dates = []; + let query = dateQuery; + if (dateQuery.in?.length > 0) { + dates = dateQuery.in; + let end = Math.max(...dates.map((d) => d.getTime())); + let start = Math.min(...dates.map((d) => d.getTime())); + query = { + gte: resetHours(new Date(start)), + lt: resetHours(addDays(end, 1)) + }; + } + return { query, dates }; + } +} diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index fed4169ca..5482b52ed 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -6,6 +6,7 @@ import { } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; +import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper'; import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config'; import { DATE_FORMAT, @@ -153,19 +154,25 @@ export class ManualService implements DataProviderInterface { }) ); - const marketData = await this.prismaService.marketData.findMany({ - distinct: ['symbol'], - orderBy: { - date: 'desc' - }, - take: symbols.length, - where: { - symbol: { - // TODO Chunk! - in: symbols - } - } - }); + const batch = new BatchPrismaClient(this.prismaService); + + const marketData = await batch + .over(symbols) + .with((prisma, _symbols) => + prisma.marketData.findMany({ + distinct: ['symbol'], + orderBy: { + date: 'desc' + }, + take: symbols.length, + where: { + symbol: { + in: _symbols + } + } + }) + ) + .then((_result) => _result.flat()); for (const symbolProfile of symbolProfiles) { response[symbolProfile.symbol] = { diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 3b75f99a3..85a62837b 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -3,6 +3,7 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.i import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { resetHours } from '@ghostfolio/common/helper'; +import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { Injectable } from '@nestjs/common'; import { @@ -11,11 +12,14 @@ import { MarketDataState, Prisma } from '@prisma/client'; +import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper'; @Injectable() export class MarketDataService { public constructor(private readonly prismaService: PrismaService) {} + private dateQueryHelper = new DateQueryHelper(); + public async deleteMany({ dataSource, symbol }: UniqueAsset) { return this.prismaService.marketData.deleteMany({ where: { @@ -64,30 +68,41 @@ export class MarketDataService { dateQuery: DateQuery; uniqueAssets: UniqueAsset[]; }): Promise { - return await this.prismaService.marketData.findMany({ - orderBy: [ - { - date: 'asc' - }, - { - symbol: 'asc' - } - ], - where: { - //TODO Chunk! - OR: uniqueAssets.map(({ dataSource, symbol }) => { - return { - AND: [ - { - dataSource, - symbol, - date: dateQuery - } - ] - }; + const batch = new BatchPrismaClient(this.prismaService); + let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery); + let marketData = await batch + .over(uniqueAssets) + .with((prisma, _assets) => + prisma.marketData.findMany({ + orderBy: [ + { + date: 'asc' + }, + { + symbol: 'asc' + } + ], + where: { + OR: _assets.map(({ dataSource, symbol }) => { + return { + AND: [ + { + dataSource, + symbol, + date: query + } + ] + }; + }) + } }) - } - }); + ) + .then((data) => data.flat()); + return marketData.filter( + (m) => + dates?.length === 0 || + dates.some((d) => m.date.getTime() === d.getTime()) + ); } public async marketDataItems(params: { @@ -98,7 +113,6 @@ export class MarketDataService { orderBy?: Prisma.MarketDataOrderByWithRelationInput; }): Promise { const { skip, take, cursor, where, orderBy } = params; - return this.prismaService.marketData.findMany({ cursor, orderBy, diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.ts index d8fe26971..3747b07f3 100644 --- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts +++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts @@ -58,7 +58,6 @@ export class SymbolProfileService { SymbolProfileOverrides: true }, where: { - // TODO: CHUNK ! OR: aUniqueAssets.map(({ dataSource, symbol }) => { return { dataSource, @@ -84,7 +83,6 @@ export class SymbolProfileService { }, where: { id: { - //TODO CHUNK!!!! in: symbolProfileIds.map((symbolProfileId) => { return symbolProfileId; }) diff --git a/libs/common/src/lib/chunkhelper.ts b/libs/common/src/lib/chunkhelper.ts new file mode 100644 index 000000000..5f2929055 --- /dev/null +++ b/libs/common/src/lib/chunkhelper.ts @@ -0,0 +1,46 @@ +import { Prisma, PrismaClient } from '@prisma/client'; + +class Chunk implements Iterable { + protected constructor( + private readonly values: readonly T[], + private readonly size: number + ) {} + + *[Symbol.iterator]() { + const copy = [...this.values]; + if (copy.length === 0) yield undefined; + while (copy.length) yield copy.splice(0, this.size); + } + + map(mapper: (items?: T[]) => U): U[] { + return Array.from(this).map((items) => mapper(items)); + } + + static of(values: readonly U[]) { + return { + by: (size: number) => new Chunk(values, size) + }; + } +} + +export type Queryable = ( + p: PrismaClient, + vs?: T[] +) => Prisma.PrismaPromise; +export class BatchPrismaClient { + constructor( + private readonly prisma: PrismaClient, + private readonly size = 32_000 + ) {} + + over(values: readonly T[]) { + return { + with: (queryable: Queryable) => + this.prisma.$transaction( + Chunk.of(values) + .by(this.size) + .map((vs) => queryable(this.prisma, vs)) + ) + }; + } +}