diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts index 58e5469a0..a87f2ad10 100644 --- a/apps/api/src/app/admin/admin.service.ts +++ b/apps/api/src/app/admin/admin.service.ts @@ -369,13 +369,13 @@ export class AdminService { symbolProfileId }); } else { - symbolProfileId = await this.symbolProfileService.getSymbolProfiles([ + let profiles = await this.symbolProfileService.getSymbolProfiles([ { dataSource, symbol } - ])[0]; - + ]); + symbolProfileId = profiles[0].id; await this.symbolProfileOverwriteService.add({ SymbolProfile: { connect: { diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index 718ec6095..3aeedff9a 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -14,9 +14,12 @@ import { flatten, isEmpty, uniqBy } from 'lodash'; import { GetValueObject } from './interfaces/get-value-object.interface'; import { GetValuesObject } from './interfaces/get-values-object.interface'; import { GetValuesParams } from './interfaces/get-values-params.interface'; +import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper'; @Injectable() export class CurrentRateService { + private dateQueryHelper = new DateQueryHelper(); + public constructor( private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, @@ -34,7 +37,7 @@ export class CurrentRateService { (!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) && (!dateQuery.gte || isBefore(dateQuery.gte, new Date())) && (!dateQuery.in || this.containsToday(dateQuery.in)); - + let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery); const promises: Promise[] = []; const quoteErrors: ResponseError['errors'] = []; const today = resetHours(new Date()); @@ -89,7 +92,7 @@ export class CurrentRateService { promises.push( this.marketDataService .getRange({ - dateQuery, + dateQuery: query, uniqueAssets }) .then((data) => { @@ -116,9 +119,12 @@ export class CurrentRateService { errors: quoteErrors.map(({ dataSource, symbol }) => { return { dataSource, symbol }; }), - values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`) + values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`).filter( + (v) => + dates?.length === 0 || + dates.some((d: Date) => d.getTime() === v.date.getTime()) + ) }; - if (!isEmpty(quoteErrors)) { for (const { dataSource, symbol } of quoteErrors) { try { diff --git a/apps/api/src/helper/dateQueryHelper.ts b/apps/api/src/helper/dateQueryHelper.ts new file mode 100644 index 000000000..2016de74a --- /dev/null +++ b/apps/api/src/helper/dateQueryHelper.ts @@ -0,0 +1,23 @@ +import { resetHours } from '@ghostfolio/common/helper'; +import { DateQuery } from '../app/portfolio/interfaces/date-query.interface'; +import { addDays } from 'date-fns'; + +export class DateQueryHelper { + public handleDateQueryIn(dateQuery: DateQuery): { + query: DateQuery; + dates: Date[]; + } { + let dates = []; + let query = dateQuery; + if (dateQuery.in?.length > 0) { + dates = dateQuery.in; + let end = Math.max(...dates.map((d) => d.getTime())); + let start = Math.min(...dates.map((d) => d.getTime())); + query = { + gte: resetHours(new Date(start)), + lt: resetHours(addDays(end, 1)) + }; + } + return { query, dates }; + } +} diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 1464a526d..5482b52ed 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -6,6 +6,7 @@ import { } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; +import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper'; import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config'; import { DATE_FORMAT, @@ -153,18 +154,25 @@ export class ManualService implements DataProviderInterface { }) ); - const marketData = await this.prismaService.marketData.findMany({ - distinct: ['symbol'], - orderBy: { - date: 'desc' - }, - take: symbols.length, - where: { - symbol: { - in: symbols - } - } - }); + const batch = new BatchPrismaClient(this.prismaService); + + const marketData = await batch + .over(symbols) + .with((prisma, _symbols) => + prisma.marketData.findMany({ + distinct: ['symbol'], + orderBy: { + date: 'desc' + }, + take: symbols.length, + where: { + symbol: { + in: _symbols + } + } + }) + ) + .then((_result) => _result.flat()); for (const symbolProfile of symbolProfiles) { response[symbolProfile.symbol] = { diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 52c833784..85a62837b 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -3,6 +3,7 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.i import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { resetHours } from '@ghostfolio/common/helper'; +import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { Injectable } from '@nestjs/common'; import { @@ -11,11 +12,14 @@ import { MarketDataState, Prisma } from '@prisma/client'; +import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper'; @Injectable() export class MarketDataService { public constructor(private readonly prismaService: PrismaService) {} + private dateQueryHelper = new DateQueryHelper(); + public async deleteMany({ dataSource, symbol }: UniqueAsset) { return this.prismaService.marketData.deleteMany({ where: { @@ -64,29 +68,41 @@ export class MarketDataService { dateQuery: DateQuery; uniqueAssets: UniqueAsset[]; }): Promise { - return await this.prismaService.marketData.findMany({ - orderBy: [ - { - date: 'asc' - }, - { - symbol: 'asc' - } - ], - where: { - OR: uniqueAssets.map(({ dataSource, symbol }) => { - return { - AND: [ - { - dataSource, - symbol, - date: dateQuery - } - ] - }; + const batch = new BatchPrismaClient(this.prismaService); + let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery); + let marketData = await batch + .over(uniqueAssets) + .with((prisma, _assets) => + prisma.marketData.findMany({ + orderBy: [ + { + date: 'asc' + }, + { + symbol: 'asc' + } + ], + where: { + OR: _assets.map(({ dataSource, symbol }) => { + return { + AND: [ + { + dataSource, + symbol, + date: query + } + ] + }; + }) + } }) - } - }); + ) + .then((data) => data.flat()); + return marketData.filter( + (m) => + dates?.length === 0 || + dates.some((d) => m.date.getTime() === d.getTime()) + ); } public async marketDataItems(params: { @@ -97,7 +113,6 @@ export class MarketDataService { orderBy?: Prisma.MarketDataOrderByWithRelationInput; }): Promise { const { skip, take, cursor, where, orderBy } = params; - return this.prismaService.marketData.findMany({ cursor, orderBy, diff --git a/libs/common/src/lib/chunkhelper.ts b/libs/common/src/lib/chunkhelper.ts new file mode 100644 index 000000000..5f2929055 --- /dev/null +++ b/libs/common/src/lib/chunkhelper.ts @@ -0,0 +1,46 @@ +import { Prisma, PrismaClient } from '@prisma/client'; + +class Chunk implements Iterable { + protected constructor( + private readonly values: readonly T[], + private readonly size: number + ) {} + + *[Symbol.iterator]() { + const copy = [...this.values]; + if (copy.length === 0) yield undefined; + while (copy.length) yield copy.splice(0, this.size); + } + + map(mapper: (items?: T[]) => U): U[] { + return Array.from(this).map((items) => mapper(items)); + } + + static of(values: readonly U[]) { + return { + by: (size: number) => new Chunk(values, size) + }; + } +} + +export type Queryable = ( + p: PrismaClient, + vs?: T[] +) => Prisma.PrismaPromise; +export class BatchPrismaClient { + constructor( + private readonly prisma: PrismaClient, + private readonly size = 32_000 + ) {} + + over(values: readonly T[]) { + return { + with: (queryable: Queryable) => + this.prisma.$transaction( + Chunk.of(values) + .by(this.size) + .map((vs) => queryable(this.prisma, vs)) + ) + }; + } +}