Fix too many bind Parameters

apps/api/src/app/order/order.service.ts

@@ -274,7 +274,6 @@ export class OrderService {
           {
             AND: [
               {
-                // TODO Chunk?
                 OR: filtersByAssetClass.map(({ id }) => {
                   return { assetClass: AssetClass[id] };
                 })
@@ -289,7 +288,6 @@ export class OrderService {
           },
           {
             SymbolProfileOverrides: {
-              // TODO Chunk?
               OR: filtersByAssetClass.map(({ id }) => {
                 return { assetClass: AssetClass[id] };
               })
@@ -306,7 +304,6 @@ export class OrderService {
             {
               tags: {
                 some: {
-                  // TODO Chunk?
                   OR: filtersByTag.map(({ id }) => {
                     return {
                       id: id
@@ -319,7 +316,6 @@ export class OrderService {
               SymbolProfile: {
                 tags: {
                   some: {
-                    // TODO Chunk?
                     OR: filtersByTag.map(({ id }) => {
                       return { id };
                     })
@@ -333,7 +329,6 @@ export class OrderService {
     }
 
     if (types) {
-      // TODO Chunk?
       where.OR = types.map((type) => {
         return {
           type: {

apps/api/src/app/portfolio/current-rate.service.ts

@@ -14,9 +14,12 @@ import { flatten, isEmpty, uniqBy } from 'lodash';
 import { GetValueObject } from './interfaces/get-value-object.interface';
 import { GetValuesObject } from './interfaces/get-values-object.interface';
 import { GetValuesParams } from './interfaces/get-values-params.interface';
+import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper';
 
 @Injectable()
 export class CurrentRateService {
+  private dateQueryHelper = new DateQueryHelper();
+
   public constructor(
     private readonly dataProviderService: DataProviderService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
@@ -34,7 +37,7 @@ export class CurrentRateService {
       (!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) &&
       (!dateQuery.gte || isBefore(dateQuery.gte, new Date())) &&
       (!dateQuery.in || this.containsToday(dateQuery.in));
-
+    let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery);
     const promises: Promise<GetValueObject[]>[] = [];
     const quoteErrors: ResponseError['errors'] = [];
     const today = resetHours(new Date());
@@ -89,7 +92,7 @@ export class CurrentRateService {
     promises.push(
       this.marketDataService
         .getRange({
-          dateQuery,
+          dateQuery: query,
           uniqueAssets
         })
         .then((data) => {
@@ -116,9 +119,12 @@ export class CurrentRateService {
       errors: quoteErrors.map(({ dataSource, symbol }) => {
         return { dataSource, symbol };
       }),
-      values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
+      values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`).filter(
+        (v) =>
+          dates?.length === 0 ||
+          dates.some((d: Date) => d.getTime() === v.date.getTime())
+      )
     };
-
     if (!isEmpty(quoteErrors)) {
       for (const { dataSource, symbol } of quoteErrors) {
         try {

apps/api/src/app/portfolio/portfolio-calculator.ts

@@ -215,7 +215,6 @@ export class PortfolioCalculator {
       await this.currentRateService.getValues({
         currencies,
         dataGatheringItems,
-        // TODO Refactor in to lte & gte
         dateQuery: {
           in: dates
         },
@@ -403,7 +402,6 @@ export class PortfolioCalculator {
     } = await this.currentRateService.getValues({
       currencies,
       dataGatheringItems,
-      // TODO Refactor to lte & gte
       dateQuery: {
         in: dates
       },

apps/api/src/helper/dateQueryHelper.ts

@@ -0,0 +1,23 @@
+import { resetHours } from '@ghostfolio/common/helper';
+import { DateQuery } from '../app/portfolio/interfaces/date-query.interface';
+import { addDays } from 'date-fns';
+
+export class DateQueryHelper {
+  public handleDateQueryIn(dateQuery: DateQuery): {
+    query: DateQuery;
+    dates: Date[];
+  } {
+    let dates = [];
+    let query = dateQuery;
+    if (dateQuery.in?.length > 0) {
+      dates = dateQuery.in;
+      let end = Math.max(...dates.map((d) => d.getTime()));
+      let start = Math.min(...dates.map((d) => d.getTime()));
+      query = {
+        gte: resetHours(new Date(start)),
+        lt: resetHours(addDays(end, 1))
+      };
+    }
+    return { query, dates };
+  }
+}

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -6,6 +6,7 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
+import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper';
 import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
 import {
   DATE_FORMAT,
@@ -153,19 +154,25 @@ export class ManualService implements DataProviderInterface {
         })
       );
 
-      const marketData = await this.prismaService.marketData.findMany({
-        distinct: ['symbol'],
-        orderBy: {
-          date: 'desc'
-        },
-        take: symbols.length,
-        where: {
-          symbol: {
-            // TODO Chunk!
-            in: symbols
-          }
-        }
-      });
+      const batch = new BatchPrismaClient(this.prismaService);
+
+      const marketData = await batch
+        .over(symbols)
+        .with((prisma, _symbols) =>
+          prisma.marketData.findMany({
+            distinct: ['symbol'],
+            orderBy: {
+              date: 'desc'
+            },
+            take: symbols.length,
+            where: {
+              symbol: {
+                in: _symbols
+              }
+            }
+          })
+        )
+        .then((_result) => _result.flat());
 
       for (const symbolProfile of symbolProfiles) {
         response[symbolProfile.symbol] = {

apps/api/src/services/market-data/market-data.service.ts

@@ -3,6 +3,7 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.i
 import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { resetHours } from '@ghostfolio/common/helper';
+import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { Injectable } from '@nestjs/common';
 import {
@@ -11,11 +12,14 @@ import {
   MarketDataState,
   Prisma
 } from '@prisma/client';
+import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper';
 
 @Injectable()
 export class MarketDataService {
   public constructor(private readonly prismaService: PrismaService) {}
 
+  private dateQueryHelper = new DateQueryHelper();
+
   public async deleteMany({ dataSource, symbol }: UniqueAsset) {
     return this.prismaService.marketData.deleteMany({
       where: {
@@ -64,30 +68,41 @@ export class MarketDataService {
     dateQuery: DateQuery;
     uniqueAssets: UniqueAsset[];
   }): Promise<MarketData[]> {
-    return await this.prismaService.marketData.findMany({
-      orderBy: [
-        {
-          date: 'asc'
-        },
-        {
-          symbol: 'asc'
-        }
-      ],
-      where: {
-        //TODO Chunk!
-        OR: uniqueAssets.map(({ dataSource, symbol }) => {
-          return {
-            AND: [
-              {
-                dataSource,
-                symbol,
-                date: dateQuery
-              }
-            ]
-          };
+    const batch = new BatchPrismaClient(this.prismaService);
+    let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery);
+    let marketData = await batch
+      .over(uniqueAssets)
+      .with((prisma, _assets) =>
+        prisma.marketData.findMany({
+          orderBy: [
+            {
+              date: 'asc'
+            },
+            {
+              symbol: 'asc'
+            }
+          ],
+          where: {
+            OR: _assets.map(({ dataSource, symbol }) => {
+              return {
+                AND: [
+                  {
+                    dataSource,
+                    symbol,
+                    date: query
+                  }
+                ]
+              };
+            })
+          }
         })
-      }
-    });
+      )
+      .then((data) => data.flat());
+    return marketData.filter(
+      (m) =>
+        dates?.length === 0 ||
+        dates.some((d) => m.date.getTime() === d.getTime())
+    );
   }
 
   public async marketDataItems(params: {
@@ -98,7 +113,6 @@ export class MarketDataService {
     orderBy?: Prisma.MarketDataOrderByWithRelationInput;
   }): Promise<MarketData[]> {
     const { skip, take, cursor, where, orderBy } = params;
-
     return this.prismaService.marketData.findMany({
       cursor,
       orderBy,

apps/api/src/services/symbol-profile/symbol-profile.service.ts

@@ -58,7 +58,6 @@ export class SymbolProfileService {
           SymbolProfileOverrides: true
         },
         where: {
-          // TODO: CHUNK !
           OR: aUniqueAssets.map(({ dataSource, symbol }) => {
             return {
               dataSource,
@@ -84,7 +83,6 @@ export class SymbolProfileService {
         },
         where: {
           id: {
-            //TODO CHUNK!!!!
             in: symbolProfileIds.map((symbolProfileId) => {
               return symbolProfileId;
             })

libs/common/src/lib/chunkhelper.ts

@@ -0,0 +1,46 @@
+import { Prisma, PrismaClient } from '@prisma/client';
+
+class Chunk<T> implements Iterable<T[] | undefined> {
+  protected constructor(
+    private readonly values: readonly T[],
+    private readonly size: number
+  ) {}
+
+  *[Symbol.iterator]() {
+    const copy = [...this.values];
+    if (copy.length === 0) yield undefined;
+    while (copy.length) yield copy.splice(0, this.size);
+  }
+
+  map<U>(mapper: (items?: T[]) => U): U[] {
+    return Array.from(this).map((items) => mapper(items));
+  }
+
+  static of<U>(values: readonly U[]) {
+    return {
+      by: (size: number) => new Chunk(values, size)
+    };
+  }
+}
+
+export type Queryable<T, Result> = (
+  p: PrismaClient,
+  vs?: T[]
+) => Prisma.PrismaPromise<Result>;
+export class BatchPrismaClient {
+  constructor(
+    private readonly prisma: PrismaClient,
+    private readonly size = 32_000
+  ) {}
+
+  over<T>(values: readonly T[]) {
+    return {
+      with: <Result>(queryable: Queryable<T, Result>) =>
+        this.prisma.$transaction(
+          Chunk.of(values)
+            .by(this.size)
+            .map((vs) => queryable(this.prisma, vs))
+        )
+    };
+  }
+}