Merge pull request #39 from dandevaud/bugfix/several-bugfixes

Bugfix/several bugfixes

apps/api/src/app/admin/admin.service.ts

@@ -369,13 +369,13 @@ export class AdminService {
           symbolProfileId
         });
       } else {
-        symbolProfileId = await this.symbolProfileService.getSymbolProfiles([
+        let profiles = await this.symbolProfileService.getSymbolProfiles([
           {
             dataSource,
             symbol
           }
-        ])[0];
+        ]);
-
+        symbolProfileId = profiles[0].id;
         await this.symbolProfileOverwriteService.add({
           SymbolProfile: {
             connect: {

apps/api/src/app/portfolio/current-rate.service.ts

@@ -14,9 +14,12 @@ import { flatten, isEmpty, uniqBy } from 'lodash';
 import { GetValueObject } from './interfaces/get-value-object.interface';
 import { GetValuesObject } from './interfaces/get-values-object.interface';
 import { GetValuesParams } from './interfaces/get-values-params.interface';
+import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper';
 
 @Injectable()
 export class CurrentRateService {
+  private dateQueryHelper = new DateQueryHelper();
+
   public constructor(
     private readonly dataProviderService: DataProviderService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
@@ -34,7 +37,7 @@ export class CurrentRateService {
       (!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) &&
       (!dateQuery.gte || isBefore(dateQuery.gte, new Date())) &&
       (!dateQuery.in || this.containsToday(dateQuery.in));
-
+    let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery);
     const promises: Promise<GetValueObject[]>[] = [];
     const quoteErrors: ResponseError['errors'] = [];
     const today = resetHours(new Date());
@@ -89,7 +92,7 @@ export class CurrentRateService {
     promises.push(
       this.marketDataService
         .getRange({
-          dateQuery,
+          dateQuery: query,
           uniqueAssets
         })
         .then((data) => {
@@ -116,9 +119,12 @@ export class CurrentRateService {
       errors: quoteErrors.map(({ dataSource, symbol }) => {
         return { dataSource, symbol };
       }),
-      values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
+      values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`).filter(
+        (v) =>
+          dates?.length === 0 ||
+          dates.some((d: Date) => d.getTime() === v.date.getTime())
+      )
     };
-
     if (!isEmpty(quoteErrors)) {
       for (const { dataSource, symbol } of quoteErrors) {
         try {

apps/api/src/helper/dateQueryHelper.ts

@@ -0,0 +1,23 @@
+import { resetHours } from '@ghostfolio/common/helper';
+import { DateQuery } from '../app/portfolio/interfaces/date-query.interface';
+import { addDays } from 'date-fns';
+
+export class DateQueryHelper {
+  public handleDateQueryIn(dateQuery: DateQuery): {
+    query: DateQuery;
+    dates: Date[];
+  } {
+    let dates = [];
+    let query = dateQuery;
+    if (dateQuery.in?.length > 0) {
+      dates = dateQuery.in;
+      let end = Math.max(...dates.map((d) => d.getTime()));
+      let start = Math.min(...dates.map((d) => d.getTime()));
+      query = {
+        gte: resetHours(new Date(start)),
+        lt: resetHours(addDays(end, 1))
+      };
+    }
+    return { query, dates };
+  }
+}

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -6,6 +6,7 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
+import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper';
 import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
 import {
   DATE_FORMAT,
@@ -153,18 +154,25 @@ export class ManualService implements DataProviderInterface {
         })
       );
 
-      const marketData = await this.prismaService.marketData.findMany({
+      const batch = new BatchPrismaClient(this.prismaService);
-        distinct: ['symbol'],
+
-        orderBy: {
+      const marketData = await batch
-          date: 'desc'
+        .over(symbols)
-        },
+        .with((prisma, _symbols) =>
-        take: symbols.length,
+          prisma.marketData.findMany({
-        where: {
+            distinct: ['symbol'],
-          symbol: {
+            orderBy: {
-            in: symbols
+              date: 'desc'
-          }
+            },
-        }
+            take: symbols.length,
-      });
+            where: {
+              symbol: {
+                in: _symbols
+              }
+            }
+          })
+        )
+        .then((_result) => _result.flat());
 
       for (const symbolProfile of symbolProfiles) {
         response[symbolProfile.symbol] = {

apps/api/src/services/market-data/market-data.service.ts

@@ -3,6 +3,7 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.i
 import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { resetHours } from '@ghostfolio/common/helper';
+import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { Injectable } from '@nestjs/common';
 import {
@@ -11,11 +12,14 @@ import {
   MarketDataState,
   Prisma
 } from '@prisma/client';
+import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper';
 
 @Injectable()
 export class MarketDataService {
   public constructor(private readonly prismaService: PrismaService) {}
 
+  private dateQueryHelper = new DateQueryHelper();
+
   public async deleteMany({ dataSource, symbol }: UniqueAsset) {
     return this.prismaService.marketData.deleteMany({
       where: {
@@ -64,29 +68,41 @@ export class MarketDataService {
     dateQuery: DateQuery;
     uniqueAssets: UniqueAsset[];
   }): Promise<MarketData[]> {
-    return await this.prismaService.marketData.findMany({
+    const batch = new BatchPrismaClient(this.prismaService);
-      orderBy: [
+    let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery);
-        {
+    let marketData = await batch
-          date: 'asc'
+      .over(uniqueAssets)
-        },
+      .with((prisma, _assets) =>
-        {
+        prisma.marketData.findMany({
-          symbol: 'asc'
+          orderBy: [
-        }
+            {
-      ],
+              date: 'asc'
-      where: {
+            },
-        OR: uniqueAssets.map(({ dataSource, symbol }) => {
+            {
-          return {
+              symbol: 'asc'
-            AND: [
+            }
-              {
+          ],
-                dataSource,
+          where: {
-                symbol,
+            OR: _assets.map(({ dataSource, symbol }) => {
-                date: dateQuery
+              return {
-              }
+                AND: [
-            ]
+                  {
-          };
+                    dataSource,
+                    symbol,
+                    date: query
+                  }
+                ]
+              };
+            })
+          }
         })
-      }
+      )
-    });
+      .then((data) => data.flat());
+    return marketData.filter(
+      (m) =>
+        dates?.length === 0 ||
+        dates.some((d) => m.date.getTime() === d.getTime())
+    );
   }
 
   public async marketDataItems(params: {
@@ -97,7 +113,6 @@ export class MarketDataService {
     orderBy?: Prisma.MarketDataOrderByWithRelationInput;
   }): Promise<MarketData[]> {
     const { skip, take, cursor, where, orderBy } = params;
-
     return this.prismaService.marketData.findMany({
       cursor,
       orderBy,

libs/common/src/lib/chunkhelper.ts

@@ -0,0 +1,46 @@
+import { Prisma, PrismaClient } from '@prisma/client';
+
+class Chunk<T> implements Iterable<T[] | undefined> {
+  protected constructor(
+    private readonly values: readonly T[],
+    private readonly size: number
+  ) {}
+
+  *[Symbol.iterator]() {
+    const copy = [...this.values];
+    if (copy.length === 0) yield undefined;
+    while (copy.length) yield copy.splice(0, this.size);
+  }
+
+  map<U>(mapper: (items?: T[]) => U): U[] {
+    return Array.from(this).map((items) => mapper(items));
+  }
+
+  static of<U>(values: readonly U[]) {
+    return {
+      by: (size: number) => new Chunk(values, size)
+    };
+  }
+}
+
+export type Queryable<T, Result> = (
+  p: PrismaClient,
+  vs?: T[]
+) => Prisma.PrismaPromise<Result>;
+export class BatchPrismaClient {
+  constructor(
+    private readonly prisma: PrismaClient,
+    private readonly size = 32_000
+  ) {}
+
+  over<T>(values: readonly T[]) {
+    return {
+      with: <Result>(queryable: Queryable<T, Result>) =>
+        this.prisma.$transaction(
+          Chunk.of(values)
+            .by(this.size)
+            .map((vs) => queryable(this.prisma, vs))
+        )
+    };
+  }
+}