From e112a6ed9d6a85afe4a8e102a74f258f5b11cd88 Mon Sep 17 00:00:00 2001
From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com>
Date: Sun, 13 Apr 2025 08:17:36 +0200
Subject: [PATCH] Introduce market data page size

---
 .../src/app/portfolio/current-rate.service.ts | 107 +++++++++---------
 .../market-data/market-data.service.ts        |  28 ++++-
 2 files changed, 83 insertions(+), 52 deletions(-)

diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts
index 058bf1dd5..1f60ca16f 100644
--- a/apps/api/src/app/portfolio/current-rate.service.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.ts
@@ -21,6 +21,8 @@ import { GetValuesParams } from './interfaces/get-values-params.interface';
 
 @Injectable()
 export class CurrentRateService {
+  private static readonly MARKET_DATA_PAGE_SIZE = 50000;
+
   public constructor(
     private readonly dataProviderService: DataProviderService,
     private readonly marketDataService: MarketDataService,
@@ -41,42 +43,37 @@ export class CurrentRateService {
       (!dateQuery.gte || isBefore(dateQuery.gte, new Date())) &&
       (!dateQuery.in || this.containsToday(dateQuery.in));
 
-    const promises: Promise<GetValueObject[]>[] = [];
     const quoteErrors: ResponseError['errors'] = [];
     const today = resetHours(new Date());
+    const values: GetValueObject[] = [];
 
     if (includesToday) {
-      promises.push(
-        this.dataProviderService
-          .getQuotes({ items: dataGatheringItems, user: this.request?.user })
-          .then((dataResultProvider) => {
-            const result: GetValueObject[] = [];
-
-            for (const { dataSource, symbol } of dataGatheringItems) {
-              if (dataResultProvider?.[symbol]?.dataProviderInfo) {
-                dataProviderInfos.push(
-                  dataResultProvider[symbol].dataProviderInfo
-                );
-              }
+      const quotesBySymbol = await this.dataProviderService.getQuotes({
+        items: dataGatheringItems,
+        user: this.request?.user
+      });
 
-              if (dataResultProvider?.[symbol]?.marketPrice) {
-                result.push({
-                  dataSource,
-                  symbol,
-                  date: today,
-                  marketPrice: dataResultProvider?.[symbol]?.marketPrice
-                });
-              } else {
-                quoteErrors.push({
-                  dataSource,
-                  symbol
-                });
-              }
-            }
+      for (const { dataSource, symbol } of dataGatheringItems) {
+        const quote = quotesBySymbol[symbol];
 
-            return result;
-          })
-      );
+        if (quote?.dataProviderInfo) {
+          dataProviderInfos.push(quote.dataProviderInfo);
+        }
+
+        if (quote?.marketPrice) {
+          values.push({
+            dataSource,
+            symbol,
+            date: today,
+            marketPrice: quote.marketPrice
+          });
+        } else {
+          quoteErrors.push({
+            dataSource,
+            symbol
+          });
+        }
+      }
     }
 
     const assetProfileIdentifiers: AssetProfileIdentifier[] =
@@ -84,34 +81,42 @@ export class CurrentRateService {
         return { dataSource, symbol };
       });
 
-    promises.push(
-      this.marketDataService
-        .getRange({
-          assetProfileIdentifiers,
-          dateQuery
-        })
-        .then((data) => {
-          return data.map(({ dataSource, date, marketPrice, symbol }) => {
-            return {
-              dataSource,
-              date,
-              marketPrice,
-              symbol
-            };
-          });
-        })
-    );
-
-    const values = await Promise.all(promises).then((array) => {
-      return array.flat();
+    const marketDataCount = await this.marketDataService.getRangeCount({
+      assetProfileIdentifiers,
+      dateQuery
     });
 
+    for (
+      let i = 0;
+      i < marketDataCount;
+      i += CurrentRateService.MARKET_DATA_PAGE_SIZE
+    ) {
+      // Use pageSize to limit the number of records fetched at once
+      const data = await this.marketDataService.getRange({
+        assetProfileIdentifiers,
+        dateQuery,
+        skip: i,
+        take: CurrentRateService.MARKET_DATA_PAGE_SIZE
+      });
+
+      values.push(
+        ...data.map(({ dataSource, date, marketPrice, symbol }) => ({
+          dataSource,
+          date,
+          marketPrice,
+          symbol
+        }))
+      );
+    }
+
     const response: GetValuesObject = {
       dataProviderInfos,
       errors: quoteErrors.map(({ dataSource, symbol }) => {
         return { dataSource, symbol };
       }),
-      values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
+      values: uniqBy(values, ({ date, symbol }) => {
+        return `${date}-${symbol}`;
+      })
     };
 
     if (!isEmpty(quoteErrors)) {
diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts
index 0f658ced3..58b9b09ec 100644
--- a/apps/api/src/services/market-data/market-data.service.ts
+++ b/apps/api/src/services/market-data/market-data.service.ts
@@ -60,12 +60,18 @@ export class MarketDataService {
 
   public async getRange({
     assetProfileIdentifiers,
-    dateQuery
+    dateQuery,
+    skip,
+    take
   }: {
     assetProfileIdentifiers: AssetProfileIdentifier[];
     dateQuery: DateQuery;
+    skip?: number;
+    take?: number;
   }): Promise<MarketData[]> {
     return this.prismaService.marketData.findMany({
+      skip,
+      take,
       orderBy: [
         {
           date: 'asc'
@@ -86,6 +92,26 @@ export class MarketDataService {
     });
   }
 
+  public async getRangeCount({
+    assetProfileIdentifiers,
+    dateQuery
+  }: {
+    assetProfileIdentifiers: AssetProfileIdentifier[];
+    dateQuery: DateQuery;
+  }): Promise<number> {
+    return this.prismaService.marketData.count({
+      where: {
+        date: dateQuery,
+        OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => {
+          return {
+            dataSource,
+            symbol
+          };
+        })
+      }
+    });
+  }
+
   public async marketDataItems(params: {
     select?: Prisma.MarketDataSelectScalar;
     skip?: number;