Merge 2e1349f6a3 into b2634db99f

1 week ago · 4459996fb8
6 changed files with 121 additions and 74 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

+## 2.152.0-beta.2 - 2025-04-13
+
+### Changed
+
+- Optimized the query of the data range functionality (`getRange()`) in the market data service
+
 ## 2.151.0 - 2025-04-11

 ### Added
--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts
@ -6,6 +6,7 @@ import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
 import { DataSource, MarketData } from '@prisma/client';

 import { CurrentRateService } from './current-rate.service';
+import { DateQuery } from './interfaces/date-query.interface';
 import { GetValuesObject } from './interfaces/get-values-object.interface';

 jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {
@ -25,33 +26,40 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {
        },
        getRange: ({
          assetProfileIdentifiers,
-          dateRangeEnd,
-          dateRangeStart
+          dateQuery
        }: {
          assetProfileIdentifiers: AssetProfileIdentifier[];
-          dateRangeEnd: Date;
-          dateRangeStart: Date;
+          dateQuery: DateQuery;
+          skip?: number;
+          take?: number;
        }) => {
          return Promise.resolve<MarketData[]>([
            {
-              createdAt: dateRangeStart,
+              createdAt: dateQuery.gte,
              dataSource: assetProfileIdentifiers[0].dataSource,
-              date: dateRangeStart,
+              date: dateQuery.gte,
              id: '8fa48fde-f397-4b0d-adbc-fb940e830e6d',
              marketPrice: 1841.823902,
              state: 'CLOSE',
              symbol: assetProfileIdentifiers[0].symbol
            },
            {
-              createdAt: dateRangeEnd,
+              createdAt: dateQuery.lt,
              dataSource: assetProfileIdentifiers[0].dataSource,
-              date: dateRangeEnd,
+              date: dateQuery.lt,
              id: '082d6893-df27-4c91-8a5d-092e84315b56',
              marketPrice: 1847.839966,
              state: 'CLOSE',
              symbol: assetProfileIdentifiers[0].symbol
            }
          ]);
+        },
+        getRangeCount: ({}: {
+          assetProfileIdentifiers: AssetProfileIdentifier[];
+          dateRangeEnd: Date;
+          dateRangeStart: Date;
+        }) => {
+          return Promise.resolve<number>(2);
        }
      };
    })
@ -128,9 +136,15 @@ describe('CurrentRateService', () => {
      values: [
        {
          dataSource: 'YAHOO',
-          date: undefined,
+          date: new Date('2020-01-01T00:00:00.000Z'),
          marketPrice: 1841.823902,
          symbol: 'AMZN'
+        },
+        {
+          dataSource: 'YAHOO',
+          date: new Date('2020-01-02T00:00:00.000Z'),
+          marketPrice: 1847.839966,
+          symbol: 'AMZN'
        }
      ]
    });
--- a/apps/api/src/app/portfolio/current-rate.service.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.ts
@ -21,6 +21,8 @@ import { GetValuesParams } from './interfaces/get-values-params.interface';

@Injectable()
 export class CurrentRateService {
+  private static readonly MARKET_DATA_PAGE_SIZE = 50000;
+
  public constructor(
    private readonly dataProviderService: DataProviderService,
    private readonly marketDataService: MarketDataService,
@ -41,42 +43,37 @@ export class CurrentRateService {
      (!dateQuery.gte || isBefore(dateQuery.gte, new Date())) &&
      (!dateQuery.in || this.containsToday(dateQuery.in));

-    const promises: Promise<GetValueObject[]>[] = [];
    const quoteErrors: ResponseError['errors'] = [];
    const today = resetHours(new Date());
+    const values: GetValueObject[] = [];

    if (includesToday) {
-      promises.push(
-        this.dataProviderService
-          .getQuotes({ items: dataGatheringItems, user: this.request?.user })
-          .then((dataResultProvider) => {
-            const result: GetValueObject[] = [];
-
-            for (const { dataSource, symbol } of dataGatheringItems) {
-              if (dataResultProvider?.[symbol]?.dataProviderInfo) {
-                dataProviderInfos.push(
-                  dataResultProvider[symbol].dataProviderInfo
-                );
-              }
+      const quotesBySymbol = await this.dataProviderService.getQuotes({
+        items: dataGatheringItems,
+        user: this.request?.user
+      });

-              if (dataResultProvider?.[symbol]?.marketPrice) {
-                result.push({
-                  dataSource,
-                  symbol,
-                  date: today,
-                  marketPrice: dataResultProvider?.[symbol]?.marketPrice
-                });
-              } else {
-                quoteErrors.push({
-                  dataSource,
-                  symbol
-                });
-              }
-            }
+      for (const { dataSource, symbol } of dataGatheringItems) {
+        const quote = quotesBySymbol[symbol];

-            return result;
-          })
-      );
+        if (quote?.dataProviderInfo) {
+          dataProviderInfos.push(quote.dataProviderInfo);
+        }
+
+        if (quote?.marketPrice) {
+          values.push({
+            dataSource,
+            symbol,
+            date: today,
+            marketPrice: quote.marketPrice
+          });
+        } else {
+          quoteErrors.push({
+            dataSource,
+            symbol
+          });
+        }
+      }
    }

    const assetProfileIdentifiers: AssetProfileIdentifier[] =
@ -84,34 +81,42 @@ export class CurrentRateService {
        return { dataSource, symbol };
      });

-    promises.push(
-      this.marketDataService
-        .getRange({
-          assetProfileIdentifiers,
-          dateQuery
-        })
-        .then((data) => {
-          return data.map(({ dataSource, date, marketPrice, symbol }) => {
-            return {
-              dataSource,
-              date,
-              marketPrice,
-              symbol
-            };
-          });
-        })
-    );
-
-    const values = await Promise.all(promises).then((array) => {
-      return array.flat();
+    const marketDataCount = await this.marketDataService.getRangeCount({
+      assetProfileIdentifiers,
+      dateQuery
    });

+    for (
+      let i = 0;
+      i < marketDataCount;
+      i += CurrentRateService.MARKET_DATA_PAGE_SIZE
+    ) {
+      // Use pageSize to limit the number of records fetched at once
+      const data = await this.marketDataService.getRange({
+        assetProfileIdentifiers,
+        dateQuery,
+        skip: i,
+        take: CurrentRateService.MARKET_DATA_PAGE_SIZE
+      });
+
+      values.push(
+        ...data.map(({ dataSource, date, marketPrice, symbol }) => ({
+          dataSource,
+          date,
+          marketPrice,
+          symbol
+        }))
+      );
+    }
+
    const response: GetValuesObject = {
      dataProviderInfos,
      errors: quoteErrors.map(({ dataSource, symbol }) => {
        return { dataSource, symbol };
      }),
-      values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
+      values: uniqBy(values, ({ date, symbol }) => {
+        return `${date}-${symbol}`;
+      })
    };

    if (!isEmpty(quoteErrors)) {
--- a/apps/api/src/services/market-data/market-data.service.ts
+++ b/apps/api/src/services/market-data/market-data.service.ts
@ -60,12 +60,18 @@ export class MarketDataService {

  public async getRange({
    assetProfileIdentifiers,
-    dateQuery
+    dateQuery,
+    skip,
+    take
  }: {
    assetProfileIdentifiers: AssetProfileIdentifier[];
    dateQuery: DateQuery;
+    skip?: number;
+    take?: number;
  }): Promise<MarketData[]> {
    return this.prismaService.marketData.findMany({
+      skip,
+      take,
      orderBy: [
        {
          date: 'asc'
@ -75,17 +81,33 @@ export class MarketDataService {
        }
      ],
      where: {
-        dataSource: {
-          in: assetProfileIdentifiers.map(({ dataSource }) => {
-            return dataSource;
-          })
-        },
        date: dateQuery,
-        symbol: {
-          in: assetProfileIdentifiers.map(({ symbol }) => {
-            return symbol;
-          })
-        }
+        OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => {
+          return {
+            dataSource,
+            symbol
+          };
+        })
+      }
+    });
+  }
+
+  public async getRangeCount({
+    assetProfileIdentifiers,
+    dateQuery
+  }: {
+    assetProfileIdentifiers: AssetProfileIdentifier[];
+    dateQuery: DateQuery;
+  }): Promise<number> {
+    return this.prismaService.marketData.count({
+      where: {
+        date: dateQuery,
+        OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => {
+          return {
+            dataSource,
+            symbol
+          };
+        })
      }
    });
  }
--- a/package-lock.json
+++ b/package-lock.json
@ -1,12 +1,12 @@
 {
  "name": "ghostfolio",
-  "version": "2.151.0",
+  "version": "2.152.0-beta.2",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "ghostfolio",
-      "version": "2.151.0",
+      "version": "2.152.0-beta.2",
      "hasInstallScript": true,
      "license": "AGPL-3.0",
      "dependencies": {
--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
  "name": "ghostfolio",
-  "version": "2.151.0",
+  "version": "2.152.0-beta.2",
  "homepage": "https://ghostfol.io",
  "license": "AGPL-3.0",
  "repository": "https://github.com/ghostfolio/ghostfolio",