Merge branch 'main' into task/improve-performance-of-last-market-price-in-market-data-table-of-admin-control

CHANGELOG.md

@@ -7,13 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Changed
+
+- Improved the performance of the historical market data table of the admin control panel by loading only the latest market price per asset profile
+
+## 3.80.0 - 2026-10-05
+
 ### Added
 
 - Exposed the `ENABLE_FEATURE_SECURITY_HEADERS` environment variable to enable HTTP security headers (experimental)
 
 ### Changed
 
-- Improved the performance of the historical market data table of the admin control panel by loading only the latest market price per asset profile
+- Improved the portfolio snapshot calculation to get the quotes of active holdings only
+- Improved the performance of getting the latest market data by optimizing the indexes of the market data table
 - Deprecated `SymbolProfile` in favor of `assetProfile` in the endpoint `POST api/v1/activities`
 - Upgraded `Nx` from version `23.1.1` to `23.2.1`
 

apps/api/src/app/portfolio/calculator/portfolio-calculator.ts

@@ -268,6 +268,7 @@ export abstract class PortfolioCalculator {
       };
     }
 
+    const assetProfileIdentifiersWithQuotes: AssetProfileIdentifier[] = [];
     const cashAssetProfileIdentifiers = new Set<string>();
     const currencies: { [assetProfileIdentifier: string]: string } = {};
     const dataGatheringItems: DataGatheringItem[] = [];
@@ -279,6 +280,7 @@ export abstract class PortfolioCalculator {
       assetSubClass,
       currency,
       dataSource,
+      quantity,
       symbol
     } of holdingBalancesByDate.at(-1).holdings) {
       // Gather data for all assets except CASH
@@ -287,6 +289,13 @@ export abstract class PortfolioCalculator {
           dataSource,
           symbol
         });
+
+        if (!quantity.eq(0)) {
+          assetProfileIdentifiersWithQuotes.push({
+            dataSource,
+            symbol
+          });
+        }
       }
 
       currencies[getAssetProfileIdentifier({ dataSource, symbol })] = currency;
@@ -305,6 +314,7 @@ export abstract class PortfolioCalculator {
       errors: currentRateErrors,
       values: marketSymbols
     } = await this.currentRateService.getValues({
+      assetProfileIdentifiersWithQuotes,
       dataGatheringItems,
       dateQuery: {
         gte: this.startDate,

apps/api/src/app/portfolio/current-rate.service.spec.ts

@@ -144,6 +144,9 @@ describe('CurrentRateService', () => {
   it('getValues', async () => {
     expect(
       await currentRateService.getValues({
+        assetProfileIdentifiersWithQuotes: [
+          { dataSource: DataSource.YAHOO, symbol: 'AMZN' }
+        ],
         dataGatheringItems: [{ dataSource: DataSource.YAHOO, symbol: 'AMZN' }],
         dateQuery: {
           lt: new Date(Date.UTC(2020, 0, 2, 0, 0, 0)),
@@ -210,7 +213,8 @@ describe('CurrentRateService', () => {
 
       const { errors, values } = await currentRateService.getValues({
         dataGatheringItems,
-        dateQuery
+        dateQuery,
+        assetProfileIdentifiersWithQuotes: dataGatheringItems
       });
 
       expect(getLatestActivity).not.toHaveBeenCalled();
@@ -234,7 +238,8 @@ describe('CurrentRateService', () => {
 
       const { errors, values } = await currentRateService.getValues({
         dataGatheringItems,
-        dateQuery
+        dateQuery,
+        assetProfileIdentifiersWithQuotes: dataGatheringItems
       });
 
       expect(getLatestActivity).toHaveBeenCalledWith({
@@ -252,5 +257,49 @@ describe('CurrentRateService', () => {
         }
       ]);
     });
+
+    it('should carry the latest market price forward without a quote request', async () => {
+      const getQuotes = jest.spyOn(dataProviderService, 'getQuotes');
+
+      jest.spyOn(marketDataService, 'getRangeCount').mockResolvedValue(1);
+
+      jest.spyOn(marketDataService, 'getRange').mockResolvedValue([
+        {
+          createdAt: yesterday,
+          dataSource: DataSource.YAHOO,
+          date: yesterday,
+          id: 'd51d4e0b-9d1f-4d2e-8a0c-9a0f5c6b1d22',
+          isCarriedForward: false,
+          marketPrice: 1841.823902,
+          state: 'CLOSE',
+          symbol: 'AMZN'
+        }
+      ]);
+
+      const { errors, values } = await currentRateService.getValues({
+        dataGatheringItems,
+        dateQuery,
+        assetProfileIdentifiersWithQuotes: []
+      });
+
+      expect(getQuotes).toHaveBeenCalledWith(
+        expect.objectContaining({ items: [] })
+      );
+      expect(errors).toEqual([]);
+      expect(values).toEqual([
+        {
+          dataSource: DataSource.YAHOO,
+          date: yesterday,
+          marketPrice: 1841.823902,
+          symbol: 'AMZN'
+        },
+        {
+          dataSource: DataSource.YAHOO,
+          date: today,
+          marketPrice: 1841.823902,
+          symbol: 'AMZN'
+        }
+      ]);
+    });
   });
 });

apps/api/src/app/portfolio/current-rate.service.ts

@@ -16,7 +16,7 @@ import { utc } from '@date-fns/utc';
 import { Injectable } from '@nestjs/common';
 import { Type as ActivityType } from '@prisma/client';
 import { isBefore, isSameDay } from 'date-fns';
-import { isEmpty, uniqBy } from 'lodash-es';
+import { uniqBy } from 'lodash-es';
 
 import { GetValueObject } from './interfaces/get-value-object.interface';
 import { GetValuesObject } from './interfaces/get-values-object.interface';
@@ -34,6 +34,7 @@ export class CurrentRateService {
 
   @LogPerformance
   public async getValues({
+    assetProfileIdentifiersWithQuotes,
     dataGatheringItems,
     dateQuery,
     subscriptionType
@@ -52,10 +53,10 @@ export class CurrentRateService {
     if (includesToday) {
       const quotes = await this.dataProviderService.getQuotes({
         subscriptionType,
-        items: dataGatheringItems
+        items: assetProfileIdentifiersWithQuotes
       });
 
-      for (const { dataSource, symbol } of dataGatheringItems) {
+      for (const { dataSource, symbol } of assetProfileIdentifiersWithQuotes) {
         const quote = quotes[getAssetProfileIdentifier({ dataSource, symbol })];
 
         if (quote?.dataProviderInfo) {
@@ -121,8 +122,25 @@ export class CurrentRateService {
       })
     };
 
-    if (!isEmpty(quoteErrors)) {
+    if (includesToday) {
-      for (const { dataSource, symbol } of quoteErrors) {
+      const assetProfileIdentifiersWithoutQuotes = [
+        ...quoteErrors,
+        ...dataGatheringItems.filter(({ dataSource, symbol }) => {
+          return !assetProfileIdentifiersWithQuotes.some(
+            (assetProfileIdentifier) => {
+              return (
+                assetProfileIdentifier.dataSource === dataSource &&
+                assetProfileIdentifier.symbol === symbol
+              );
+            }
+          );
+        })
+      ];
+
+      for (const {
+        dataSource,
+        symbol
+      } of assetProfileIdentifiersWithoutQuotes) {
         try {
           // If missing quote, fallback to the latest available historical market price
           let value: GetValueObject = response.values.find((currentValue) => {

apps/api/src/app/portfolio/interfaces/get-values-params.interface.ts

@@ -1,9 +1,11 @@
 import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
 import { SubscriptionType } from '@ghostfolio/common/enums';
+import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
 
 import { DateQuery } from './date-query.interface';
 
 export interface GetValuesParams {
+  assetProfileIdentifiersWithQuotes: AssetProfileIdentifier[];
   dataGatheringItems: DataGatheringItem[];
   dateQuery: DateQuery;
   subscriptionType?: SubscriptionType;

apps/api/src/app/portfolio/portfolio.service.spec.ts

@@ -997,6 +997,9 @@ describe('PortfolioService', () => {
       // The closed holding (MSFT) does not need a market price
       expect(getValues).toHaveBeenCalledWith(
         expect.objectContaining({
+          assetProfileIdentifiersWithQuotes: [
+            { dataSource: DataSource.YAHOO, symbol: 'AAPL' }
+          ],
           dataGatheringItems: [{ dataSource: DataSource.YAHOO, symbol: 'AAPL' }]
         })
       );

apps/api/src/app/portfolio/portfolio.service.ts

@@ -2545,6 +2545,16 @@ export class PortfolioService {
       return !quantity.eq(0);
     });
 
+    const assetProfileIdentifiers: AssetProfileIdentifier[] = openHoldings.map(
+      ({
+        latestActivity: {
+          assetProfile: { dataSource, symbol }
+        }
+      }) => {
+        return { dataSource, symbol };
+      }
+    );
+
     const now = new Date();
 
     // Get the market prices of today with the same fallback as the portfolio
@@ -2553,15 +2563,8 @@ export class PortfolioService {
       openHoldings.length > 0
         ? await this.currentRateService.getValues({
             subscriptionType,
-            dataGatheringItems: openHoldings.map(
+            assetProfileIdentifiersWithQuotes: assetProfileIdentifiers,
-              ({
+            dataGatheringItems: assetProfileIdentifiers,
-                latestActivity: {
-                  assetProfile: { dataSource, symbol }
-                }
-              }) => {
-                return { dataSource, symbol };
-              }
-            ),
             dateQuery: {
               gte: startOfDay(now),
               lt: endOfDay(now)

apps/api/src/services/market-data/market-data.service.ts

@@ -248,7 +248,7 @@ export class MarketDataService {
             state: state as MarketDataState
           },
           where: {
-            dataSource_date_symbol: {
+            dataSource_symbol_date: {
               dataSource: dataSource as DataSource,
               date: date as Date,
               symbol: symbol as string

apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@@ -348,7 +348,7 @@ export class DataGatheringService {
           },
           update: { marketPrice, isCarriedForward: false },
           where: {
-            dataSource_date_symbol: {
+            dataSource_symbol_date: {
               dataSource,
               symbol,
               date: startOfUtcDate

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "ghostfolio",
-  "version": "3.79.0",
+  "version": "3.80.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "ghostfolio",
-      "version": "3.79.0",
+      "version": "3.80.0",
       "hasInstallScript": true,
       "license": "AGPL-3.0",
       "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ghostfolio",
-  "version": "3.79.0",
+  "version": "3.80.0",
   "homepage": "https://ghostfol.io",
   "license": "AGPL-3.0",
   "repository": "https://github.com/ghostfolio/ghostfolio",

prisma/migrations/20261005000000_updated_indexes_of_market_data/migration.sql

@@ -0,0 +1,8 @@
+-- CreateIndex
+CREATE UNIQUE INDEX "MarketData_dataSource_symbol_date_key" ON "MarketData"("dataSource", "symbol", "date");
+
+-- DropIndex
+DROP INDEX "MarketData_dataSource_symbol_idx";
+
+-- DropIndex
+DROP INDEX "MarketData_dataSource_date_symbol_key";

prisma/schema.prisma

@@ -173,9 +173,8 @@ model MarketData {
   state            MarketDataState @default(CLOSE)
   symbol           String
 
-  @@unique([dataSource, date, symbol])
+  @@unique([dataSource, symbol, date])
   @@index([dataSource])
-  @@index([dataSource, symbol])
   @@index([date])
   @@index([state])
   @@index([symbol])