Feature/optimize import validation by reducing to unique asset profiles (#2198)

* Optimize activities validation

* Optimize data gathering in import

* Update changelog

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Optimized the validation in the activities import by reducing the list to unique asset profiles
+- Optimized the data gathering in the activities import
+
 ## 1.295.0 - 2023-07-30
 
 ### Added

apps/api/src/app/admin/admin.controller.ts

@@ -7,6 +7,7 @@ import {
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import {
   AdminData,
   AdminMarketData,
@@ -116,7 +117,7 @@ export class AdminController {
           name: GATHER_ASSET_PROFILE_PROCESS,
           opts: {
             ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}`
+            jobId: getAssetProfileIdentifier({ dataSource, symbol })
           }
         };
       })
@@ -152,7 +153,7 @@ export class AdminController {
           name: GATHER_ASSET_PROFILE_PROCESS,
           opts: {
             ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}`
+            jobId: getAssetProfileIdentifier({ dataSource, symbol })
           }
         };
       })
@@ -185,7 +186,7 @@ export class AdminController {
       name: GATHER_ASSET_PROFILE_PROCESS,
       opts: {
         ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-        jobId: `${dataSource}-${symbol}`
+        jobId: getAssetProfileIdentifier({ dataSource, symbol })
       }
     });
   }

apps/api/src/app/import/import.service.ts

@@ -8,10 +8,14 @@ import {
 import { OrderService } from '@ghostfolio/api/app/order/order.service';
 import { PlatformService } from '@ghostfolio/api/app/platform/platform.service';
 import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
-import { parseDate } from '@ghostfolio/common/helper';
+import {
+  getAssetProfileIdentifier,
+  parseDate
+} from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import {
   AccountWithPlatform,
@@ -21,12 +25,14 @@ import { Injectable } from '@nestjs/common';
 import { DataSource, Prisma, SymbolProfile } from '@prisma/client';
 import Big from 'big.js';
 import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';
+import { uniqBy } from 'lodash';
 import { v4 as uuidv4 } from 'uuid';
 
 @Injectable()
 export class ImportService {
   public constructor(
     private readonly accountService: AccountService,
+    private readonly dataGatheringService: DataGatheringService,
     private readonly dataProviderService: DataProviderService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly orderService: OrderService,
@@ -220,8 +226,7 @@ export class ImportService {
 
     const assetProfiles = await this.validateActivities({
       activitiesDto,
-      maxActivitiesToImport,
+      maxActivitiesToImport
-      userId
     });
 
     const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({
@@ -250,10 +255,37 @@ export class ImportService {
       error,
       fee,
       quantity,
-      SymbolProfile: assetProfile,
+      SymbolProfile,
       type,
       unitPrice
     } of activitiesExtendedWithErrors) {
+      const assetProfile = assetProfiles[
+        getAssetProfileIdentifier({
+          dataSource: SymbolProfile.dataSource,
+          symbol: SymbolProfile.symbol
+        })
+      ] ?? {
+        currency: SymbolProfile.currency,
+        dataSource: SymbolProfile.dataSource,
+        symbol: SymbolProfile.symbol
+      };
+      const {
+        assetClass,
+        assetSubClass,
+        countries,
+        createdAt,
+        currency,
+        dataSource,
+        id,
+        isin,
+        name,
+        scraperConfiguration,
+        sectors,
+        symbol,
+        symbolMapping,
+        url,
+        updatedAt
+      } = assetProfile;
       const validatedAccount = accounts.find(({ id }) => {
         return id === accountId;
       });
@@ -279,23 +311,22 @@ export class ImportService {
           id: uuidv4(),
           isDraft: isAfter(date, endOfToday()),
           SymbolProfile: {
-            assetClass: assetProfile.assetClass,
+            assetClass,
-            assetSubClass: assetProfile.assetSubClass,
+            assetSubClass,
-            comment: assetProfile.comment,
+            countries,
-            countries: assetProfile.countries,
+            createdAt,
-            createdAt: assetProfile.createdAt,
+            currency,
-            currency: assetProfile.currency,
+            dataSource,
-            dataSource: assetProfile.dataSource,
+            id,
-            id: assetProfile.id,
+            isin,
-            isin: assetProfile.isin,
+            name,
-            name: assetProfile.name,
+            scraperConfiguration,
-            scraperConfiguration: assetProfile.scraperConfiguration,
+            sectors,
-            sectors: assetProfile.sectors,
+            symbol,
-            symbol: assetProfile.currency,
+            symbolMapping,
-            symbolMapping: assetProfile.symbolMapping,
+            updatedAt,
-            updatedAt: assetProfile.updatedAt,
+            url,
-            url: assetProfile.url,
+            comment: assetProfile.comment
-            ...assetProfiles[assetProfile.symbol]
           },
           Account: validatedAccount,
           symbolProfileId: undefined,
@@ -318,14 +349,14 @@ export class ImportService {
           SymbolProfile: {
             connectOrCreate: {
               create: {
-                currency: assetProfile.currency,
+                currency,
-                dataSource: assetProfile.dataSource,
+                dataSource,
-                symbol: assetProfile.symbol
+                symbol
               },
               where: {
                 dataSource_symbol: {
-                  dataSource: assetProfile.dataSource,
+                  dataSource,
-                  symbol: assetProfile.symbol
+                  symbol
                 }
               }
             }
@@ -337,24 +368,49 @@ export class ImportService {
 
       const value = new Big(quantity).mul(unitPrice).toNumber();
 
-      //@ts-ignore
       activities.push({
         ...order,
         error,
         value,
         feeInBaseCurrency: this.exchangeRateDataService.toCurrency(
           fee,
-          assetProfile.currency,
+          currency,
           userCurrency
         ),
+        //@ts-ignore
+        SymbolProfile: assetProfile,
         valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
           value,
-          assetProfile.currency,
+          currency,
           userCurrency
         )
       });
     }
 
+    activities.sort((activity1, activity2) => {
+      return Number(activity1.date) - Number(activity2.date);
+    });
+
+    if (!isDryRun) {
+      // Gather symbol data in the background, if not dry run
+      const uniqueActivities = uniqBy(activities, ({ SymbolProfile }) => {
+        return getAssetProfileIdentifier({
+          dataSource: SymbolProfile.dataSource,
+          symbol: SymbolProfile.symbol
+        });
+      });
+
+      this.dataGatheringService.gatherSymbols(
+        uniqueActivities.map(({ date, SymbolProfile }) => {
+          return {
+            date,
+            dataSource: SymbolProfile.dataSource,
+            symbol: SymbolProfile.symbol
+          };
+        })
+      );
+    }
+
     return activities;
   }
 
@@ -446,25 +502,30 @@ export class ImportService {
 
   private async validateActivities({
     activitiesDto,
-    maxActivitiesToImport,
+    maxActivitiesToImport
-    userId
   }: {
     activitiesDto: Partial<CreateOrderDto>[];
     maxActivitiesToImport: number;
-    userId: string;
   }) {
     if (activitiesDto?.length > maxActivitiesToImport) {
       throw new Error(`Too many activities (${maxActivitiesToImport} at most)`);
     }
 
     const assetProfiles: {
-      [symbol: string]: Partial<SymbolProfile>;
+      [assetProfileIdentifier: string]: Partial<SymbolProfile>;
     } = {};
 
+    const uniqueActivitiesDto = uniqBy(
+      activitiesDto,
+      ({ dataSource, symbol }) => {
+        return getAssetProfileIdentifier({ dataSource, symbol });
+      }
+    );
+
     for (const [
       index,
       { currency, dataSource, symbol }
-    ] of activitiesDto.entries()) {
+    ] of uniqueActivitiesDto.entries()) {
       if (dataSource !== 'MANUAL') {
         const assetProfile = (
           await this.dataProviderService.getAssetProfiles([
@@ -484,7 +545,8 @@ export class ImportService {
           );
         }
 
-        assetProfiles[symbol] = assetProfile;
+        assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })] =
+          assetProfile;
       }
     }
 

apps/api/src/app/order/order.controller.ts

@@ -2,6 +2,7 @@ import { RedactValuesInResponseInterceptor } from '@ghostfolio/api/interceptors/
 import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
 import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
 import { ApiService } from '@ghostfolio/api/services/api/api.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
 import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
 import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
 import { hasPermission, permissions } from '@ghostfolio/common/permissions';
@@ -36,6 +37,7 @@ import { UpdateOrderDto } from './update-order.dto';
 export class OrderController {
   public constructor(
     private readonly apiService: ApiService,
+    private readonly dataGatheringService: DataGatheringService,
     private readonly impersonationService: ImpersonationService,
     private readonly orderService: OrderService,
     @Inject(REQUEST) private readonly request: RequestWithUser
@@ -123,7 +125,7 @@ export class OrderController {
       );
     }
 
-    return this.orderService.createOrder({
+    const order = await this.orderService.createOrder({
       ...data,
       date: parseISO(data.date),
       SymbolProfile: {
@@ -144,6 +146,19 @@ export class OrderController {
       User: { connect: { id: this.request.user.id } },
       userId: this.request.user.id
     });
+
+    if (!order.isDraft) {
+      // Gather symbol data in the background, if not draft
+      this.dataGatheringService.gatherSymbols([
+        {
+          dataSource: data.dataSource,
+          date: order.date,
+          symbol: data.symbol
+        }
+      ]);
+    }
+
+    return order;
   }
 
   @Put(':id')

apps/api/src/app/order/order.service.ts

@@ -7,6 +7,7 @@ import {
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import { Filter } from '@ghostfolio/common/interfaces';
 import { OrderWithAccount } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
@@ -117,7 +118,7 @@ export class OrderService {
       };
     }
 
-    await this.dataGatheringService.addJobToQueue({
+    this.dataGatheringService.addJobToQueue({
       data: {
         dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
         symbol: data.SymbolProfile.connectOrCreate.create.symbol
@@ -125,25 +126,12 @@ export class OrderService {
       name: GATHER_ASSET_PROFILE_PROCESS,
       opts: {
         ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-        jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}`
+        jobId: getAssetProfileIdentifier({
-      }
-    });
-
-    const isDraft =
-      data.type === 'LIABILITY'
-        ? false
-        : isAfter(data.date as Date, endOfToday());
-
-    if (!isDraft) {
-      // Gather symbol data of order in the background, if not draft
-      this.dataGatheringService.gatherSymbols([
-        {
           dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
-          date: <Date>data.date,
           symbol: data.SymbolProfile.connectOrCreate.create.symbol
-        }
+        })
-      ]);
+      }
-    }
+    });
 
     delete data.accountId;
     delete data.assetClass;
@@ -162,6 +150,11 @@ export class OrderService {
 
     const orderData: Prisma.OrderCreateInput = data;
 
+    const isDraft =
+      data.type === 'LIABILITY'
+        ? false
+        : isAfter(data.date as Date, endOfToday());
+
     const order = await this.prismaService.order.create({
       data: {
         ...orderData,

apps/api/src/app/redis-cache/redis-cache.service.ts

@@ -1,4 +1,5 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { CACHE_MANAGER, Inject, Injectable, Logger } from '@nestjs/common';
 
@@ -22,7 +23,7 @@ export class RedisCacheService {
   }
 
   public getQuoteKey({ dataSource, symbol }: UniqueAsset) {
-    return `quote-${dataSource}-${symbol}`;
+    return `quote-${getAssetProfileIdentifier({ dataSource, symbol })}`;
   }
 
   public async remove(key: string) {

apps/api/src/services/cron.service.ts

@@ -2,6 +2,7 @@ import {
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import { Injectable } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 
@@ -48,7 +49,7 @@ export class CronService {
           name: GATHER_ASSET_PROFILE_PROCESS,
           opts: {
             ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}`
+            jobId: getAssetProfileIdentifier({ dataSource, symbol })
           }
         };
       })

apps/api/src/services/data-gathering/data-gathering.service.ts

@@ -10,7 +10,11 @@ import {
   GATHER_HISTORICAL_MARKET_DATA_PROCESS,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
-import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
+import {
+  DATE_FORMAT,
+  getAssetProfileIdentifier,
+  resetHours
+} from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { InjectQueue } from '@nestjs/bull';
 import { Inject, Injectable, Logger } from '@nestjs/common';
@@ -221,7 +225,10 @@ export class DataGatheringService {
           name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
           opts: {
             ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}`
+            jobId: `${getAssetProfileIdentifier({
+              dataSource,
+              symbol
+            })}-${format(date, DATE_FORMAT)}`
           }
         };
       })

libs/common/src/lib/helper.ts

@@ -5,7 +5,7 @@ import { getDate, getMonth, getYear, parse, subDays } from 'date-fns';
 import { de, es, fr, it, nl, pt } from 'date-fns/locale';
 
 import { ghostfolioScraperApiSymbolPrefix, locale } from './config';
-import { Benchmark } from './interfaces';
+import { Benchmark, UniqueAsset } from './interfaces';
 import { ColorScheme } from './types';
 
 const NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g;
@@ -64,6 +64,10 @@ export function extractNumberFromString(aString: string): number {
   }
 }
 
+export function getAssetProfileIdentifier({ dataSource, symbol }: UniqueAsset) {
+  return `${dataSource}-${symbol}`;
+}
+
 export function getBackgroundColor(aColorScheme: ColorScheme) {
   return getCssVariable(
     aColorScheme === 'DARK' ||