Optimize activities validation

apps/api/src/app/admin/admin.controller.ts

@@ -7,6 +7,7 @@ import {
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import {
   AdminData,
   AdminMarketData,
@@ -116,7 +117,7 @@ export class AdminController {
           name: GATHER_ASSET_PROFILE_PROCESS,
           opts: {
             ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}`
+            jobId: getAssetProfileIdentifier({ dataSource, symbol })
           }
         };
       })
@@ -152,7 +153,7 @@ export class AdminController {
           name: GATHER_ASSET_PROFILE_PROCESS,
           opts: {
             ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}`
+            jobId: getAssetProfileIdentifier({ dataSource, symbol })
           }
         };
       })
@@ -185,7 +186,7 @@ export class AdminController {
       name: GATHER_ASSET_PROFILE_PROCESS,
       opts: {
         ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-        jobId: `${dataSource}-${symbol}`
+        jobId: getAssetProfileIdentifier({ dataSource, symbol })
       }
     });
   }

apps/api/src/app/import/import.service.ts

@@ -11,7 +11,10 @@ import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.servic
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
-import { parseDate } from '@ghostfolio/common/helper';
+import {
+  getAssetProfileIdentifier,
+  parseDate
+} from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import {
   AccountWithPlatform,
@@ -21,6 +24,7 @@ import { Injectable } from '@nestjs/common';
 import { DataSource, Prisma, SymbolProfile } from '@prisma/client';
 import Big from 'big.js';
 import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';
+import { uniqBy } from 'lodash';
 import { v4 as uuidv4 } from 'uuid';
 
 @Injectable()
@@ -220,8 +224,7 @@ export class ImportService {
 
     const assetProfiles = await this.validateActivities({
       activitiesDto,
-      maxActivitiesToImport,
+      maxActivitiesToImport
-      userId
     });
 
     const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({
@@ -295,7 +298,12 @@ export class ImportService {
             symbolMapping: assetProfile.symbolMapping,
             updatedAt: assetProfile.updatedAt,
             url: assetProfile.url,
-            ...assetProfiles[assetProfile.symbol]
+            ...assetProfiles[
+              getAssetProfileIdentifier({
+                dataSource: assetProfile.dataSource,
+                symbol: assetProfile.symbol
+              })
+            ]
           },
           Account: validatedAccount,
           symbolProfileId: undefined,
@@ -446,25 +454,30 @@ export class ImportService {
 
   private async validateActivities({
     activitiesDto,
-    maxActivitiesToImport,
+    maxActivitiesToImport
-    userId
   }: {
     activitiesDto: Partial<CreateOrderDto>[];
     maxActivitiesToImport: number;
-    userId: string;
   }) {
     if (activitiesDto?.length > maxActivitiesToImport) {
       throw new Error(`Too many activities (${maxActivitiesToImport} at most)`);
     }
 
     const assetProfiles: {
-      [symbol: string]: Partial<SymbolProfile>;
+      [assetProfileIdentifier: string]: Partial<SymbolProfile>;
     } = {};
 
+    const uniqueActivitiesDto = uniqBy(
+      activitiesDto,
+      ({ dataSource, symbol }) => {
+        return getAssetProfileIdentifier({ dataSource, symbol });
+      }
+    );
+
     for (const [
       index,
       { currency, dataSource, symbol }
-    ] of activitiesDto.entries()) {
+    ] of uniqueActivitiesDto.entries()) {
       if (dataSource !== 'MANUAL') {
         const assetProfile = (
           await this.dataProviderService.getAssetProfiles([
@@ -484,7 +497,8 @@ export class ImportService {
           );
         }
 
-        assetProfiles[symbol] = assetProfile;
+        assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })] =
+          assetProfile;
       }
     }
 

apps/api/src/app/order/order.service.ts

@@ -7,6 +7,7 @@ import {
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import { Filter } from '@ghostfolio/common/interfaces';
 import { OrderWithAccount } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
@@ -125,7 +126,10 @@ export class OrderService {
       name: GATHER_ASSET_PROFILE_PROCESS,
       opts: {
         ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-        jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}`
+        jobId: getAssetProfileIdentifier({
+          dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
+          symbol: data.SymbolProfile.connectOrCreate.create.symbol
+        })
       }
     });
 

apps/api/src/app/redis-cache/redis-cache.service.ts

@@ -1,4 +1,5 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { CACHE_MANAGER, Inject, Injectable, Logger } from '@nestjs/common';
 
@@ -22,7 +23,7 @@ export class RedisCacheService {
   }
 
   public getQuoteKey({ dataSource, symbol }: UniqueAsset) {
-    return `quote-${dataSource}-${symbol}`;
+    return `quote-${getAssetProfileIdentifier({ dataSource, symbol })}`;
   }
 
   public async remove(key: string) {

apps/api/src/services/cron.service.ts

@@ -2,6 +2,7 @@ import {
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import { Injectable } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 
@@ -48,7 +49,7 @@ export class CronService {
           name: GATHER_ASSET_PROFILE_PROCESS,
           opts: {
             ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}`
+            jobId: getAssetProfileIdentifier({ dataSource, symbol })
           }
         };
       })

apps/api/src/services/data-gathering/data-gathering.service.ts

@@ -10,7 +10,11 @@ import {
   GATHER_HISTORICAL_MARKET_DATA_PROCESS,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
-import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
+import {
+  DATE_FORMAT,
+  getAssetProfileIdentifier,
+  resetHours
+} from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { InjectQueue } from '@nestjs/bull';
 import { Inject, Injectable, Logger } from '@nestjs/common';
@@ -221,7 +225,10 @@ export class DataGatheringService {
           name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
           opts: {
             ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}`
+            jobId: `${getAssetProfileIdentifier({
+              dataSource,
+              symbol
+            })}-${format(date, DATE_FORMAT)}`
           }
         };
       })

libs/common/src/lib/helper.ts

@@ -5,7 +5,7 @@ import { getDate, getMonth, getYear, parse, subDays } from 'date-fns';
 import { de, es, fr, it, nl, pt } from 'date-fns/locale';
 
 import { ghostfolioScraperApiSymbolPrefix, locale } from './config';
-import { Benchmark } from './interfaces';
+import { Benchmark, UniqueAsset } from './interfaces';
 import { ColorScheme } from './types';
 
 const NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g;
@@ -64,6 +64,10 @@ export function extractNumberFromString(aString: string): number {
   }
 }
 
+export function getAssetProfileIdentifier({ dataSource, symbol }: UniqueAsset) {
+  return `${dataSource}-${symbol}`;
+}
+
 export function getBackgroundColor(aColorScheme: ColorScheme) {
   return getCssVariable(
     aColorScheme === 'DARK' ||