Improve symbol validation for assets with manual data source

apps/api/src/app/import/import.service.ts

@@ -198,6 +198,20 @@ export class ImportService {
     const tagIdMapping: { [oldTagId: string]: string } = {};
     const userCurrency = user.settings.settings.baseCurrency;
 
+    // Validate the symbols before any data is persisted
+    for (const [index, assetProfileWithMarketData] of (
+      assetProfilesWithMarketDataDto ?? []
+    ).entries()) {
+      if (
+        assetProfileWithMarketData.dataSource === DataSource.MANUAL &&
+        !isValidCustomAssetProfileSymbol(assetProfileWithMarketData.symbol)
+      ) {
+        throw new Error(
+          `assetProfiles.${index}.symbol ("${assetProfileWithMarketData.symbol}") must be a UUID or start with the prefix "${ghostfolioPrefix}_" for the data source ("${DataSource.MANUAL}")`
+        );
+      }
+    }
+
     if (platformsDto?.length) {
       const canCreatePlatform = hasPermission(
         user.permissions,
@@ -236,6 +250,26 @@ export class ImportService {
       }
     }
 
+    for (const [index, activity] of activitiesDto.entries()) {
+      if (!activity.dataSource) {
+        if (['FEE', 'INTEREST', 'LIABILITY'].includes(activity.type)) {
+          activity.dataSource = DataSource.MANUAL;
+        } else {
+          activity.dataSource =
+            this.dataProviderService.getDataSourceForImport();
+        }
+      }
+
+      if (
+        activity.dataSource === DataSource.MANUAL &&
+        !isValidCustomAssetProfileSymbol(activity.symbol)
+      ) {
+        throw new Error(
+          `activities.${index}.symbol ("${activity.symbol}") must be a UUID or start with the prefix "${ghostfolioPrefix}_" for the data source ("${DataSource.MANUAL}")`
+        );
+      }
+    }
+
     const existingTagsOfUser =
       tagsDto?.length || (!isDryRun && accountsWithBalancesDto?.length)
         ? await this.tagService.getTagsForUser(user.id)
@@ -386,93 +420,119 @@ export class ImportService {
       }
     }
 
-    for (const [index, assetProfileWithMarketData] of (
+    if (assetProfilesWithMarketDataDto?.length) {
-      assetProfilesWithMarketDataDto ?? []
+      const customAssetProfileNames = assetProfilesWithMarketDataDto
-    ).entries()) {
+        .filter(({ dataSource, name }) => {
-      if (
+          return dataSource === DataSource.MANUAL && Boolean(name);
-        assetProfileWithMarketData.dataSource === DataSource.MANUAL &&
+        })
-        !isValidCustomAssetProfileSymbol(assetProfileWithMarketData.symbol)
+        .map(({ name }) => {
-      ) {
+          return name;
-        throw new Error(
+        });
-          `assetProfiles.${index}.symbol ("${assetProfileWithMarketData.symbol}") must be a UUID or start with the prefix "${ghostfolioPrefix}_" for the data source ("${DataSource.MANUAL}")`
-        );
-      }
-    }
 
-    if (!isDryRun && assetProfilesWithMarketDataDto?.length) {
+      const [existingAssetProfiles, existingCustomAssetProfilesOfUser] =
-      const existingAssetProfiles =
+        await Promise.all([
-        await this.symbolProfileService.getSymbolProfiles(
+          this.symbolProfileService.getSymbolProfiles(
-          assetProfilesWithMarketDataDto.map(({ dataSource, symbol }) => {
+            assetProfilesWithMarketDataDto.map(({ dataSource, symbol }) => {
-            return { dataSource, symbol };
+              return { dataSource, symbol };
+            })
+          ),
+          this.symbolProfileService.getCustomSymbolProfilesByNames({
+            names: customAssetProfileNames,
+            userId: user.id
           })
-        );
+        ]);
 
       for (const assetProfileWithMarketData of assetProfilesWithMarketDataDto) {
+        let symbol = assetProfileWithMarketData.symbol;
+
         // Check if there is any existing asset profile
         const existingAssetProfile = existingAssetProfiles.find(
-          ({ dataSource, symbol }) => {
+          (assetProfile) => {
             return (
-              dataSource === assetProfileWithMarketData.dataSource &&
+              assetProfile.dataSource ===
-              symbol === assetProfileWithMarketData.symbol
+                assetProfileWithMarketData.dataSource &&
+              assetProfile.symbol === assetProfileWithMarketData.symbol
             );
           }
         );
 
-        // If there is no asset profile or if the asset profile belongs to a different user, then create a new asset profile
+        // If there is no asset profile or if the asset profile belongs to a different user, then reuse the custom asset profile of the user or create a new asset profile
         if (!existingAssetProfile || existingAssetProfile.userId !== user.id) {
-          const assetProfile: CreateAssetProfileDto = omit(
+          // Check if the user has a custom asset profile with the same name
-            assetProfileWithMarketData,
+          const existingCustomAssetProfileOfUser =
-            'marketData'
+            assetProfileWithMarketData.dataSource === DataSource.MANUAL
-          );
+              ? existingCustomAssetProfilesOfUser.find(({ name }) => {
+                  return name === assetProfileWithMarketData.name;
+                })
+              : undefined;
+
+          if (existingCustomAssetProfileOfUser) {
+            // Reuse the custom asset profile of the user instead of creating a duplicate
+            symbol = existingCustomAssetProfileOfUser.symbol;
+          } else {
+            const assetProfile: CreateAssetProfileDto = omit(
+              assetProfileWithMarketData,
+              'marketData'
+            );
+
+            // Asset profile belongs to a different user
+            if (existingAssetProfile) {
+              symbol = randomUUID();
+            }
 
-          // Asset profile belongs to a different user
-          if (existingAssetProfile) {
-            const symbol = randomUUID();
-            assetProfileSymbolMapping[assetProfile.symbol] = symbol;
             assetProfile.symbol = symbol;
-          }
 
-          // Create a new asset profile
+            if (!isDryRun) {
-          const assetProfileObject: Prisma.SymbolProfileCreateInput = {
+              // Create a new asset profile
-            ...assetProfile,
+              const assetProfileObject: Prisma.SymbolProfileCreateInput = {
-            user: { connect: { id: user.id } }
+                ...assetProfile,
-          };
+                user: { connect: { id: user.id } }
+              };
 
-          await this.symbolProfileService.add(assetProfileObject);
+              await this.symbolProfileService.add(assetProfileObject);
-        }
+            }
 
-        // Insert or update market data
+            if (
-        const marketDataObjects = assetProfileWithMarketData.marketData.map(
+              assetProfile.dataSource === DataSource.MANUAL &&
-          (marketData) => {
+              Boolean(assetProfile.name)
-            return {
+            ) {
-              ...marketData,
+              existingCustomAssetProfilesOfUser.push({
-              dataSource: assetProfileWithMarketData.dataSource,
+                name: assetProfile.name,
-              symbol: assetProfileWithMarketData.symbol
+                symbol: assetProfile.symbol
-            } as Prisma.MarketDataUpdateInput;
+              });
+            }
           }
-        );
 
-        await this.marketDataService.updateMany({ data: marketDataObjects });
+          if (symbol !== assetProfileWithMarketData.symbol) {
-      }
+            assetProfileSymbolMapping[assetProfileWithMarketData.symbol] =
-    }
+              symbol;
 
-    for (const [index, activity] of activitiesDto.entries()) {
+            // Keep the asset profile in sync with the activities to validate
-      if (!activity.dataSource) {
+            assetProfileWithMarketData.symbol = symbol;
-        if (['FEE', 'INTEREST', 'LIABILITY'].includes(activity.type)) {
+          }
-          activity.dataSource = DataSource.MANUAL;
+        }
-        } else {
+
-          activity.dataSource =
+        if (!isDryRun) {
-            this.dataProviderService.getDataSourceForImport();
+          // Insert or update market data
+          const marketDataObjects = assetProfileWithMarketData.marketData.map(
+            (marketData) => {
+              return {
+                ...marketData,
+                symbol,
+                dataSource: assetProfileWithMarketData.dataSource
+              } as Prisma.MarketDataUpdateInput;
+            }
+          );
+
+          await this.marketDataService.updateMany({ data: marketDataObjects });
         }
       }
+    }
 
-      if (
+    for (const activity of activitiesDto) {
-        activity.dataSource === DataSource.MANUAL &&
+      // If an asset profile is created or reused, then update the symbol in all activities
-        !isValidCustomAssetProfileSymbol(activity.symbol)
+      if (assetProfileSymbolMapping[activity.symbol]) {
-      ) {
+        activity.symbol = assetProfileSymbolMapping[activity.symbol];
-        throw new Error(
-          `activities.${index}.symbol ("${activity.symbol}") must be a UUID or start with the prefix "${ghostfolioPrefix}_" for the data source ("${DataSource.MANUAL}")`
-        );
       }
 
       if (!isDryRun) {
@@ -481,11 +541,6 @@ export class ImportService {
           activity.accountId = accountIdMapping[activity.accountId];
         }
 
-        // If a new asset profile is created, then update the symbol in all activities
-        if (assetProfileSymbolMapping[activity.symbol]) {
-          activity.symbol = assetProfileSymbolMapping[activity.symbol];
-        }
-
         // If a new tag is created, then update the tag ID in all activities
         activity.tags = (activity.tags ?? []).map((tagId) => {
           return tagIdMapping[tagId] ?? tagId;

apps/api/src/services/data-provider/data-provider.service.ts

@@ -268,6 +268,15 @@ export class DataProviderService implements OnModuleInit {
       });
 
       if (!assetProfiles[assetProfileIdentifier]) {
+        if (
+          dataSource === DataSource.MANUAL &&
+          !isValidCustomAssetProfileSymbol(symbol)
+        ) {
+          throw new Error(
+            `${activityPath}.symbol ("${symbol}") must be a UUID or start with the prefix "${ghostfolioPrefix}_" for the data source ("${DataSource.MANUAL}")`
+          );
+        }
+
         if (
           (dataSource === DataSource.MANUAL && type === 'BUY') ||
           ['FEE', 'INTEREST', 'LIABILITY'].includes(type)
@@ -291,15 +300,6 @@ export class DataProviderService implements OnModuleInit {
           continue;
         }
 
-        if (
-          dataSource === DataSource.MANUAL &&
-          !isValidCustomAssetProfileSymbol(symbol)
-        ) {
-          throw new Error(
-            `${activityPath}.symbol ("${symbol}") must be a UUID or start with the prefix "${ghostfolioPrefix}_" for the data source ("${DataSource.MANUAL}")`
-          );
-        }
-
         let assetProfile: Partial<SymbolProfile> = { currency };
 
         try {

apps/api/src/services/symbol-profile/symbol-profile.service.ts

@@ -105,6 +105,27 @@ export class SymbolProfileService {
     };
   }
 
+  public async getCustomSymbolProfilesByNames({
+    names,
+    userId
+  }: {
+    names: string[];
+    userId: string;
+  }): Promise<Pick<SymbolProfile, 'name' | 'symbol'>[]> {
+    if (names.length === 0) {
+      return [];
+    }
+
+    return this.prismaService.symbolProfile.findMany({
+      select: { name: true, symbol: true },
+      where: {
+        userId,
+        dataSource: DataSource.MANUAL,
+        name: { in: names }
+      }
+    });
+  }
+
   public async getSymbolProfiles(
     aAssetProfileIdentifiers: AssetProfileIdentifier[]
   ): Promise<EnhancedAssetProfile[]> {