Merge e39156dc71 into 3e6f5b61a1

apps/api/src/app/import/import.service.spec.ts

@@ -0,0 +1,206 @@
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
+import { UserWithSettings } from '@ghostfolio/common/types';
+
+import { DataSource, Prisma } from '@prisma/client';
+
+import { ImportService } from './import.service';
+
+describe('ImportService', () => {
+  const accountService = {
+    getAccounts: jest.fn()
+  };
+  const activitiesService = {
+    createActivity: jest.fn(),
+    getActivities: jest.fn()
+  };
+  const apiService = {};
+  const dataGatheringService = {
+    gatherSymbols: jest.fn()
+  };
+  const dataProviderService = {
+    validateActivities: jest.fn()
+  };
+  const exchangeRateDataService = {
+    toCurrencyAtDate: jest.fn()
+  };
+  const marketDataService = {};
+  const platformService = {};
+  const portfolioService = {};
+  const symbolProfileService = {
+    add: jest.fn(),
+    getSymbolProfiles: jest.fn()
+  };
+  const tagService = {
+    getTagsForUser: jest.fn()
+  };
+  const user = {
+    id: 'user-1',
+    permissions: [],
+    settings: {
+      settings: {
+        baseCurrency: 'USD'
+      }
+    }
+  } as unknown as UserWithSettings;
+
+  let importService: ImportService;
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+
+    accountService.getAccounts.mockResolvedValue([]);
+    activitiesService.getActivities.mockResolvedValue({ activities: [] });
+    activitiesService.createActivity.mockImplementation(async (data) => {
+      const assetProfile = data.SymbolProfile.connectOrCreate.create;
+
+      return {
+        SymbolProfile: assetProfile,
+        date: data.date,
+        id: `activity-${data.date.toISOString()}`,
+        userId: user.id
+      };
+    });
+    exchangeRateDataService.toCurrencyAtDate.mockResolvedValue(1);
+    tagService.getTagsForUser.mockResolvedValue([]);
+
+    importService = new ImportService(
+      accountService as never,
+      activitiesService as never,
+      apiService as never,
+      dataGatheringService as never,
+      dataProviderService as never,
+      exchangeRateDataService as never,
+      marketDataService as never,
+      platformService as never,
+      portfolioService as never,
+      symbolProfileService as never,
+      tagService as never
+    );
+  });
+
+  it('creates a new symbol profile once for duplicate activities', async () => {
+    const symbolProfile = createSymbolProfile();
+    symbolProfileService.getSymbolProfiles.mockResolvedValue([]);
+    symbolProfileService.add.mockResolvedValue(symbolProfile);
+
+    const activities = await importDuplicateActivities(symbolProfile);
+
+    expect(symbolProfileService.add).toHaveBeenCalledTimes(1);
+    expect(activitiesService.createActivity).toHaveBeenCalledTimes(2);
+    expect(activities).toHaveLength(2);
+  });
+
+  it('reuses an existing symbol profile without creating it', async () => {
+    const symbolProfile = createSymbolProfile();
+    symbolProfileService.getSymbolProfiles.mockResolvedValue([symbolProfile]);
+
+    const activities = await importDuplicateActivities(symbolProfile);
+
+    expect(symbolProfileService.add).not.toHaveBeenCalled();
+    expect(activitiesService.createActivity).toHaveBeenCalledTimes(2);
+    expect(activities).toHaveLength(2);
+  });
+
+  it('refetches the canonical profile after a matching P2002 conflict', async () => {
+    const symbolProfile = createSymbolProfile();
+    const conflict = new Prisma.PrismaClientKnownRequestError(
+      'Unique constraint failed',
+      {
+        clientVersion: 'test',
+        code: 'P2002',
+        meta: {
+          driverAdapterError: {
+            cause: {
+              constraint: {
+                fields: ['"dataSource"', 'symbol']
+              }
+            }
+          }
+        }
+      }
+    );
+    symbolProfileService.getSymbolProfiles
+      .mockResolvedValueOnce([])
+      .mockResolvedValueOnce([symbolProfile]);
+    symbolProfileService.add.mockRejectedValue(conflict);
+
+    const activities = await importDuplicateActivities(symbolProfile);
+
+    expect(symbolProfileService.getSymbolProfiles).toHaveBeenCalledTimes(2);
+    expect(activitiesService.createActivity).toHaveBeenCalledTimes(2);
+    expect(activities).toHaveLength(2);
+  });
+
+  it('rethrows an unrelated P2002 conflict', async () => {
+    const symbolProfile = createSymbolProfile();
+    const conflict = new Prisma.PrismaClientKnownRequestError(
+      'Unique constraint failed',
+      {
+        clientVersion: 'test',
+        code: 'P2002',
+        meta: {
+          target: ['id']
+        }
+      }
+    );
+    symbolProfileService.getSymbolProfiles.mockResolvedValue([]);
+    symbolProfileService.add.mockRejectedValue(conflict);
+
+    await expect(importDuplicateActivities(symbolProfile)).rejects.toBe(
+      conflict
+    );
+    expect(activitiesService.createActivity).not.toHaveBeenCalled();
+  });
+
+  function createSymbolProfile() {
+    return {
+      currency: 'USD',
+      dataSource: DataSource.MANUAL,
+      name: 'Repeated fee',
+      symbol: 'GF_REPEATED_FEE'
+    };
+  }
+
+  async function importDuplicateActivities(
+    symbolProfile: ReturnType<typeof createSymbolProfile>
+  ) {
+    const activitiesDto = [
+      {
+        currency: symbolProfile.currency,
+        dataSource: symbolProfile.dataSource,
+        date: '2026-01-01T00:00:00.000Z',
+        fee: 0,
+        quantity: 1,
+        symbol: symbolProfile.symbol,
+        tags: [],
+        type: 'FEE' as const,
+        unitPrice: 1
+      },
+      {
+        currency: symbolProfile.currency,
+        dataSource: symbolProfile.dataSource,
+        date: '2026-01-02T00:00:00.000Z',
+        fee: 0,
+        quantity: 1,
+        symbol: symbolProfile.symbol,
+        tags: [],
+        type: 'FEE' as const,
+        unitPrice: 1
+      }
+    ];
+    const assetProfileIdentifier = getAssetProfileIdentifier(symbolProfile);
+    dataProviderService.validateActivities.mockResolvedValue({
+      [assetProfileIdentifier]: symbolProfile
+    });
+
+    return importService.import({
+      activitiesDto,
+      maxActivitiesToImport: 10,
+      user,
+      accountsWithBalancesDto: [],
+      assetProfilesWithMarketDataDto: [],
+      platformsDto: [],
+      tagsDto: []
+    });
+  }
+});

apps/api/src/app/import/import.service.ts

@@ -40,7 +40,7 @@ import {
 } from '@ghostfolio/common/types';
 
 import { Injectable } from '@nestjs/common';
-import { Account, DataSource, Prisma } from '@prisma/client';
+import { Account, DataSource, Prisma, SymbolProfile } from '@prisma/client';
 import { Big } from 'big.js';
 import { isISIN } from 'class-validator';
 import { isSameSecond, parseISO } from 'date-fns';
@@ -720,6 +720,10 @@ export class ImportService {
     }) ?? { id: TAG_ID_DRAFT, name: 'DRAFT' };
 
     const activities: Activity[] = [];
+    const pendingSymbolProfiles = new Map<
+      string,
+      Promise<AssetProfileIdentifier>
+    >();
 
     for (const activity of activitiesExtendedWithErrors) {
       const accountId = activity.accountId;
@@ -748,7 +752,6 @@ export class ImportService {
         countries,
         createdAt,
         cusip,
-        dataSource,
         figi,
         figiComposite,
         figiShareClass,
@@ -759,11 +762,12 @@ export class ImportService {
         name,
         scraperConfiguration,
         sectors,
-        symbol,
         symbolMapping,
         url,
         updatedAt
       } = assetProfile;
+      let dataSource = assetProfile.dataSource;
+      let symbol = assetProfile.symbol;
       const validatedAccount = accounts.find(({ id }) => {
         return id === accountId;
       });
@@ -838,6 +842,22 @@ export class ImportService {
           continue;
         }
 
+        const symbolProfile = await this.resolveSymbolProfile({
+          assetProfile,
+          assetProfileIdentifier: {
+            dataSource: activity.assetProfile.dataSource,
+            symbol: activity.assetProfile.symbol
+          },
+          pendingSymbolProfiles,
+          type,
+          userId: user.id
+        });
+
+        assetProfile.dataSource = symbolProfile.dataSource;
+        assetProfile.symbol = symbolProfile.symbol;
+        dataSource = symbolProfile.dataSource;
+        symbol = symbolProfile.symbol;
+
         order = await this.activitiesService.createActivity({
           comment,
           currency,
@@ -926,6 +946,153 @@ export class ImportService {
     return activities;
   }
 
+  private async createOrGetSymbolProfile({
+    assetProfile,
+    assetProfileIdentifier: requestedAssetProfileIdentifier,
+    type,
+    userId
+  }: {
+    assetProfile: Partial<SymbolProfile>;
+    assetProfileIdentifier: AssetProfileIdentifier;
+    type: Activity['type'];
+    userId: string;
+  }): Promise<AssetProfileIdentifier> {
+    let dataSource = requestedAssetProfileIdentifier.dataSource;
+    let symbol = requestedAssetProfileIdentifier.symbol;
+
+    if (
+      (requestedAssetProfileIdentifier.dataSource === DataSource.MANUAL &&
+        type === 'BUY') ||
+      NON_INVESTMENT_ACTIVITY_TYPES.includes(type)
+    ) {
+      dataSource = DataSource.MANUAL;
+
+      if (!isValidCustomAssetProfileSymbol(symbol)) {
+        symbol = randomUUID();
+      }
+    }
+
+    const symbolProfileIdentifier = {
+      dataSource,
+      symbol
+    };
+    const [existingSymbolProfile] =
+      await this.symbolProfileService.getSymbolProfiles([
+        symbolProfileIdentifier
+      ]);
+
+    if (existingSymbolProfile) {
+      return {
+        dataSource: existingSymbolProfile.dataSource,
+        symbol: existingSymbolProfile.symbol
+      };
+    }
+
+    try {
+      const newSymbolProfile = await this.symbolProfileService.add({
+        ...symbolProfileIdentifier,
+        currency: assetProfile.currency,
+        name: assetProfile.name,
+        ...(dataSource === DataSource.MANUAL
+          ? { user: { connect: { id: userId } } }
+          : {})
+      });
+
+      return {
+        dataSource: newSymbolProfile.dataSource,
+        symbol: newSymbolProfile.symbol
+      };
+    } catch (error) {
+      if (!this.isSymbolProfileIdentifierConflict(error)) {
+        throw error;
+      }
+
+      const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
+        [symbolProfileIdentifier]
+      );
+
+      if (!symbolProfile) {
+        throw error;
+      }
+
+      return {
+        dataSource: symbolProfile.dataSource,
+        symbol: symbolProfile.symbol
+      };
+    }
+  }
+
+  private isSymbolProfileIdentifierConflict(error: unknown): boolean {
+    if (
+      !(error instanceof Prisma.PrismaClientKnownRequestError) ||
+      error.code !== 'P2002'
+    ) {
+      return false;
+    }
+
+    const target = error.meta?.target;
+
+    if (
+      typeof target === 'string' &&
+      target.replace(/"/g, '') === 'SymbolProfile_dataSource_symbol_key'
+    ) {
+      return true;
+    }
+
+    const adapterFields = (
+      error.meta?.driverAdapterError as {
+        cause?: {
+          constraint?: {
+            fields?: unknown;
+          };
+        };
+      }
+    )?.cause?.constraint?.fields;
+    const fields = Array.isArray(target)
+      ? target
+      : Array.isArray(adapterFields)
+        ? adapterFields
+        : [];
+    const normalizedFields = fields.map((field) => {
+      return String(field).replace(/"/g, '');
+    });
+
+    return (
+      normalizedFields.length === 2 &&
+      normalizedFields.includes('dataSource') &&
+      normalizedFields.includes('symbol')
+    );
+  }
+
+  private resolveSymbolProfile({
+    assetProfile,
+    assetProfileIdentifier,
+    pendingSymbolProfiles,
+    type,
+    userId
+  }: {
+    assetProfile: Partial<SymbolProfile>;
+    assetProfileIdentifier: AssetProfileIdentifier;
+    pendingSymbolProfiles: Map<string, Promise<AssetProfileIdentifier>>;
+    type: Activity['type'];
+    userId: string;
+  }): Promise<AssetProfileIdentifier> {
+    const identifier = getAssetProfileIdentifier(assetProfileIdentifier);
+    let pendingSymbolProfile = pendingSymbolProfiles.get(identifier);
+
+    if (!pendingSymbolProfile) {
+      pendingSymbolProfile = this.createOrGetSymbolProfile({
+        assetProfile,
+        assetProfileIdentifier,
+        type,
+        userId
+      });
+      pendingSymbolProfiles.set(identifier, pendingSymbolProfile);
+    }
+
+    return pendingSymbolProfile;
+  }
+
   private async extendActivitiesWithErrors({
     activitiesDto,
     userCurrency,