Feature/improve check for duplicates in import #1932 (#1940)

* Improve check for duplicates in import

* Update changelog

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Improved the preview step of the activities import by unchecking duplicates
+
 ## 1.267.0 - 2023-05-07
 
 ### Added

apps/api/src/app/import/import.service.ts

@@ -84,6 +84,7 @@ export class ImportService {
           feeInBaseCurrency: 0,
           id: assetProfile.id,
           isDraft: false,
+          isDuplicate: false, // TODO: Use evaluated state
           SymbolProfile: <SymbolProfile>(<unknown>assetProfile),
           symbolProfileId: assetProfile.id,
           type: 'DIVIDEND',
@@ -204,9 +205,14 @@ export class ImportService {
       userId
     });
 
+    const activitiesMarkedAsDuplicates = await this.markActivitiesAsDuplicates({
+      activitiesDto,
+      userId
+    });
+
     const accounts = (await this.accountService.getAccounts(userId)).map(
-      (account) => {
+      ({ id, name }) => {
-        return { id: account.id, name: account.name };
+        return { id, name };
       }
     );
 
@@ -221,16 +227,14 @@ export class ImportService {
     for (const {
       accountId,
       comment,
-      currency,
+      date,
-      dataSource,
-      date: dateString,
       fee,
+      isDuplicate,
       quantity,
-      symbol,
+      SymbolProfile: assetProfile,
       type,
       unitPrice
-    } of activitiesDto) {
+    } of activitiesMarkedAsDuplicates) {
-      const date = parseISO(<string>(<unknown>dateString));
       const validatedAccount = accounts.find(({ id }) => {
         return id === accountId;
       });
@@ -256,29 +260,33 @@ export class ImportService {
           id: uuidv4(),
           isDraft: isAfter(date, endOfToday()),
           SymbolProfile: {
-            currency,
+            assetClass: assetProfile.assetClass,
-            dataSource,
+            assetSubClass: assetProfile.assetSubClass,
-            symbol,
+            comment: assetProfile.comment,
-            assetClass: null,
+            countries: assetProfile.countries,
-            assetSubClass: null,
+            createdAt: assetProfile.createdAt,
-            comment: null,
+            currency: assetProfile.currency,
-            countries: null,
+            dataSource: assetProfile.dataSource,
-            createdAt: undefined,
+            id: assetProfile.id,
-            id: undefined,
+            isin: assetProfile.isin,
-            isin: null,
+            name: assetProfile.name,
-            name: null,
+            scraperConfiguration: assetProfile.scraperConfiguration,
-            scraperConfiguration: null,
+            sectors: assetProfile.sectors,
-            sectors: null,
+            symbol: assetProfile.currency,
-            symbolMapping: null,
+            symbolMapping: assetProfile.symbolMapping,
-            updatedAt: undefined,
+            updatedAt: assetProfile.updatedAt,
-            url: null,
+            url: assetProfile.url,
-            ...assetProfiles[symbol]
+            ...assetProfiles[assetProfile.symbol]
           },
           Account: validatedAccount,
           symbolProfileId: undefined,
           updatedAt: new Date()
         };
       } else {
+        if (isDuplicate) {
+          continue;
+        }
+
         order = await this.orderService.createOrder({
           comment,
           date,
@@ -291,14 +299,14 @@ export class ImportService {
           SymbolProfile: {
             connectOrCreate: {
               create: {
-                currency,
+                currency: assetProfile.currency,
-                dataSource,
+                dataSource: assetProfile.dataSource,
-                symbol
+                symbol: assetProfile.symbol
               },
               where: {
                 dataSource_symbol: {
-                  dataSource,
+                  dataSource: assetProfile.dataSource,
-                  symbol
+                  symbol: assetProfile.symbol
                 }
               }
             }
@@ -313,15 +321,16 @@ export class ImportService {
       //@ts-ignore
       activities.push({
         ...order,
+        isDuplicate,
         value,
         feeInBaseCurrency: this.exchangeRateDataService.toCurrency(
           fee,
-          currency,
+          assetProfile.currency,
           userCurrency
         ),
         valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
           value,
-          currency,
+          assetProfile.currency,
           userCurrency
         )
       });
@@ -340,6 +349,78 @@ export class ImportService {
     return uniqueAccountIds.size === 1;
   }
 
+  private async markActivitiesAsDuplicates({
+    activitiesDto,
+    userId
+  }: {
+    activitiesDto: Partial<CreateOrderDto>[];
+    userId: string;
+  }): Promise<Partial<Activity>[]> {
+    const existingActivities = await this.orderService.orders({
+      include: { SymbolProfile: true },
+      orderBy: { date: 'desc' },
+      where: { userId }
+    });
+
+    return activitiesDto.map(
+      ({
+        accountId,
+        comment,
+        currency,
+        dataSource,
+        date: dateString,
+        fee,
+        quantity,
+        symbol,
+        type,
+        unitPrice
+      }) => {
+        const date = parseISO(<string>(<unknown>dateString));
+        const isDuplicate = existingActivities.some((activity) => {
+          return (
+            activity.SymbolProfile.currency === currency &&
+            activity.SymbolProfile.dataSource === dataSource &&
+            isSameDay(activity.date, date) &&
+            activity.fee === fee &&
+            activity.quantity === quantity &&
+            activity.SymbolProfile.symbol === symbol &&
+            activity.type === type &&
+            activity.unitPrice === unitPrice
+          );
+        });
+
+        return {
+          accountId,
+          comment,
+          date,
+          fee,
+          isDuplicate,
+          quantity,
+          type,
+          unitPrice,
+          SymbolProfile: {
+            currency,
+            dataSource,
+            symbol,
+            assetClass: null,
+            assetSubClass: null,
+            comment: null,
+            countries: null,
+            createdAt: undefined,
+            id: undefined,
+            isin: null,
+            name: null,
+            scraperConfiguration: null,
+            sectors: null,
+            symbolMapping: null,
+            updatedAt: undefined,
+            url: null
+          }
+        };
+      }
+    );
+  }
+
   private async validateActivities({
     activitiesDto,
     maxActivitiesToImport,
@@ -356,33 +437,11 @@ export class ImportService {
     const assetProfiles: {
       [symbol: string]: Partial<SymbolProfile>;
     } = {};
-    const existingActivities = await this.orderService.orders({
-      include: { SymbolProfile: true },
-      orderBy: { date: 'desc' },
-      where: { userId }
-    });
 
     for (const [
       index,
-      { currency, dataSource, date, fee, quantity, symbol, type, unitPrice }
+      { currency, dataSource, symbol }
     ] of activitiesDto.entries()) {
-      const duplicateActivity = existingActivities.find((activity) => {
-        return (
-          activity.SymbolProfile.currency === currency &&
-          activity.SymbolProfile.dataSource === dataSource &&
-          isSameDay(activity.date, parseISO(<string>(<unknown>date))) &&
-          activity.fee === fee &&
-          activity.quantity === quantity &&
-          activity.SymbolProfile.symbol === symbol &&
-          activity.type === type &&
-          activity.unitPrice === unitPrice
-        );
-      });
-
-      if (duplicateActivity) {
-        throw new Error(`activities.${index} is a duplicate activity`);
-      }
-
       if (dataSource !== 'MANUAL') {
         const assetProfile = (
           await this.dataProviderService.getAssetProfiles([

apps/api/src/app/order/interfaces/activities.interface.ts

@@ -6,6 +6,7 @@ export interface Activities {
 
 export interface Activity extends OrderWithAccount {
   feeInBaseCurrency: number;
+  isDuplicate: boolean;
   updateAccountBalance?: boolean;
   value: number;
   valueInBaseCurrency: number;

apps/api/src/app/order/order.service.ts

@@ -333,6 +333,7 @@ export class OrderService {
             order.SymbolProfile.currency,
             userCurrency
           ),
+          isDuplicate: false,
           valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
             value,
             order.SymbolProfile.currency,

libs/ui/src/lib/activities-table/activities-table.component.html

@@ -76,7 +76,6 @@
     <ng-container matColumnDef="select">
       <th *matHeaderCellDef class="px-1" mat-header-cell>
         <mat-checkbox
-          class="mt-2"
           color="primary"
           [checked]="selectedRows.hasValue() && areAllRowsSelected()"
           [indeterminate]="selectedRows.hasValue() && !areAllRowsSelected()"
@@ -85,9 +84,11 @@
       </th>
       <td *matCellDef="let element" class="px-1" mat-cell>
         <mat-checkbox
-          class="mt-2"
           color="primary"
-          [checked]="selectedRows.isSelected(element)"
+          [checked]="
+            element.isDuplicate ? false : selectedRows.isSelected(element)
+          "
+          [disabled]="element.isDuplicate"
           (change)="$event ? selectedRows.toggle(element) : null"
           (click)="$event.stopPropagation()"
         ></mat-checkbox>

libs/ui/src/lib/activities-table/activities-table.component.ts

@@ -177,7 +177,9 @@ export class ActivitiesTableComponent implements OnChanges, OnDestroy, OnInit {
 
   public onClickActivity(activity: Activity) {
     if (this.showCheckbox) {
-      this.selectedRows.toggle(activity);
+      if (!activity.isDuplicate) {
+        this.selectedRows.toggle(activity);
+      }
     } else if (
       this.hasPermissionToOpenDetails &&
       !activity.isDraft &&