Feature/improve check for duplicates in import #1932

apps/api/src/app/import/import.service.ts

@@ -1,6 +1,9 @@
 import { AccountService } from '@ghostfolio/api/app/account/account.service';
 import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
-import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
+import {
+  CreateOrderDto,
+  OrderDto
+} from '@ghostfolio/api/app/order/create-order.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { OrderService } from '@ghostfolio/api/app/order/order.service';
 import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service';
@@ -94,7 +97,8 @@ export class ImportService {
             value,
             assetProfile.currency,
             userCurrency
-          )
+          ),
+          isDuplicate: false
         };
       });
     } catch {
@@ -204,6 +208,11 @@ export class ImportService {
       userId
     });
 
+    const activitiesDtoWithDuplication = await this.labelDuplicateActivities({
+      activitiesDto,
+      userId
+    });
+
     const accounts = (await this.accountService.getAccounts(userId)).map(
       (account) => {
         return { id: account.id, name: account.name };
@@ -228,8 +237,9 @@ export class ImportService {
       quantity,
       symbol,
       type,
-      unitPrice
+      unitPrice,
-    } of activitiesDto) {
+      isDuplicate
+    } of activitiesDtoWithDuplication) {
       const date = parseISO(<string>(<unknown>dateString));
       const validatedAccount = accounts.find(({ id }) => {
         return id === accountId;
@@ -279,6 +289,10 @@ export class ImportService {
           updatedAt: new Date()
         };
       } else {
+        if (isDuplicate) {
+          continue;
+        }
+
         order = await this.orderService.createOrder({
           comment,
           date,
@@ -322,13 +336,70 @@ export class ImportService {
           value,
           currency,
           userCurrency
-        )
+        ),
+        isDuplicate
       });
     }
-
+    console.log(activities);
     return activities;
   }
 
+  private async labelDuplicateActivities({
+    activitiesDto,
+    userId
+  }: {
+    activitiesDto: Partial<CreateOrderDto>[];
+    userId: string;
+  }): Promise<Partial<OrderDto>[]> {
+    const existingActivities = await this.orderService.orders({
+      include: { SymbolProfile: true },
+      orderBy: { date: 'desc' },
+      where: { userId }
+    });
+
+    const activitiesDtoWithDuplication: Partial<OrderDto>[] = [];
+
+    for (const activitiesDtoEntry of activitiesDto) {
+      const {
+        currency,
+        dataSource,
+        date,
+        fee,
+        quantity,
+        symbol,
+        type,
+        unitPrice
+      } = activitiesDtoEntry;
+
+      const duplicateActivity = existingActivities.find((activity) => {
+        return (
+          activity.SymbolProfile.currency === currency &&
+          activity.SymbolProfile.dataSource === dataSource &&
+          isSameDay(activity.date, parseISO(<string>(<unknown>date))) &&
+          activity.fee === fee &&
+          activity.quantity === quantity &&
+          activity.SymbolProfile.symbol === symbol &&
+          activity.type === type &&
+          activity.unitPrice === unitPrice
+        );
+      });
+
+      if (duplicateActivity) {
+        activitiesDtoWithDuplication.push({
+          ...activitiesDtoEntry,
+          isDuplicate: true
+        });
+      } else {
+        activitiesDtoWithDuplication.push({
+          ...activitiesDtoEntry,
+          isDuplicate: false
+        });
+      }
+    }
+
+    return activitiesDtoWithDuplication;
+  }
+
   private isUniqueAccount(accounts: AccountWithPlatform[]) {
     const uniqueAccountIds = new Set<string>();
 
@@ -355,33 +426,11 @@ export class ImportService {
     const assetProfiles: {
       [symbol: string]: Partial<SymbolProfile>;
     } = {};
-    const existingActivities = await this.orderService.orders({
-      include: { SymbolProfile: true },
-      orderBy: { date: 'desc' },
-      where: { userId }
-    });
 
     for (const [
       index,
-      { currency, dataSource, date, fee, quantity, symbol, type, unitPrice }
+      { currency, dataSource, symbol }
     ] of activitiesDto.entries()) {
-      const duplicateActivity = existingActivities.find((activity) => {
-        return (
-          activity.SymbolProfile.currency === currency &&
-          activity.SymbolProfile.dataSource === dataSource &&
-          isSameDay(activity.date, parseISO(<string>(<unknown>date))) &&
-          activity.fee === fee &&
-          activity.quantity === quantity &&
-          activity.SymbolProfile.symbol === symbol &&
-          activity.type === type &&
-          activity.unitPrice === unitPrice
-        );
-      });
-
-      if (duplicateActivity) {
-        throw new Error(`activities.${index} is a duplicate activity`);
-      }
-
       if (dataSource !== 'MANUAL') {
         const assetProfile = (
           await this.dataProviderService.getAssetProfiles([

apps/api/src/app/order/create-order.dto.ts

@@ -8,6 +8,7 @@ import {
 import { Transform, TransformFnParams } from 'class-transformer';
 import {
   IsArray,
+  IsBoolean,
   IsEnum,
   IsISO8601,
   IsNumber,
@@ -15,6 +16,7 @@ import {
   IsString
 } from 'class-validator';
 import { isString } from 'lodash';
+import { CreateAccessDto } from '../access/create-access.dto';
 
 export class CreateOrderDto {
   @IsOptional()
@@ -65,3 +67,56 @@ export class CreateOrderDto {
   @IsNumber()
   unitPrice: number;
 }
+
+export class OrderDto {
+  @IsOptional()
+  @IsString()
+  accountId?: string;
+
+  @IsOptional()
+  @IsEnum(AssetClass, { each: true })
+  assetClass?: AssetClass;
+
+  @IsOptional()
+  @IsEnum(AssetSubClass, { each: true })
+  assetSubClass?: AssetSubClass;
+
+  @IsOptional()
+  @IsString()
+  @Transform(({ value }: TransformFnParams) =>
+    isString(value) ? value.trim() : value
+  )
+  comment?: string;
+
+  @IsString()
+  currency: string;
+
+  @IsOptional()
+  @IsEnum(DataSource, { each: true })
+  dataSource?: DataSource;
+
+  @IsISO8601()
+  date: string;
+
+  @IsNumber()
+  fee: number;
+
+  @IsNumber()
+  quantity: number;
+
+  @IsString()
+  symbol: string;
+
+  @IsArray()
+  @IsOptional()
+  tags?: Tag[];
+
+  @IsEnum(Type, { each: true })
+  type: Type;
+
+  @IsNumber()
+  unitPrice: number;
+
+  @IsBoolean()
+  isDuplicate: boolean;
+}

apps/api/src/app/order/interfaces/activities.interface.ts

@@ -8,4 +8,5 @@ export interface Activity extends OrderWithAccount {
   feeInBaseCurrency: number;
   value: number;
   valueInBaseCurrency: number;
+  isDuplicate: boolean;
 }

apps/api/src/app/order/order.service.ts

@@ -310,7 +310,8 @@ export class OrderService {
             value,
             order.SymbolProfile.currency,
             userCurrency
-          )
+          ),
+          isDuplicate: false
         };
       });
   }

libs/ui/src/lib/activities-table/activities-table.component.html

@@ -85,12 +85,22 @@
       </th>
       <td *matCellDef="let element" class="px-1" mat-cell>
         <mat-checkbox
+          *ngIf="!element.isDuplicate"
           class="mt-2"
           color="primary"
           [checked]="selectedRows.isSelected(element)"
           (change)="$event ? selectedRows.toggle(element) : null"
           (click)="$event.stopPropagation()"
         ></mat-checkbox>
+        <mat-checkbox
+          *ngIf="element.isDuplicate"
+          class="mt-2"
+          color="primary"
+          [checked]="false"
+          [disabled]="true"
+          (change)="(null)"
+          (click)="(null)"
+        ></mat-checkbox>
       </td>
       <td *matFooterCellDef class="px-1" mat-footer-cell></td>
     </ng-container>

libs/ui/src/lib/activities-table/activities-table.component.ts

@@ -177,7 +177,9 @@ export class ActivitiesTableComponent implements OnChanges, OnDestroy, OnInit {
 
   public onClickActivity(activity: Activity) {
     if (this.showCheckbox) {
-      this.selectedRows.toggle(activity);
+      if (!activity.isDuplicate) {
+        this.selectedRows.toggle(activity);
+      }
     } else if (
       this.hasPermissionToOpenDetails &&
       !activity.isDraft &&