From a868cd99b65cad23da7c416e0dfac96773979bc9 Mon Sep 17 00:00:00 2001 From: visrut Date: Sat, 6 May 2023 22:22:22 +0530 Subject: [PATCH] Feature/improve check for duplicates in import #1932 --- apps/api/src/app/import/import.service.ts | 107 +++++++++++++----- apps/api/src/app/order/create-order.dto.ts | 55 +++++++++ .../order/interfaces/activities.interface.ts | 1 + apps/api/src/app/order/order.service.ts | 3 +- .../activities-table.component.html | 10 ++ .../activities-table.component.ts | 4 +- 6 files changed, 149 insertions(+), 31 deletions(-) diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index c3b8f63b3..2af8016b9 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -1,6 +1,9 @@ import { AccountService } from '@ghostfolio/api/app/account/account.service'; import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto'; -import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto'; +import { + CreateOrderDto, + OrderDto +} from '@ghostfolio/api/app/order/create-order.dto'; import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface'; import { OrderService } from '@ghostfolio/api/app/order/order.service'; import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service'; @@ -94,7 +97,8 @@ export class ImportService { value, assetProfile.currency, userCurrency - ) + ), + isDuplicate: false }; }); } catch { @@ -204,6 +208,11 @@ export class ImportService { userId }); + const activitiesDtoWithDuplication = await this.labelDuplicateActivities({ + activitiesDto, + userId + }); + const accounts = (await this.accountService.getAccounts(userId)).map( (account) => { return { id: account.id, name: account.name }; @@ -228,8 +237,9 @@ export class ImportService { quantity, symbol, type, - unitPrice - } of activitiesDto) { + unitPrice, + isDuplicate + } of activitiesDtoWithDuplication) { const date = parseISO((dateString)); const validatedAccount = accounts.find(({ id }) => { return id === accountId; @@ -279,6 +289,10 @@ export class ImportService { updatedAt: new Date() }; } else { + if (isDuplicate) { + continue; + } + order = await this.orderService.createOrder({ comment, date, @@ -322,13 +336,70 @@ export class ImportService { value, currency, userCurrency - ) + ), + isDuplicate }); } - + console.log(activities); return activities; } + private async labelDuplicateActivities({ + activitiesDto, + userId + }: { + activitiesDto: Partial[]; + userId: string; + }): Promise[]> { + const existingActivities = await this.orderService.orders({ + include: { SymbolProfile: true }, + orderBy: { date: 'desc' }, + where: { userId } + }); + + const activitiesDtoWithDuplication: Partial[] = []; + + for (const activitiesDtoEntry of activitiesDto) { + const { + currency, + dataSource, + date, + fee, + quantity, + symbol, + type, + unitPrice + } = activitiesDtoEntry; + + const duplicateActivity = existingActivities.find((activity) => { + return ( + activity.SymbolProfile.currency === currency && + activity.SymbolProfile.dataSource === dataSource && + isSameDay(activity.date, parseISO((date))) && + activity.fee === fee && + activity.quantity === quantity && + activity.SymbolProfile.symbol === symbol && + activity.type === type && + activity.unitPrice === unitPrice + ); + }); + + if (duplicateActivity) { + activitiesDtoWithDuplication.push({ + ...activitiesDtoEntry, + isDuplicate: true + }); + } else { + activitiesDtoWithDuplication.push({ + ...activitiesDtoEntry, + isDuplicate: false + }); + } + } + + return activitiesDtoWithDuplication; + } + private isUniqueAccount(accounts: AccountWithPlatform[]) { const uniqueAccountIds = new Set(); @@ -355,33 +426,11 @@ export class ImportService { const assetProfiles: { [symbol: string]: Partial; } = {}; - const existingActivities = await this.orderService.orders({ - include: { SymbolProfile: true }, - orderBy: { date: 'desc' }, - where: { userId } - }); for (const [ index, - { currency, dataSource, date, fee, quantity, symbol, type, unitPrice } + { currency, dataSource, symbol } ] of activitiesDto.entries()) { - const duplicateActivity = existingActivities.find((activity) => { - return ( - activity.SymbolProfile.currency === currency && - activity.SymbolProfile.dataSource === dataSource && - isSameDay(activity.date, parseISO((date))) && - activity.fee === fee && - activity.quantity === quantity && - activity.SymbolProfile.symbol === symbol && - activity.type === type && - activity.unitPrice === unitPrice - ); - }); - - if (duplicateActivity) { - throw new Error(`activities.${index} is a duplicate activity`); - } - if (dataSource !== 'MANUAL') { const assetProfile = ( await this.dataProviderService.getAssetProfiles([ diff --git a/apps/api/src/app/order/create-order.dto.ts b/apps/api/src/app/order/create-order.dto.ts index 33e6f9cc8..ba88fe3df 100644 --- a/apps/api/src/app/order/create-order.dto.ts +++ b/apps/api/src/app/order/create-order.dto.ts @@ -8,6 +8,7 @@ import { import { Transform, TransformFnParams } from 'class-transformer'; import { IsArray, + IsBoolean, IsEnum, IsISO8601, IsNumber, @@ -15,6 +16,7 @@ import { IsString } from 'class-validator'; import { isString } from 'lodash'; +import { CreateAccessDto } from '../access/create-access.dto'; export class CreateOrderDto { @IsOptional() @@ -65,3 +67,56 @@ export class CreateOrderDto { @IsNumber() unitPrice: number; } + +export class OrderDto { + @IsOptional() + @IsString() + accountId?: string; + + @IsOptional() + @IsEnum(AssetClass, { each: true }) + assetClass?: AssetClass; + + @IsOptional() + @IsEnum(AssetSubClass, { each: true }) + assetSubClass?: AssetSubClass; + + @IsOptional() + @IsString() + @Transform(({ value }: TransformFnParams) => + isString(value) ? value.trim() : value + ) + comment?: string; + + @IsString() + currency: string; + + @IsOptional() + @IsEnum(DataSource, { each: true }) + dataSource?: DataSource; + + @IsISO8601() + date: string; + + @IsNumber() + fee: number; + + @IsNumber() + quantity: number; + + @IsString() + symbol: string; + + @IsArray() + @IsOptional() + tags?: Tag[]; + + @IsEnum(Type, { each: true }) + type: Type; + + @IsNumber() + unitPrice: number; + + @IsBoolean() + isDuplicate: boolean; +} diff --git a/apps/api/src/app/order/interfaces/activities.interface.ts b/apps/api/src/app/order/interfaces/activities.interface.ts index 31b345b46..c0cd66f8d 100644 --- a/apps/api/src/app/order/interfaces/activities.interface.ts +++ b/apps/api/src/app/order/interfaces/activities.interface.ts @@ -8,4 +8,5 @@ export interface Activity extends OrderWithAccount { feeInBaseCurrency: number; value: number; valueInBaseCurrency: number; + isDuplicate: boolean; } diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 9bc2c6a8e..5cb3d9d54 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -310,7 +310,8 @@ export class OrderService { value, order.SymbolProfile.currency, userCurrency - ) + ), + isDuplicate: false }; }); } diff --git a/libs/ui/src/lib/activities-table/activities-table.component.html b/libs/ui/src/lib/activities-table/activities-table.component.html index 4ac853436..4520c868b 100644 --- a/libs/ui/src/lib/activities-table/activities-table.component.html +++ b/libs/ui/src/lib/activities-table/activities-table.component.html @@ -85,12 +85,22 @@ + diff --git a/libs/ui/src/lib/activities-table/activities-table.component.ts b/libs/ui/src/lib/activities-table/activities-table.component.ts index 1c3fb0367..0ba6e4f58 100644 --- a/libs/ui/src/lib/activities-table/activities-table.component.ts +++ b/libs/ui/src/lib/activities-table/activities-table.component.ts @@ -177,7 +177,9 @@ export class ActivitiesTableComponent implements OnChanges, OnDestroy, OnInit { public onClickActivity(activity: Activity) { if (this.showCheckbox) { - this.selectedRows.toggle(activity); + if (!activity.isDuplicate) { + this.selectedRows.toggle(activity); + } } else if ( this.hasPermissionToOpenDetails && !activity.isDraft &&