diff --git a/CHANGELOG.md b/CHANGELOG.md index 278afc1823..778efd0e24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Activity imports now detect identical rows within the same payload. Identical broker partial fills in the same second can also be flagged as duplicates. - Moved the investment chart component to `@ghostfolio/ui` - Hardened the validation of the date in the endpoint to create an account balance - Upgraded `bootstrap` from version `4.6.2` to `5.3.8` @@ -20,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Scoped import duplicate detection to the same account, including unassigned activities, and aligned currency matching for saved and incoming activities - Fixed an exception in the authentication interceptor when the time zone of the browser is not available ## 3.86.0 - 2026-10-13 diff --git a/apps/api/src/app/import/import.service.spec.ts b/apps/api/src/app/import/import.service.spec.ts index d6d41134a5..06060aea5c 100644 --- a/apps/api/src/app/import/import.service.spec.ts +++ b/apps/api/src/app/import/import.service.spec.ts @@ -33,6 +33,8 @@ describe('ImportService', () => { let addSymbolProfile: jest.Mock; let createActivity: jest.Mock; let gatherSymbols: jest.Mock; + let getAccounts: jest.Mock; + let getActivities: jest.Mock; let importService: ImportService; let updateManyMarketData: jest.Mock; @@ -46,6 +48,8 @@ describe('ImportService', () => { addSymbolProfile = jest.fn(); createActivity = jest.fn(); gatherSymbols = jest.fn(); + getAccounts = jest.fn().mockResolvedValue([]); + getActivities = jest.fn().mockResolvedValue({ activities: [], count: 0 }); updateManyMarketData = jest.fn(); const validateActivities = jest.fn( @@ -56,7 +60,7 @@ describe('ImportService', () => { for (const { currency, dataSource, symbol } of activitiesDto) { assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })] ??= { - currency, + currency: currency ?? 'USD', dataSource, symbol, name: symbol @@ -69,11 +73,11 @@ describe('ImportService', () => { importService = new ImportService( { - accounts: jest.fn().mockResolvedValue([]) + accounts: getAccounts } as unknown as AccountService, { createActivity, - getActivities: jest.fn().mockResolvedValue({ activities: [], count: 0 }) + getActivities } as unknown as ActivitiesService, null, { @@ -508,6 +512,212 @@ describe('ImportService', () => { { dataSource: DataSource.YAHOO, symbol: 'AAPL' } ]); }); + + it('detects duplicate activities within the same import payload', async () => { + const activities = await importActivities({ + activitiesDto: [ + createActivityDto({ + dataSource: DataSource.YAHOO, + date: '2024-01-01T00:00:00.000Z', + symbol: 'AAPL', + type: 'BUY' + }), + createActivityDto({ + dataSource: DataSource.YAHOO, + date: '2024-01-01T00:00:00.000Z', + symbol: 'AAPL', + type: 'BUY' + }) + ], + isDryRun: true + }); + + expect(activities[0].error).toBeUndefined(); + expect(activities[1].error).toEqual({ code: 'IS_DUPLICATE' }); + }); + + it('does not flag identical activities across different accounts as duplicates', async () => { + getAccounts.mockResolvedValue([ + { id: 'account-1', name: 'First account' }, + { id: 'account-2', name: 'Second account' } + ]); + const activities = await importActivities({ + activitiesDto: [ + { + ...createActivityDto({ + dataSource: DataSource.YAHOO, + date: '2024-01-01T00:00:00.000Z', + symbol: 'AAPL', + type: 'BUY' + }), + accountId: 'account-1' + }, + { + ...createActivityDto({ + dataSource: DataSource.YAHOO, + date: '2024-01-01T00:00:00.000Z', + symbol: 'AAPL', + type: 'BUY' + }), + accountId: 'account-2' + } + ], + isDryRun: true + }); + + expect(activities[0].error).toBeUndefined(); + expect(activities[1].error).toBeUndefined(); + }); + + it.each([ + { accountId: 'account-1', isDuplicate: true }, + { accountId: 'account-2', isDuplicate: false }, + { accountId: undefined, isDuplicate: false } + ])( + 'checks saved activities against account $accountId', + async ({ accountId, isDuplicate }) => { + getAccounts.mockResolvedValue([ + { id: 'account-1', name: 'First account' }, + { id: 'account-2', name: 'Second account' } + ]); + const dto = createActivityDto({ + dataSource: DataSource.YAHOO, + date: '2024-01-01T00:00:00.000Z', + symbol: 'AAPL', + type: 'BUY' + }); + getActivities.mockResolvedValue({ + count: 1, + activities: [ + { + ...dto, + accountId: 'account-1', + currency: null, + date: parseISO(dto.date), + assetProfile: { + currency: 'USD', + dataSource: dto.dataSource, + symbol: dto.symbol + } + } + ] + }); + + const activities = await importActivities({ + activitiesDto: [{ ...dto, accountId }], + isDryRun: true + }); + + expect(activities[0].error).toEqual( + isDuplicate ? { code: 'IS_DUPLICATE' } : undefined + ); + expect(createActivity).not.toHaveBeenCalled(); + } + ); + + it.each([ + [undefined, 'account-1'], + ['account-1', undefined] + ])( + 'keeps assigned and unassigned rows distinct in either order (%s, %s)', + async (firstAccountId, secondAccountId) => { + getAccounts.mockResolvedValue([ + { id: 'account-1', name: 'First account' } + ]); + const dto = createActivityDto({ + dataSource: DataSource.YAHOO, + date: '2024-01-01T00:00:00.000Z', + symbol: 'AAPL', + type: 'BUY' + }); + + const activities = await importActivities({ + activitiesDto: [ + { ...dto, accountId: firstAccountId }, + { ...dto, accountId: secondAccountId } + ], + isDryRun: true + }); + + expect(activities.map(({ error }) => error)).toEqual([ + undefined, + undefined + ]); + } + ); + + it('treats null and omitted accounts as the same unassigned account', async () => { + const dto = createActivityDto({ + dataSource: DataSource.YAHOO, + date: '2024-01-01T00:00:00.000Z', + symbol: 'AAPL', + type: 'BUY' + }); + const activities = await importActivities({ + activitiesDto: [{ ...dto, accountId: null }, dto], + isDryRun: true + }); + + expect(activities[0].error).toBeUndefined(); + expect(activities[1].error).toEqual({ code: 'IS_DUPLICATE' }); + }); + + it.each([ + [undefined, 'USD'], + ['USD', undefined] + ])( + 'uses the asset currency consistently within a batch (%s, %s)', + async (firstCurrency, secondCurrency) => { + const dto = createActivityDto({ + dataSource: DataSource.YAHOO, + date: '2024-01-01T00:00:00.000Z', + symbol: 'AAPL', + type: 'BUY' + }); + const activities = await importActivities({ + activitiesDto: [ + { ...dto, currency: firstCurrency }, + { ...dto, currency: secondCurrency } + ], + isDryRun: true + }); + + expect(activities[0].error).toBeUndefined(); + expect(activities[1].error).toEqual({ code: 'IS_DUPLICATE' }); + } + ); + + it('does not replace an explicit saved transaction currency with the asset currency', async () => { + const dto = createActivityDto({ + dataSource: DataSource.YAHOO, + date: '2024-01-01T00:00:00.000Z', + symbol: 'AAPL', + type: 'BUY' + }); + getActivities.mockResolvedValue({ + count: 1, + activities: [ + { + ...dto, + accountId: null, + currency: 'EUR', + date: parseISO(dto.date), + assetProfile: { + currency: 'USD', + dataSource: dto.dataSource, + symbol: dto.symbol + } + } + ] + }); + + const activities = await importActivities({ + activitiesDto: [dto], + isDryRun: true + }); + + expect(activities[0].error).toBeUndefined(); + }); }); function importActivities({ diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index 6f036479c7..e09692f551 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -45,7 +45,7 @@ import { } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; -import { Account, DataSource, Prisma } from '@prisma/client'; +import { Account, DataSource, Prisma, SymbolProfile } from '@prisma/client'; import { Big } from 'big.js'; import { isISIN } from 'class-validator'; import { isSameSecond, parseISO, subDays } from 'date-fns'; @@ -766,6 +766,7 @@ export class ImportService { const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({ activitiesDto, + assetProfiles, userCurrency, userId: user.id }); @@ -1034,10 +1035,14 @@ export class ImportService { private async extendActivitiesWithErrors({ activitiesDto, + assetProfiles, userCurrency, userId }: { activitiesDto: Partial[]; + assetProfiles: { + [assetProfileIdentifier: string]: Partial; + }; userCurrency: string; userId: string; }): Promise<(Partial & Pick)[]> { @@ -1049,6 +1054,9 @@ export class ImportService { withExcludedAccountsAndActivities: true }); + const processedActivities: (Partial & + Pick)[] = []; + return activitiesDto.map( ({ accountId, @@ -1064,12 +1072,18 @@ export class ImportService { unitPrice }) => { const date = parseISO(dateString); + const assetCurrency = + assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })] + ?.currency; - const isDuplicate = existingActivities.some((activity) => { + const isMatchingActivity = ( + activity: Partial & Pick + ) => { return ( + (activity.accountId || null) === (accountId || null) && (activity.comment || null) === (comment || null) && - (activity.currency === currency || - activity.assetProfile.currency === currency) && + (activity.currency ?? activity.assetProfile.currency ?? null) === + (currency ?? assetCurrency ?? null) && activity.assetProfile.dataSource === dataSource && isSameSecond(activity.date, date) && activity.fee === fee && @@ -1078,13 +1092,16 @@ export class ImportService { activity.type === type && activity.unitPrice === unitPrice ); - }); + }; + const isDuplicate = + existingActivities.some(isMatchingActivity) || + processedActivities.some(isMatchingActivity); const error: ActivityError = isDuplicate ? { code: 'IS_DUPLICATE' } : undefined; - return { + const activityExtended = { accountId, comment, currency, @@ -1102,7 +1119,7 @@ export class ImportService { assetSubClass: undefined, countries: undefined, createdAt: undefined, - currency: undefined, + currency: assetCurrency, holdings: undefined, id: undefined, isActive: true, @@ -1111,6 +1128,12 @@ export class ImportService { }, tagIds: tags }; + + if (!error) { + processedActivities.push(activityExtended); + } + + return activityExtended; } ); }