Browse Source

fix(import): unify account and currency duplicate matching

pull/8119/head
hrushikeshramilla 1 day ago
parent
commit
045cb64c82
  1. 2
      CHANGELOG.md
  2. 164
      apps/api/src/app/import/import.service.spec.ts
  3. 42
      apps/api/src/app/import/import.service.ts

2
CHANGELOG.md

@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Activity imports now detect identical rows within the same payload. Identical broker partial fills in the same second can also be flagged as duplicates.
- Moved the investment chart component to `@ghostfolio/ui`
- Hardened the validation of the date in the endpoint to create an account balance
- Upgraded `bootstrap` from version `4.6.2` to `5.3.8`
@ -20,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- Scoped import duplicate detection to the same account, including unassigned activities, and aligned currency matching for saved and incoming activities
- Fixed an exception in the authentication interceptor when the time zone of the browser is not available
## 3.86.0 - 2026-10-13

164
apps/api/src/app/import/import.service.spec.ts

@ -33,6 +33,8 @@ describe('ImportService', () => {
let addSymbolProfile: jest.Mock;
let createActivity: jest.Mock;
let gatherSymbols: jest.Mock;
let getAccounts: jest.Mock;
let getActivities: jest.Mock;
let importService: ImportService;
let updateManyMarketData: jest.Mock;
@ -46,6 +48,8 @@ describe('ImportService', () => {
addSymbolProfile = jest.fn();
createActivity = jest.fn();
gatherSymbols = jest.fn();
getAccounts = jest.fn().mockResolvedValue([]);
getActivities = jest.fn().mockResolvedValue({ activities: [], count: 0 });
updateManyMarketData = jest.fn();
const validateActivities = jest.fn(
@ -56,7 +60,7 @@ describe('ImportService', () => {
for (const { currency, dataSource, symbol } of activitiesDto) {
assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })] ??= {
currency,
currency: currency ?? 'USD',
dataSource,
symbol,
name: symbol
@ -69,11 +73,11 @@ describe('ImportService', () => {
importService = new ImportService(
{
accounts: jest.fn().mockResolvedValue([])
accounts: getAccounts
} as unknown as AccountService,
{
createActivity,
getActivities: jest.fn().mockResolvedValue({ activities: [], count: 0 })
getActivities
} as unknown as ActivitiesService,
null,
{
@ -533,6 +537,10 @@ describe('ImportService', () => {
});
it('does not flag identical activities across different accounts as duplicates', async () => {
getAccounts.mockResolvedValue([
{ id: 'account-1', name: 'First account' },
{ id: 'account-2', name: 'Second account' }
]);
const activities = await importActivities({
activitiesDto: [
{
@ -560,6 +568,156 @@ describe('ImportService', () => {
expect(activities[0].error).toBeUndefined();
expect(activities[1].error).toBeUndefined();
});
it.each([
{ accountId: 'account-1', isDuplicate: true },
{ accountId: 'account-2', isDuplicate: false },
{ accountId: undefined, isDuplicate: false }
])(
'checks saved activities against account $accountId',
async ({ accountId, isDuplicate }) => {
getAccounts.mockResolvedValue([
{ id: 'account-1', name: 'First account' },
{ id: 'account-2', name: 'Second account' }
]);
const dto = createActivityDto({
dataSource: DataSource.YAHOO,
date: '2024-01-01T00:00:00.000Z',
symbol: 'AAPL',
type: 'BUY'
});
getActivities.mockResolvedValue({
count: 1,
activities: [
{
...dto,
accountId: 'account-1',
currency: null,
date: parseISO(dto.date),
assetProfile: {
currency: 'USD',
dataSource: dto.dataSource,
symbol: dto.symbol
}
}
]
});
const activities = await importActivities({
activitiesDto: [{ ...dto, accountId }],
isDryRun: true
});
expect(activities[0].error).toEqual(
isDuplicate ? { code: 'IS_DUPLICATE' } : undefined
);
expect(createActivity).not.toHaveBeenCalled();
}
);
it.each([
[undefined, 'account-1'],
['account-1', undefined]
])(
'keeps assigned and unassigned rows distinct in either order (%s, %s)',
async (firstAccountId, secondAccountId) => {
getAccounts.mockResolvedValue([
{ id: 'account-1', name: 'First account' }
]);
const dto = createActivityDto({
dataSource: DataSource.YAHOO,
date: '2024-01-01T00:00:00.000Z',
symbol: 'AAPL',
type: 'BUY'
});
const activities = await importActivities({
activitiesDto: [
{ ...dto, accountId: firstAccountId },
{ ...dto, accountId: secondAccountId }
],
isDryRun: true
});
expect(activities.map(({ error }) => error)).toEqual([
undefined,
undefined
]);
}
);
it('treats null and omitted accounts as the same unassigned account', async () => {
const dto = createActivityDto({
dataSource: DataSource.YAHOO,
date: '2024-01-01T00:00:00.000Z',
symbol: 'AAPL',
type: 'BUY'
});
const activities = await importActivities({
activitiesDto: [{ ...dto, accountId: null }, dto],
isDryRun: true
});
expect(activities[0].error).toBeUndefined();
expect(activities[1].error).toEqual({ code: 'IS_DUPLICATE' });
});
it.each([
[undefined, 'USD'],
['USD', undefined]
])(
'uses the asset currency consistently within a batch (%s, %s)',
async (firstCurrency, secondCurrency) => {
const dto = createActivityDto({
dataSource: DataSource.YAHOO,
date: '2024-01-01T00:00:00.000Z',
symbol: 'AAPL',
type: 'BUY'
});
const activities = await importActivities({
activitiesDto: [
{ ...dto, currency: firstCurrency },
{ ...dto, currency: secondCurrency }
],
isDryRun: true
});
expect(activities[0].error).toBeUndefined();
expect(activities[1].error).toEqual({ code: 'IS_DUPLICATE' });
}
);
it('does not replace an explicit saved transaction currency with the asset currency', async () => {
const dto = createActivityDto({
dataSource: DataSource.YAHOO,
date: '2024-01-01T00:00:00.000Z',
symbol: 'AAPL',
type: 'BUY'
});
getActivities.mockResolvedValue({
count: 1,
activities: [
{
...dto,
accountId: null,
currency: 'EUR',
date: parseISO(dto.date),
assetProfile: {
currency: 'USD',
dataSource: dto.dataSource,
symbol: dto.symbol
}
}
]
});
const activities = await importActivities({
activitiesDto: [dto],
isDryRun: true
});
expect(activities[0].error).toBeUndefined();
});
});
function importActivities({

42
apps/api/src/app/import/import.service.ts

@ -45,7 +45,7 @@ import {
} from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { Account, DataSource, Prisma } from '@prisma/client';
import { Account, DataSource, Prisma, SymbolProfile } from '@prisma/client';
import { Big } from 'big.js';
import { isISIN } from 'class-validator';
import { isSameSecond, parseISO, subDays } from 'date-fns';
@ -766,6 +766,7 @@ export class ImportService {
const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({
activitiesDto,
assetProfiles,
userCurrency,
userId: user.id
});
@ -1034,10 +1035,14 @@ export class ImportService {
private async extendActivitiesWithErrors({
activitiesDto,
assetProfiles,
userCurrency,
userId
}: {
activitiesDto: Partial<CreateOrderDto>[];
assetProfiles: {
[assetProfileIdentifier: string]: Partial<SymbolProfile>;
};
userCurrency: string;
userId: string;
}): Promise<(Partial<Activity> & Pick<Activity, 'assetProfile'>)[]> {
@ -1067,28 +1072,18 @@ export class ImportService {
unitPrice
}) => {
const date = parseISO(dateString);
const assetCurrency =
assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })]
?.currency;
const isDuplicate =
existingActivities.some((activity) => {
return (
(!accountId || activity.accountId === accountId) &&
(activity.comment || null) === (comment || null) &&
((activity.currency || null) === (currency || null) ||
activity.assetProfile.currency === currency) &&
activity.assetProfile.dataSource === dataSource &&
isSameSecond(activity.date, date) &&
activity.fee === fee &&
activity.quantity === quantity &&
activity.assetProfile.symbol === symbol &&
activity.type === type &&
activity.unitPrice === unitPrice
);
}) ||
processedActivities.some((activity) => {
const isMatchingActivity = (
activity: Partial<Activity> & Pick<Activity, 'assetProfile'>
) => {
return (
(!accountId || activity.accountId === accountId) &&
(activity.accountId || null) === (accountId || null) &&
(activity.comment || null) === (comment || null) &&
(activity.currency || null) === (currency || null) &&
(activity.currency ?? activity.assetProfile.currency ?? null) ===
(currency ?? assetCurrency ?? null) &&
activity.assetProfile.dataSource === dataSource &&
isSameSecond(activity.date, date) &&
activity.fee === fee &&
@ -1097,7 +1092,10 @@ export class ImportService {
activity.type === type &&
activity.unitPrice === unitPrice
);
});
};
const isDuplicate =
existingActivities.some(isMatchingActivity) ||
processedActivities.some(isMatchingActivity);
const error: ActivityError = isDuplicate
? { code: 'IS_DUPLICATE' }
@ -1121,7 +1119,7 @@ export class ImportService {
assetSubClass: undefined,
countries: undefined,
createdAt: undefined,
currency: undefined,
currency: assetCurrency,
holdings: undefined,
id: undefined,
isActive: true,

Loading…
Cancel
Save