Browse Source

Improve check for duplicates

pull/7541/head
Thomas Kaul 4 weeks ago
parent
commit
1515b84530
  1. 161
      apps/api/src/app/import/import.service.ts

161
apps/api/src/app/import/import.service.ts

@ -326,18 +326,22 @@ export class ImportService {
} }
if (!isDryRun && accountsWithBalancesDto?.length) { if (!isDryRun && accountsWithBalancesDto?.length) {
const [existingAccounts, existingPlatforms] = await Promise.all([ const [existingAccounts, existingAccountsOfUser, existingPlatforms] =
this.accountService.accounts({ await Promise.all([
where: { this.accountService.accounts({
id: { where: {
in: accountsWithBalancesDto.map(({ id }) => { id: {
return id; in: accountsWithBalancesDto.map(({ id }) => {
}) return id;
})
}
} }
} }),
}), this.accountService.accounts({
this.platformService.getPlatforms() where: { userId: user.id }
]); }),
this.platformService.getPlatforms()
]);
const existingTagIds = new Set( const existingTagIds = new Set(
existingTagsOfUser.map(({ id }) => { existingTagsOfUser.map(({ id }) => {
@ -351,71 +355,92 @@ export class ImportService {
return existingAccount.id === accountWithBalances.id; return existingAccount.id === accountWithBalances.id;
}); });
// If there is no account or if the account belongs to a different user then create a new account // If there is no account or if the account belongs to a different
// user, then reuse an existing account of the user with the same name
// or create a new account
if (!accountWithSameId || accountWithSameId.userId !== user.id) { if (!accountWithSameId || accountWithSameId.userId !== user.id) {
const account = omit(accountWithBalances, [ // Check if the user has an account with the same name
'balances', const accountWithSameNameOfUser = existingAccountsOfUser.find(
'isExcluded', ({ name }) => {
'tags' return name === accountWithBalances.name;
]); }
);
let oldAccountId: string;
const platformId =
platformIdMapping[account.platformId] ?? account.platformId;
delete account.platformId; if (accountWithSameNameOfUser) {
// Reuse the account of the user instead of creating a duplicate
if (
accountWithBalances.id &&
accountWithBalances.id !== accountWithSameNameOfUser.id
) {
// Store the new to old account ID mappings for updating activities
accountIdMapping[accountWithBalances.id] =
accountWithSameNameOfUser.id;
}
} else {
const account = omit(accountWithBalances, [
'balances',
'isExcluded',
'tags'
]);
if (accountWithSameId) { let oldAccountId: string;
oldAccountId = account.id; const platformId =
delete account.id; platformIdMapping[account.platformId] ?? account.platformId;
}
const tagIds = (accountWithBalances.tags ?? []) delete account.platformId;
.map((tagId) => {
return tagIdMapping[tagId] ?? tagId;
})
.filter((tagId) => {
return existingTagIds.has(tagId);
});
// Map the legacy isExcluded attribute of old export files to if (accountWithSameId) {
// the "Exclude from Analysis" tag oldAccountId = account.id;
if ( delete account.id;
accountWithBalances.isExcluded && }
existingTagIds.has(TAG_ID_EXCLUDE_FROM_ANALYSIS) &&
!tagIds.includes(TAG_ID_EXCLUDE_FROM_ANALYSIS)
) {
tagIds.push(TAG_ID_EXCLUDE_FROM_ANALYSIS);
}
let accountObject: Prisma.AccountCreateInput = { const tagIds = (accountWithBalances.tags ?? [])
...account, .map((tagId) => {
balances: { return tagIdMapping[tagId] ?? tagId;
create: accountWithBalances.balances ?? [] })
}, .filter((tagId) => {
user: { connect: { id: user.id } } return existingTagIds.has(tagId);
}; });
// Map the legacy isExcluded attribute of old export files to
// the "Exclude from Analysis" tag
if (
accountWithBalances.isExcluded &&
existingTagIds.has(TAG_ID_EXCLUDE_FROM_ANALYSIS) &&
!tagIds.includes(TAG_ID_EXCLUDE_FROM_ANALYSIS)
) {
tagIds.push(TAG_ID_EXCLUDE_FROM_ANALYSIS);
}
if ( let accountObject: Prisma.AccountCreateInput = {
existingPlatforms.some(({ id }) => { ...account,
return id === platformId; balances: {
}) create: accountWithBalances.balances ?? []
) { },
accountObject = { user: { connect: { id: user.id } }
...accountObject,
platform: { connect: { id: platformId } }
}; };
}
const newAccount = await this.accountService.createAccount( if (
accountObject, existingPlatforms.some(({ id }) => {
user.id, return id === platformId;
tagIds })
); ) {
accountObject = {
...accountObject,
platform: { connect: { id: platformId } }
};
}
// Store the new to old account ID mappings for updating activities const newAccount = await this.accountService.createAccount(
if (accountWithSameId && oldAccountId) { accountObject,
accountIdMapping[oldAccountId] = newAccount.id; user.id,
tagIds
);
// Store the new to old account ID mappings for updating activities
if (accountWithSameId && oldAccountId) {
accountIdMapping[oldAccountId] = newAccount.id;
}
} }
} }
} }
@ -823,10 +848,10 @@ export class ImportService {
unitPrice unitPrice
}) => { }) => {
const date = parseISO(dateString); const date = parseISO(dateString);
const isDuplicate = existingActivities.some((activity) => { const isDuplicate = existingActivities.some((activity) => {
return ( return (
activity.accountId === accountId && (activity.comment ?? null) === (comment ?? null) &&
activity.comment === comment &&
(activity.currency === currency || (activity.currency === currency ||
activity.assetProfile.currency === currency) && activity.assetProfile.currency === currency) &&
activity.assetProfile.dataSource === dataSource && activity.assetProfile.dataSource === dataSource &&

Loading…
Cancel
Save