Browse Source

Improve validation of manual symbols

pull/7409/head
Thomas Kaul 1 month ago
parent
commit
08c18643f0
  1. 5
      apps/api/src/app/activities/activities.service.ts
  2. 7
      apps/api/src/app/import/import.service.ts
  3. 3
      apps/api/src/services/data-provider/data-provider.service.ts
  4. 9
      apps/client/src/app/services/import-activities.service.ts
  5. 9
      libs/common/src/lib/config.ts
  6. 2
      libs/common/src/lib/helper.ts
  7. 71
      libs/common/src/lib/validator-constraints/has-valid-manual-symbols.ts
  8. 6
      libs/common/src/lib/validator-constraints/interfaces/interfaces.ts
  9. 2
      test/import/ok/penthouse-apartment.csv
  10. 1
      test/import/ok/sample.csv
  11. 10
      test/import/ok/without-accounts.json

5
apps/api/src/app/activities/activities.service.ts

@ -14,6 +14,7 @@ import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathe
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import { TagService } from '@ghostfolio/api/services/tag/tag.service';
import {
ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL,
DATA_GATHERING_QUEUE_PRIORITY_HIGH,
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS,
@ -188,7 +189,7 @@ export class ActivitiesService {
const userId = data.userId;
if (
['FEE', 'INTEREST', 'LIABILITY'].includes(data.type) ||
ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL.includes(data.type) ||
(data.SymbolProfile.connectOrCreate.create.dataSource === 'MANUAL' &&
data.type === 'BUY')
) {
@ -957,7 +958,7 @@ export class ActivitiesService {
let isDraft = false;
if (
['FEE', 'INTEREST', 'LIABILITY'].includes(data.type) ||
ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL.includes(data.type) ||
(data.SymbolProfile.connect.dataSource_symbol.dataSource === 'MANUAL' &&
data.type === 'BUY')
) {

7
apps/api/src/app/import/import.service.ts

@ -9,7 +9,10 @@ import { MarketDataService } from '@ghostfolio/api/services/market-data/market-d
import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import { TagService } from '@ghostfolio/api/services/tag/tag.service';
import { DATA_GATHERING_QUEUE_PRIORITY_HIGH } from '@ghostfolio/common/config';
import {
ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL,
DATA_GATHERING_QUEUE_PRIORITY_HIGH
} from '@ghostfolio/common/config';
import { CreateAssetProfileDto, CreateOrderDto } from '@ghostfolio/common/dtos';
import {
getAssetProfileIdentifier,
@ -384,7 +387,7 @@ export class ImportService {
for (const activity of activitiesDto) {
if (!activity.dataSource) {
if (['FEE', 'INTEREST', 'LIABILITY'].includes(activity.type)) {
if (ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL.includes(activity.type)) {
activity.dataSource = DataSource.MANUAL;
} else {
activity.dataSource =

3
apps/api/src/services/data-provider/data-provider.service.ts

@ -7,6 +7,7 @@ import { MarketDataService } from '@ghostfolio/api/services/market-data/market-d
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import {
ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL,
DEFAULT_CURRENCY,
DERIVED_CURRENCIES,
PROPERTY_API_KEY_GHOSTFOLIO,
@ -267,7 +268,7 @@ export class DataProviderService implements OnModuleInit {
if (!assetProfiles[assetProfileIdentifier]) {
if (
(dataSource === DataSource.MANUAL && type === 'BUY') ||
['FEE', 'INTEREST', 'LIABILITY'].includes(type)
ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL.includes(type)
) {
const assetProfileInImport = assetProfilesWithMarketDataDto?.find(
(assetProfile) => {

9
apps/client/src/app/services/import-activities.service.ts

@ -1,3 +1,4 @@
import { ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL } from '@ghostfolio/common/config';
import {
CreateAccountWithBalancesDto,
CreateAssetProfileWithMarketDataDto,
@ -77,8 +78,12 @@ export class ImportActivitiesService {
updateAccountBalance: false
});
if (dataSource === DataSource.MANUAL) {
// Create synthetic asset profile for MANUAL data source
if (
dataSource === DataSource.MANUAL &&
!ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL.includes(type)
) {
// Create synthetic asset profile for MANUAL data source. Activity types
// with a generated symbol are skipped, as the backend assigns a UUID.
assetProfiles.push({
currency,
symbol,

9
libs/common/src/lib/config.ts

@ -32,6 +32,15 @@ export const warnColorRgb = {
b: 69
};
// The symbol of these activity types is a label rather than an asset
// identifier, so a symbol profile with a generated UUID is created for each of
// them
export const ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL: Type[] = [
Type.FEE,
Type.INTEREST,
Type.LIABILITY
];
export const ASSET_CLASS_MAPPING = new Map<AssetClass, AssetSubClass[]>([
[AssetClass.ALTERNATIVE_INVESTMENT, [AssetSubClass.COLLECTIBLE]],
[AssetClass.COMMODITY, [AssetSubClass.PRECIOUS_METAL]],

2
libs/common/src/lib/helper.ts

@ -510,7 +510,7 @@ export function isRootCurrency(aCurrency: string) {
});
}
export function isValidManualSymbol(aSymbol: string) {
export function isValidManualSymbol(aSymbol?: string) {
if (!aSymbol) {
return false;
}

71
libs/common/src/lib/validator-constraints/has-valid-manual-symbols.ts

@ -1,46 +1,67 @@
import { DataSource } from '@prisma/client';
import {
ValidationArguments,
ValidatorConstraint,
ValidatorConstraintInterface
} from 'class-validator';
import { ghostfolioPrefix } from '../config';
import {
ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL,
ghostfolioPrefix
} from '../config';
import { isValidManualSymbol } from '../helper';
interface ImportDataLike {
activities?: { dataSource?: string; symbol?: string; type?: string }[];
assetProfiles?: { dataSource?: string; symbol?: string }[];
}
import { ImportDataLike } from './interfaces/interfaces';
@ValidatorConstraint({ name: 'hasValidManualSymbols' })
export class HasValidManualSymbolsConstraint implements ValidatorConstraintInterface {
public defaultMessage() {
return `manual symbols must be a UUID or start with "${ghostfolioPrefix}_"`;
public defaultMessage(args: ValidationArguments) {
const entry = this.getEntryWithInvalidManualSymbol(
args.object as ImportDataLike
);
return `manual symbols must be a UUID or start with "${ghostfolioPrefix}_", but got "${entry?.symbol ?? ''}"`;
}
public validate(_: unknown, args: ValidationArguments) {
const { activities = [], assetProfiles = [] } =
args.object as ImportDataLike;
return !this.getEntryWithInvalidManualSymbol(args.object as ImportDataLike);
}
const activitiesAreValid = activities.every(
private getEntryWithInvalidManualSymbol({
activities,
assetProfiles
}: ImportDataLike) {
// Defer to @IsArray() and @ValidateNested() for malformed input
const activityWithInvalidSymbol = this.toEntries(activities).find(
({ dataSource, symbol, type }) => {
// FEE, INTEREST and LIABILITY default to the MANUAL data source
// (resolved in the backend), so treat them as manual when no data
// source is set
const isManual =
dataSource === 'MANUAL' ||
(!dataSource && ['FEE', 'INTEREST', 'LIABILITY'].includes(type));
return !isManual || isValidManualSymbol(symbol);
}
);
const hasGeneratedUuidSymbol =
ACTIVITY_TYPES_WITH_GENERATED_UUID_SYMBOL.some((activityType) => {
return activityType === type;
});
const assetProfilesAreValid = assetProfiles.every(
({ dataSource, symbol }) => {
return dataSource !== 'MANUAL' || isValidManualSymbol(symbol);
return (
dataSource === DataSource.MANUAL &&
!hasGeneratedUuidSymbol &&
!isValidManualSymbol(symbol)
);
}
);
return activitiesAreValid && assetProfilesAreValid;
if (activityWithInvalidSymbol) {
return activityWithInvalidSymbol;
}
return this.toEntries(assetProfiles).find(({ dataSource, symbol }) => {
return dataSource === DataSource.MANUAL && !isValidManualSymbol(symbol);
});
}
private toEntries<T>(aValue?: T[]) {
if (!Array.isArray(aValue)) {
return [];
}
return aValue.filter((entry) => {
return entry instanceof Object;
});
}
}

6
libs/common/src/lib/validator-constraints/interfaces/interfaces.ts

@ -0,0 +1,6 @@
import { DataSource, Type } from '@prisma/client';
export interface ImportDataLike {
activities?: { dataSource?: DataSource; symbol?: string; type?: Type }[];
assetProfiles?: { dataSource?: DataSource; symbol?: string }[];
}

2
test/import/ok/penthouse-apartment.csv

@ -1,2 +0,0 @@
Date,Code,DataSource,Currency,Price,Quantity,Action,Fee,Note
01.01.2022,Penthouse Apartment,MANUAL,USD,500000.0,1,buy,0.00,
1 Date Code DataSource Currency Price Quantity Action Fee Note
2 01.01.2022 Penthouse Apartment MANUAL USD 500000.0 1 buy 0.00

1
test/import/ok/sample.csv

@ -2,5 +2,4 @@ Date,Code,DataSource,Currency,Price,Quantity,Action,Fee,Note
01-09-2021,Account Opening Fee,MANUAL,USD,0,0,fee,49,
16-09-2021,MSFT,YAHOO,USD,298.580,5,buy,19.00,My first order 🤓
17/11/2021,MSFT,YAHOO,USD,0.62,5,dividend,0.00,
01.01.2022,Penthouse Apartment,MANUAL,USD,500000.0,1,buy,0.00,
20500606,US5949181045,YAHOO,USD,0.00,0,buy,0.00,

1 Date Code DataSource Currency Price Quantity Action Fee Note
2 01-09-2021 Account Opening Fee MANUAL USD 0 0 fee 49
3 16-09-2021 MSFT YAHOO USD 298.580 5 buy 19.00 My first order 🤓
4 17/11/2021 MSFT YAHOO USD 0.62 5 dividend 0.00
01.01.2022 Penthouse Apartment MANUAL USD 500000.0 1 buy 0.00
5 20500606 US5949181045 YAHOO USD 0.00 0 buy 0.00

10
test/import/ok/without-accounts.json

@ -14,16 +14,6 @@
"date": "2050-06-06T00:00:00.000Z",
"symbol": "MSFT"
},
{
"fee": 0,
"quantity": 1,
"type": "BUY",
"unitPrice": 500000,
"currency": "USD",
"dataSource": "MANUAL",
"date": "2022-01-01T00:00:00.000Z",
"symbol": "Penthouse Apartment"
},
{
"fee": 0,
"quantity": 5,

Loading…
Cancel
Save