Feature/extend activities import by custom asset profiles (#5243)

* Extend activities import by custom asset profiles

* Update changelog

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Extended the import functionality by custom asset profiles
 - Migrated the get country and sector weightings, dividends, ETF holdings, ETF info, historical price, profile, quote and symbol search functionalities of the _Financial Modeling Prep_ service to its stable API version
 - Refactored the toggle component to standalone
 - Improved the language localization for Dutch (`nl`)

apps/api/src/app/admin/create-asset-profile.dto.ts

@@ -0,0 +1,92 @@
+import { IsCurrencyCode } from '@ghostfolio/api/validators/is-currency-code';
+
+import { AssetClass, AssetSubClass, DataSource, Prisma } from '@prisma/client';
+import {
+  IsArray,
+  IsBoolean,
+  IsEnum,
+  IsObject,
+  IsOptional,
+  IsString,
+  IsUrl
+} from 'class-validator';
+
+export class CreateAssetProfileDto {
+  @IsEnum(AssetClass, { each: true })
+  @IsOptional()
+  assetClass?: AssetClass;
+
+  @IsEnum(AssetSubClass, { each: true })
+  @IsOptional()
+  assetSubClass?: AssetSubClass;
+
+  @IsOptional()
+  @IsString()
+  comment?: string;
+
+  @IsArray()
+  @IsOptional()
+  countries?: Prisma.InputJsonArray;
+
+  @IsCurrencyCode()
+  currency: string;
+
+  @IsOptional()
+  @IsString()
+  cusip?: string;
+
+  @IsEnum(DataSource)
+  dataSource: DataSource;
+
+  @IsOptional()
+  @IsString()
+  figi?: string;
+
+  @IsOptional()
+  @IsString()
+  figiComposite?: string;
+
+  @IsOptional()
+  @IsString()
+  figiShareClass?: string;
+
+  @IsArray()
+  @IsOptional()
+  holdings?: Prisma.InputJsonArray;
+
+  @IsBoolean()
+  @IsOptional()
+  isActive?: boolean;
+
+  @IsOptional()
+  @IsString()
+  isin?: string;
+
+  @IsOptional()
+  @IsString()
+  name?: string;
+
+  @IsObject()
+  @IsOptional()
+  scraperConfiguration?: Prisma.InputJsonObject;
+
+  @IsArray()
+  @IsOptional()
+  sectors?: Prisma.InputJsonArray;
+
+  @IsString()
+  symbol: string;
+
+  @IsObject()
+  @IsOptional()
+  symbolMapping?: {
+    [dataProvider: string]: string;
+  };
+
+  @IsOptional()
+  @IsUrl({
+    protocols: ['https'],
+    require_protocol: true
+  })
+  url?: string;
+}

apps/api/src/app/export/export.service.ts

@@ -195,7 +195,6 @@ export class ExportService {
             figiComposite,
             figiShareClass,
             holdings: holdings as unknown as Prisma.JsonArray,
-            id,
             isActive,
             isin,
             marketData: marketDataByAssetProfile[id],

apps/api/src/app/import/create-asset-profile-with-market-data.dto.ts

@@ -0,0 +1,17 @@
+import { MarketData } from '@ghostfolio/common/interfaces';
+
+import { DataSource } from '@prisma/client';
+import { IsArray, IsEnum, IsOptional } from 'class-validator';
+
+import { CreateAssetProfileDto } from '../admin/create-asset-profile.dto';
+
+export class CreateAssetProfileWithMarketDataDto extends CreateAssetProfileDto {
+  @IsEnum([DataSource.MANUAL], {
+    message: `dataSource must be '${DataSource.MANUAL}'`
+  })
+  dataSource: DataSource;
+
+  @IsArray()
+  @IsOptional()
+  marketData?: MarketData[];
+}

apps/api/src/app/import/import-data.dto.ts

@@ -4,16 +4,23 @@ import { Type } from 'class-transformer';
 import { IsArray, IsOptional, ValidateNested } from 'class-validator';
 
 import { CreateAccountWithBalancesDto } from './create-account-with-balances.dto';
+import { CreateAssetProfileWithMarketDataDto } from './create-asset-profile-with-market-data.dto';
 
 export class ImportDataDto {
-  @IsOptional()
   @IsArray()
+  @IsOptional()
   @Type(() => CreateAccountWithBalancesDto)
   @ValidateNested({ each: true })
-  accounts: CreateAccountWithBalancesDto[];
+  accounts?: CreateAccountWithBalancesDto[];
 
   @IsArray()
   @Type(() => CreateOrderDto)
   @ValidateNested({ each: true })
   activities: CreateOrderDto[];
+
+  @IsArray()
+  @IsOptional()
+  @Type(() => CreateAssetProfileWithMarketDataDto)
+  @ValidateNested({ each: true })
+  assetProfiles?: CreateAssetProfileWithMarketDataDto[];
 }

apps/api/src/app/import/import.controller.ts

@@ -73,6 +73,7 @@ export class ImportController {
         maxActivitiesToImport,
         accountsWithBalancesDto: importData.accounts ?? [],
         activitiesDto: importData.activities,
+        assetProfilesWithMarketDataDto: importData.assetProfiles ?? [],
         user: this.request.user
       });
 

apps/api/src/app/import/import.module.ts

@@ -9,6 +9,7 @@ import { TransformDataSourceInResponseModule } from '@ghostfolio/api/interceptor
 import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
 import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
+import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
 import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
 import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
@@ -27,6 +28,7 @@ import { ImportService } from './import.service';
     DataGatheringModule,
     DataProviderModule,
     ExchangeRateDataModule,
+    MarketDataModule,
     OrderModule,
     PlatformModule,
     PortfolioModule,

apps/api/src/app/import/import.service.ts

@@ -10,6 +10,7 @@ import { PlatformService } from '@ghostfolio/api/app/platform/platform.service';
 import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import { DATA_GATHERING_QUEUE_PRIORITY_HIGH } from '@ghostfolio/common/config';
@@ -31,6 +32,7 @@ import { endOfToday, isAfter, isSameSecond, parseISO } from 'date-fns';
 import { omit, uniqBy } from 'lodash';
 import { v4 as uuidv4 } from 'uuid';
 
+import { CreateAssetProfileDto } from '../admin/create-asset-profile.dto';
 import { ImportDataDto } from './import-data.dto';
 
 @Injectable()
@@ -40,6 +42,7 @@ export class ImportService {
     private readonly configurationService: ConfigurationService,
     private readonly dataGatheringService: DataGatheringService,
     private readonly dataProviderService: DataProviderService,
+    private readonly marketDataService: MarketDataService,
     private readonly orderService: OrderService,
     private readonly platformService: PlatformService,
     private readonly portfolioService: PortfolioService,
@@ -148,17 +151,20 @@ export class ImportService {
   public async import({
     accountsWithBalancesDto,
     activitiesDto,
+    assetProfilesWithMarketDataDto,
     isDryRun = false,
     maxActivitiesToImport,
     user
   }: {
     accountsWithBalancesDto: ImportDataDto['accounts'];
     activitiesDto: ImportDataDto['activities'];
+    assetProfilesWithMarketDataDto: ImportDataDto['assetProfiles'];
     isDryRun?: boolean;
     maxActivitiesToImport: number;
     user: UserWithSettings;
   }): Promise<Activity[]> {
     const accountIdMapping: { [oldAccountId: string]: string } = {};
+    const assetProfileSymbolMapping: { [oldSymbol: string]: string } = {};
     const userCurrency = user.settings.settings.baseCurrency;
 
     if (!isDryRun && accountsWithBalancesDto?.length) {
@@ -230,6 +236,63 @@ export class ImportService {
       }
     }
 
+    if (!isDryRun && assetProfilesWithMarketDataDto?.length) {
+      const existingAssetProfiles =
+        await this.symbolProfileService.getSymbolProfiles(
+          assetProfilesWithMarketDataDto.map(({ dataSource, symbol }) => {
+            return { dataSource, symbol };
+          })
+        );
+
+      for (const assetProfileWithMarketData of assetProfilesWithMarketDataDto) {
+        // Check if there is any existing asset profile
+        const existingAssetProfile = existingAssetProfiles.find(
+          ({ dataSource, symbol }) => {
+            return (
+              dataSource === assetProfileWithMarketData.dataSource &&
+              symbol === assetProfileWithMarketData.symbol
+            );
+          }
+        );
+
+        // If there is no asset profile or if the asset profile belongs to a different user, then create a new asset profile
+        if (!existingAssetProfile || existingAssetProfile.userId !== user.id) {
+          const assetProfile: CreateAssetProfileDto = omit(
+            assetProfileWithMarketData,
+            'marketData'
+          );
+
+          // Asset profile belongs to a different user
+          if (existingAssetProfile) {
+            const symbol = uuidv4();
+            assetProfileSymbolMapping[assetProfile.symbol] = symbol;
+            assetProfile.symbol = symbol;
+          }
+
+          // Create a new asset profile
+          const assetProfileObject: Prisma.SymbolProfileCreateInput = {
+            ...assetProfile,
+            user: { connect: { id: user.id } }
+          };
+
+          await this.symbolProfileService.add(assetProfileObject);
+        }
+
+        // Insert or update market data
+        const marketDataObjects = assetProfileWithMarketData.marketData.map(
+          (marketData) => {
+            return {
+              ...marketData,
+              dataSource: assetProfileWithMarketData.dataSource,
+              symbol: assetProfileWithMarketData.symbol
+            } as Prisma.MarketDataUpdateInput;
+          }
+        );
+
+        await this.marketDataService.updateMany({ data: marketDataObjects });
+      }
+    }
+
     for (const activity of activitiesDto) {
       if (!activity.dataSource) {
         if (['FEE', 'INTEREST', 'LIABILITY'].includes(activity.type)) {
@@ -240,11 +303,16 @@ export class ImportService {
         }
       }
 
-      // If a new account is created, then update the accountId in all activities
       if (!isDryRun) {
-        if (Object.keys(accountIdMapping).includes(activity.accountId)) {
+        // If a new account is created, then update the accountId in all activities
+        if (accountIdMapping[activity.accountId]) {
           activity.accountId = accountIdMapping[activity.accountId];
         }
+
+        // If a new asset profile is created, then update the symbol in all activities
+        if (assetProfileSymbolMapping[activity.symbol]) {
+          activity.symbol = assetProfileSymbolMapping[activity.symbol];
+        }
       }
     }
 

apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts

@@ -1,4 +1,5 @@
-import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
+import { CreateAccountWithBalancesDto } from '@ghostfolio/api/app/import/create-account-with-balances.dto';
+import { CreateAssetProfileWithMarketDataDto } from '@ghostfolio/api/app/import/create-asset-profile-with-market-data.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { GfDialogFooterModule } from '@ghostfolio/client/components/dialog-footer/dialog-footer.module';
 import { GfDialogHeaderModule } from '@ghostfolio/client/components/dialog-header/dialog-header.module';
@@ -75,9 +76,10 @@ import { ImportActivitiesDialogParams } from './interfaces/interfaces';
   templateUrl: 'import-activities-dialog.html'
 })
 export class GfImportActivitiesDialog implements OnDestroy {
-  public accounts: CreateAccountDto[] = [];
+  public accounts: CreateAccountWithBalancesDto[] = [];
   public activities: Activity[] = [];
   public assetProfileForm: FormGroup;
+  public assetProfiles: CreateAssetProfileWithMarketDataDto[] = [];
   public dataSource: MatTableDataSource<Activity>;
   public details: any[] = [];
   public deviceType: string;
@@ -166,7 +168,8 @@ export class GfImportActivitiesDialog implements OnDestroy {
 
       await this.importActivitiesService.importSelectedActivities({
         accounts: this.accounts,
-        activities: this.selectedActivities
+        activities: this.selectedActivities,
+        assetProfiles: this.assetProfiles
       });
 
       this.snackBar.open(
@@ -293,6 +296,7 @@ export class GfImportActivitiesDialog implements OnDestroy {
           const content = JSON.parse(fileContent);
 
           this.accounts = content.accounts;
+          this.assetProfiles = content.assetProfiles;
 
           if (!isArray(content.activities)) {
             if (isArray(content.orders)) {
@@ -323,6 +327,7 @@ export class GfImportActivitiesDialog implements OnDestroy {
               await this.importActivitiesService.importJson({
                 accounts: content.accounts,
                 activities: content.activities,
+                assetProfiles: content.assetProfiles,
                 isDryRun: true
               });
             this.activities = activities;

apps/client/src/app/services/import-activities.service.ts

@@ -1,4 +1,5 @@
-import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
+import { CreateAccountWithBalancesDto } from '@ghostfolio/api/app/import/create-account-with-balances.dto';
+import { CreateAssetProfileWithMarketDataDto } from '@ghostfolio/api/app/import/create-asset-profile-with-market-data.dto';
 import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { parseDate as parseDateHelper } from '@ghostfolio/common/helper';
@@ -73,20 +74,22 @@ export class ImportActivitiesService {
   public importJson({
     accounts,
     activities,
+    assetProfiles,
     isDryRun = false
   }: {
     activities: CreateOrderDto[];
-    accounts?: CreateAccountDto[];
+    accounts?: CreateAccountWithBalancesDto[];
+    assetProfiles?: CreateAssetProfileWithMarketDataDto[];
     isDryRun?: boolean;
   }): Promise<{
     activities: Activity[];
-    accounts?: CreateAccountDto[];
   }> {
     return new Promise((resolve, reject) => {
       this.postImport(
         {
           accounts,
-          activities
+          activities,
+          assetProfiles
         },
         isDryRun
       )
@@ -106,13 +109,14 @@ export class ImportActivitiesService {
 
   public importSelectedActivities({
     accounts,
-    activities
+    activities,
+    assetProfiles
   }: {
-    accounts: CreateAccountDto[];
+    accounts?: CreateAccountWithBalancesDto[];
     activities: Activity[];
+    assetProfiles?: CreateAssetProfileWithMarketDataDto[];
   }): Promise<{
     activities: Activity[];
-    accounts?: CreateAccountDto[];
   }> {
     const importData: CreateOrderDto[] = [];
 
@@ -120,7 +124,7 @@ export class ImportActivitiesService {
       importData.push(this.convertToCreateOrderDto(activity));
     }
 
-    return this.importJson({ accounts, activities: importData });
+    return this.importJson({ accounts, assetProfiles, activities: importData });
   }
 
   private convertToCreateOrderDto({
@@ -383,7 +387,11 @@ export class ImportActivitiesService {
   }
 
   private postImport(
-    aImportData: { accounts: CreateAccountDto[]; activities: CreateOrderDto[] },
+    aImportData: {
+      accounts?: CreateAccountWithBalancesDto[];
+      activities: CreateOrderDto[];
+      assetProfiles?: CreateAssetProfileWithMarketDataDto[];
+    },
     aIsDryRun = false
   ) {
     return this.http.post<{ activities: Activity[] }>(

libs/common/src/lib/interfaces/export.interface.ts

@@ -8,6 +8,7 @@ import {
 } from '@prisma/client';
 
 import { AccountBalance } from './account-balance.interface';
+import { MarketData } from './market-data.interface';
 
 export interface Export {
   accounts: (Omit<Account, 'createdAt' | 'updatedAt' | 'userId'> & {
@@ -23,8 +24,11 @@ export interface Export {
     | 'updatedAt'
     | 'userId'
   > & { dataSource: DataSource; date: string; symbol: string })[];
-  assetProfiles: (Omit<SymbolProfile, 'createdAt' | 'updatedAt' | 'userId'> & {
-    marketData: { date: string; marketPrice: number }[];
+  assetProfiles: (Omit<
+    SymbolProfile,
+    'createdAt' | 'id' | 'updatedAt' | 'userId'
+  > & {
+    marketData: MarketData[];
   })[];
   meta: {
     date: string;

libs/common/src/lib/interfaces/index.ts

@@ -25,6 +25,7 @@ import type { InfoItem } from './info-item.interface';
 import type { InvestmentItem } from './investment-item.interface';
 import type { LineChartItem } from './line-chart-item.interface';
 import type { LookupItem } from './lookup-item.interface';
+import type { MarketData } from './market-data.interface';
 import type { PortfolioChart } from './portfolio-chart.interface';
 import type { PortfolioDetails } from './portfolio-details.interface';
 import type { PortfolioDividends } from './portfolio-dividends.interface';
@@ -111,6 +112,7 @@ export {
   LineChartItem,
   LookupItem,
   LookupResponse,
+  MarketData,
   MarketDataDetailsResponse,
   MarketDataOfMarketsResponse,
   OAuthResponse,

libs/common/src/lib/interfaces/market-data.interface.ts

@@ -0,0 +1,4 @@
+export interface MarketData {
+  date: string;
+  marketPrice: number;
+}

test/import/ok/sample.json

@@ -16,6 +16,7 @@
           "value": 1000
         }
       ],
+      "comment": null,
       "currency": "USD",
       "id": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
       "isExcluded": false,
@@ -23,30 +24,80 @@
       "platformId": null
     }
   ],
+  "assetProfiles": [
+    {
+      "assetClass": null,
+      "assetSubClass": null,
+      "comment": null,
+      "countries": [],
+      "currency": "USD",
+      "cusip": null,
+      "dataSource": "MANUAL",
+      "figi": null,
+      "figiComposite": null,
+      "figiShareClass": null,
+      "holdings": [],
+      "isActive": true,
+      "isin": null,
+      "marketData": [],
+      "name": "Account Opening Fee",
+      "scraperConfiguration": null,
+      "sectors": [],
+      "symbol": "14a69cb9-1e31-43fa-b320-83703d8ed74b",
+      "symbolMapping": {},
+      "url": null
+    },
+    {
+      "assetClass": null,
+      "assetSubClass": null,
+      "comment": null,
+      "countries": [],
+      "currency": "USD",
+      "cusip": null,
+      "dataSource": "MANUAL",
+      "figi": null,
+      "figiComposite": null,
+      "figiShareClass": null,
+      "holdings": [],
+      "isActive": true,
+      "isin": null,
+      "marketData": [],
+      "name": "Penthouse Apartment",
+      "scraperConfiguration": null,
+      "sectors": [],
+      "symbol": "7e91b7d4-1430-4212-8380-289a06c9bbc1",
+      "symbolMapping": {},
+      "url": null
+    }
+  ],
+  "platforms": [],
+  "tags": [],
   "activities": [
     {
       "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
       "comment": null,
-      "fee": 0,
+      "fee": 49,
       "quantity": 0,
-      "type": "BUY",
+      "type": "FEE",
       "unitPrice": 0,
       "currency": "USD",
-      "dataSource": "YAHOO",
-      "date": "2050-06-06T00:00:00.000Z",
-      "symbol": "US5949181045"
+      "dataSource": "MANUAL",
+      "date": "2021-09-01T00:00:00.000Z",
+      "symbol": "14a69cb9-1e31-43fa-b320-83703d8ed74b",
+      "tags": []
     },
     {
-      "accountId": null,
-      "comment": null,
-      "fee": 0,
-      "quantity": 1,
+      "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
+      "comment": "My first order 🤓",
+      "fee": 19,
+      "quantity": 5,
       "type": "BUY",
-      "unitPrice": 500000,
+      "unitPrice": 298.58,
       "currency": "USD",
-      "dataSource": "MANUAL",
-      "date": "2022-01-01T00:00:00.000Z",
-      "symbol": "Penthouse Apartment"
+      "dataSource": "YAHOO",
+      "date": "2021-09-16T00:00:00.000Z",
+      "symbol": "MSFT",
+      "tags": []
     },
     {
       "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
@@ -58,31 +109,39 @@
       "currency": "USD",
       "dataSource": "YAHOO",
       "date": "2021-11-17T00:00:00.000Z",
-      "symbol": "MSFT"
+      "symbol": "MSFT",
+      "tags": []
     },
     {
-      "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
-      "comment": "My first order 🤓",
-      "fee": 19,
-      "quantity": 5,
+      "accountId": null,
+      "comment": null,
+      "fee": 0,
+      "quantity": 1,
       "type": "BUY",
-      "unitPrice": 298.58,
+      "unitPrice": 500000,
       "currency": "USD",
-      "dataSource": "YAHOO",
-      "date": "2021-09-16T00:00:00.000Z",
-      "symbol": "MSFT"
+      "dataSource": "MANUAL",
+      "date": "2022-01-01T00:00:00.000Z",
+      "symbol": "7e91b7d4-1430-4212-8380-289a06c9bbc1",
+      "tags": []
     },
     {
       "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
       "comment": null,
-      "fee": 49,
+      "fee": 0,
       "quantity": 0,
-      "type": "FEE",
+      "type": "BUY",
       "unitPrice": 0,
       "currency": "USD",
-      "dataSource": "MANUAL",
-      "date": "2021-09-01T00:00:00.000Z",
-      "symbol": "Account Opening Fee"
+      "dataSource": "YAHOO",
+      "date": "2050-06-06T00:00:00.000Z",
+      "symbol": "MSFT",
+      "tags": []
+    }
+  ],
+  "user": {
+    "settings": {
+      "currency": "USD"
     }
-  ]
+  }
 }