From 2e5176bacf43a34adf44923637f47f01a2613631 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Fri, 15 Oct 2021 22:22:45 +0200 Subject: [PATCH] Feature/extend import by csv files (#419) * Support import of csv files * Update changelog --- CHANGELOG.md | 11 + apps/api/src/app/info/info.service.ts | 3 + .../src/app/portfolio/current-rate.service.ts | 6 +- .../data-provider/data-provider.service.ts | 4 + .../services/exchange-rate-data.service.ts | 2 +- .../transactions-page.component.ts | 85 +++++--- .../transactions/transactions-page.module.ts | 3 +- apps/client/src/app/services/data.service.ts | 4 - .../services/import-transactions.service.ts | 195 ++++++++++++++++++ .../src/lib/interfaces/info-item.interface.ts | 2 + package.json | 2 + yarn.lock | 12 ++ 12 files changed, 291 insertions(+), 38 deletions(-) create mode 100644 apps/client/src/app/services/import-transactions.service.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 537a20d7c..e6cdd7cea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- Extended the import functionality for transactions by `csv` files +- Introduced the primary data source + +### Changed + +- Restricted the file selector of the import functionality for transactions to `csv` and `json` + ## 1.60.0 - 13.10.2021 ### Added diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.ts index 9486f1527..c84daab1a 100644 --- a/apps/api/src/app/info/info.service.ts +++ b/apps/api/src/app/info/info.service.ts @@ -1,5 +1,6 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; +import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service'; import { PrismaService } from '@ghostfolio/api/services/prisma.service'; import { InfoItem } from '@ghostfolio/common/interfaces'; @@ -16,6 +17,7 @@ export class InfoService { public constructor( private readonly configurationService: ConfigurationService, + private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, private readonly dataGatheringService: DataGatheringService, private readonly jwtService: JwtService, @@ -60,6 +62,7 @@ export class InfoService { currencies: this.exchangeRateDataService.getCurrencies(), demoAuthToken: this.getDemoAuthToken(), lastDataGathering: await this.getLastDataGathering(), + primaryDataSource: this.dataProviderService.getPrimaryDataSource(), statistics: await this.getStatistics(), subscriptions: await this.getSubscriptions() }; diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index 7f2813c6e..c5dff75f9 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -2,7 +2,6 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service'; import { resetHours } from '@ghostfolio/common/helper'; import { Injectable } from '@nestjs/common'; -import { DataSource } from '@prisma/client'; import { isBefore, isToday } from 'date-fns'; import { flatten } from 'lodash'; @@ -27,7 +26,10 @@ export class CurrentRateService { }: GetValueParams): Promise { if (isToday(date)) { const dataProviderResult = await this.dataProviderService.get([ - { symbol, dataSource: DataSource.YAHOO } + { + symbol, + dataSource: this.dataProviderService.getPrimaryDataSource() + } ]); return { symbol, diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 11109ed43..efb2a4ec0 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -199,6 +199,10 @@ export class DataProviderService { }; } + public getPrimaryDataSource(): DataSource { + return DataSource[this.configurationService.get('DATA_SOURCES')[0]]; + } + private getDataProvider(providerName: DataSource) { switch (providerName) { case DataSource.ALPHA_VANTAGE: diff --git a/apps/api/src/services/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data.service.ts index 729d803b3..f58c71190 100644 --- a/apps/api/src/services/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data.service.ts @@ -210,7 +210,7 @@ export class ExchangeRateDataService { return { currency1: baseCurrency, currency2: currency, - dataSource: DataSource.YAHOO, + dataSource: this.dataProviderService.getPrimaryDataSource(), symbol: `${baseCurrency}${currency}` }; }); diff --git a/apps/client/src/app/pages/portfolio/transactions/transactions-page.component.ts b/apps/client/src/app/pages/portfolio/transactions/transactions-page.component.ts index 337948d14..81186cc2d 100644 --- a/apps/client/src/app/pages/portfolio/transactions/transactions-page.component.ts +++ b/apps/client/src/app/pages/portfolio/transactions/transactions-page.component.ts @@ -6,14 +6,15 @@ import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto'; import { UpdateOrderDto } from '@ghostfolio/api/app/order/update-order.dto'; import { DataService } from '@ghostfolio/client/services/data.service'; import { ImpersonationStorageService } from '@ghostfolio/client/services/impersonation-storage.service'; +import { ImportTransactionsService } from '@ghostfolio/client/services/import-transactions.service'; import { UserService } from '@ghostfolio/client/services/user/user.service'; import { User } from '@ghostfolio/common/interfaces'; import { hasPermission, permissions } from '@ghostfolio/common/permissions'; -import { Order as OrderModel } from '@prisma/client'; +import { DataSource, Order as OrderModel } from '@prisma/client'; import { format, parseISO } from 'date-fns'; import { DeviceDetectorService } from 'ngx-device-detector'; -import { EMPTY, Subject, Subscription } from 'rxjs'; -import { catchError, takeUntil } from 'rxjs/operators'; +import { Subject, Subscription } from 'rxjs'; +import { takeUntil } from 'rxjs/operators'; import { CreateOrUpdateTransactionDialog } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.component'; import { ImportTransactionDialog } from './import-transaction-dialog/import-transaction-dialog.component'; @@ -35,6 +36,7 @@ export class TransactionsPageComponent implements OnDestroy, OnInit { public transactions: OrderModel[]; public user: User; + private primaryDataSource: DataSource; private unsubscribeSubject = new Subject(); /** @@ -46,11 +48,15 @@ export class TransactionsPageComponent implements OnDestroy, OnInit { private deviceService: DeviceDetectorService, private dialog: MatDialog, private impersonationStorageService: ImpersonationStorageService, + private importTransactionsService: ImportTransactionsService, private route: ActivatedRoute, private router: Router, private snackBar: MatSnackBar, private userService: UserService ) { + const { primaryDataSource } = this.dataService.fetchInfo(); + this.primaryDataSource = primaryDataSource; + this.routeQueryParams = route.queryParams .pipe(takeUntil(this.unsubscribeSubject)) .subscribe((params) => { @@ -58,8 +64,8 @@ export class TransactionsPageComponent implements OnDestroy, OnInit { this.openCreateTransactionDialog(); } else if (params['editDialog']) { if (this.transactions) { - const transaction = this.transactions.find((transaction) => { - return transaction.id === params['transactionId']; + const transaction = this.transactions.find(({ id }) => { + return id === params['transactionId']; }); this.openUpdateTransactionDialog(transaction); @@ -164,9 +170,12 @@ export class TransactionsPageComponent implements OnDestroy, OnInit { public onImport() { const input = document.createElement('input'); + input.accept = 'application/JSON, .csv'; input.type = 'file'; input.onchange = (event) => { + this.snackBar.open('⏳ Importing data...'); + // Getting the file reference const file = (event.target as HTMLInputElement).files[0]; @@ -174,35 +183,43 @@ export class TransactionsPageComponent implements OnDestroy, OnInit { const reader = new FileReader(); reader.readAsText(file, 'UTF-8'); - reader.onload = (readerEvent) => { + reader.onload = async (readerEvent) => { + const fileContent = readerEvent.target.result as string; + try { - const content = JSON.parse(readerEvent.target.result as string); + if (file.type === 'application/json') { + const content = JSON.parse(fileContent); + try { + await this.importTransactionsService.importJson({ + content: content.orders, + defaultAccountId: this.defaultAccountId + }); + + this.handleImportSuccess(); + } catch (error) { + this.handleImportError(error); + } - this.snackBar.open('⏳ Importing data...'); + return; + } else if (file.type === 'text/csv') { + try { + await this.importTransactionsService.importCsv({ + fileContent, + defaultAccountId: this.defaultAccountId, + primaryDataSource: this.primaryDataSource + }); + + this.handleImportSuccess(); + } catch (error) { + this.handleImportError({ + error: { message: error?.error?.message ?? [error?.message] } + }); + } - this.dataService - .postImport({ - orders: content.orders.map((order) => { - return { ...order, accountId: this.defaultAccountId }; - }) - }) - .pipe( - catchError((error) => { - this.handleImportError(error); - - return EMPTY; - }), - takeUntil(this.unsubscribeSubject) - ) - .subscribe({ - next: () => { - this.fetchOrders(); + return; + } - this.snackBar.open('✅ Import has been completed', undefined, { - duration: 3000 - }); - } - }); + throw new Error(); } catch (error) { this.handleImportError({ error: { message: ['Unexpected format'] } }); } @@ -302,6 +319,14 @@ export class TransactionsPageComponent implements OnDestroy, OnInit { }); } + private handleImportSuccess() { + this.fetchOrders(); + + this.snackBar.open('✅ Import has been completed', undefined, { + duration: 3000 + }); + } + private openCreateTransactionDialog(aTransaction?: OrderModel): void { const dialogRef = this.dialog.open(CreateOrUpdateTransactionDialog, { data: { diff --git a/apps/client/src/app/pages/portfolio/transactions/transactions-page.module.ts b/apps/client/src/app/pages/portfolio/transactions/transactions-page.module.ts index efdac22e2..f4194fbc3 100644 --- a/apps/client/src/app/pages/portfolio/transactions/transactions-page.module.ts +++ b/apps/client/src/app/pages/portfolio/transactions/transactions-page.module.ts @@ -4,6 +4,7 @@ import { MatButtonModule } from '@angular/material/button'; import { MatSnackBarModule } from '@angular/material/snack-bar'; import { RouterModule } from '@angular/router'; import { GfTransactionsTableModule } from '@ghostfolio/client/components/transactions-table/transactions-table.module'; +import { ImportTransactionsService } from '@ghostfolio/client/services/import-transactions.service'; import { GfCreateOrUpdateTransactionDialogModule } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.module'; import { GfImportTransactionDialogModule } from './import-transaction-dialog/import-transaction-dialog.module'; @@ -23,7 +24,7 @@ import { TransactionsPageComponent } from './transactions-page.component'; RouterModule, TransactionsPageRoutingModule ], - providers: [], + providers: [ImportTransactionsService], schemas: [CUSTOM_ELEMENTS_SCHEMA] }) export class TransactionsPageModule {} diff --git a/apps/client/src/app/services/data.service.ts b/apps/client/src/app/services/data.service.ts index 3bac0486d..30f64d660 100644 --- a/apps/client/src/app/services/data.service.ts +++ b/apps/client/src/app/services/data.service.ts @@ -194,10 +194,6 @@ export class DataService { return this.http.post(`/api/account`, aAccount); } - public postImport(aImportData: ImportDataDto) { - return this.http.post('/api/import', aImportData); - } - public postOrder(aOrder: CreateOrderDto) { return this.http.post(`/api/order`, aOrder); } diff --git a/apps/client/src/app/services/import-transactions.service.ts b/apps/client/src/app/services/import-transactions.service.ts new file mode 100644 index 000000000..dad503350 --- /dev/null +++ b/apps/client/src/app/services/import-transactions.service.ts @@ -0,0 +1,195 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto'; +import { DataSource, Type } from '@prisma/client'; +import { parse } from 'date-fns'; +import { isNumber } from 'lodash'; +import { parse as csvToJson } from 'papaparse'; +import { EMPTY } from 'rxjs'; +import { catchError } from 'rxjs/operators'; + +@Injectable({ + providedIn: 'root' +}) +export class ImportTransactionsService { + private static CURRENCY_KEYS = ['ccy', 'currency']; + private static DATE_KEYS = ['date']; + private static FEE_KEYS = ['commission', 'fee']; + private static QUANTITY_KEYS = ['qty', 'quantity', 'shares']; + private static SYMBOL_KEYS = ['code', 'symbol']; + private static TYPE_KEYS = ['action', 'type']; + private static UNIT_PRICE_KEYS = ['price', 'unitprice', 'value']; + + public constructor(private http: HttpClient) {} + + public async importCsv({ + defaultAccountId, + fileContent, + primaryDataSource + }: { + defaultAccountId: string; + fileContent: string; + primaryDataSource: DataSource; + }) { + const content = csvToJson(fileContent, { + dynamicTyping: true, + header: true, + skipEmptyLines: true + }).data; + + const orders: CreateOrderDto[] = []; + + for (const item of content) { + orders.push({ + accountId: defaultAccountId, + currency: this.parseCurrency(item), + dataSource: primaryDataSource, + date: this.parseDate(item), + fee: this.parseFee(item), + quantity: this.parseQuantity(item), + symbol: this.parseSymbol(item), + type: this.parseType(item), + unitPrice: this.parseUnitPrice(item) + }); + } + + await this.importJson({ defaultAccountId, content: orders }); + } + + public importJson({ + content, + defaultAccountId + }: { + content: CreateOrderDto[]; + defaultAccountId: string; + }): Promise { + return new Promise((resolve, reject) => { + this.postImport({ + orders: content.map((order) => { + return { ...order, accountId: defaultAccountId }; + }) + }) + .pipe( + catchError((error) => { + reject(error); + return EMPTY; + }) + ) + .subscribe({ + next: () => { + resolve(); + } + }); + }); + } + + private lowercaseKeys(aObject: any) { + return Object.keys(aObject).reduce((acc, key) => { + acc[key.toLowerCase()] = aObject[key]; + return acc; + }, {}); + } + + private parseCurrency(aItem: any) { + const item = this.lowercaseKeys(aItem); + + for (const key of ImportTransactionsService.CURRENCY_KEYS) { + if (item[key]) { + return item[key]; + } + } + + throw new Error('Could not parse currency'); + } + + private parseDate(aItem: any) { + const item = this.lowercaseKeys(aItem); + let date: string; + + for (const key of ImportTransactionsService.DATE_KEYS) { + if (item[key]) { + try { + date = parse(item[key], 'dd-MM-yyyy', new Date()).toISOString(); + } catch {} + + try { + date = parse(item[key], 'dd/MM/yyyy', new Date()).toISOString(); + } catch {} + + if (date) { + return date; + } + } + } + + throw new Error('Could not parse date'); + } + + private parseFee(aItem: any) { + const item = this.lowercaseKeys(aItem); + + for (const key of ImportTransactionsService.FEE_KEYS) { + if ((item[key] || item[key] === 0) && isNumber(item[key])) { + return item[key]; + } + } + + throw new Error('Could not parse fee'); + } + + private parseQuantity(aItem: any) { + const item = this.lowercaseKeys(aItem); + + for (const key of ImportTransactionsService.QUANTITY_KEYS) { + if (item[key] && isNumber(item[key])) { + return item[key]; + } + } + + throw new Error('Could not parse quantity'); + } + + private parseSymbol(aItem: any) { + const item = this.lowercaseKeys(aItem); + + for (const key of ImportTransactionsService.SYMBOL_KEYS) { + if (item[key]) { + return item[key]; + } + } + + throw new Error('Could not parse symbol'); + } + + private parseType(aItem: any) { + const item = this.lowercaseKeys(aItem); + + for (const key of ImportTransactionsService.TYPE_KEYS) { + if (item[key]) { + if (item[key].toLowerCase() === 'buy') { + return Type.BUY; + } else if (item[key].toLowerCase() === 'sell') { + return Type.SELL; + } + } + } + + throw new Error('Could not parse type'); + } + + private parseUnitPrice(aItem: any) { + const item = this.lowercaseKeys(aItem); + + for (const key of ImportTransactionsService.UNIT_PRICE_KEYS) { + if (item[key] && isNumber(item[key])) { + return item[key]; + } + } + + throw new Error('Could not parse unit price (unitPrice)'); + } + + private postImport(aImportData: { orders: CreateOrderDto[] }) { + return this.http.post('/api/import', aImportData); + } +} diff --git a/libs/common/src/lib/interfaces/info-item.interface.ts b/libs/common/src/lib/interfaces/info-item.interface.ts index 9fad3d393..35061cc75 100644 --- a/libs/common/src/lib/interfaces/info-item.interface.ts +++ b/libs/common/src/lib/interfaces/info-item.interface.ts @@ -1,3 +1,4 @@ +import { DataSource } from '@prisma/client'; import { Statistics } from './statistics.interface'; import { Subscription } from './subscription.interface'; @@ -11,6 +12,7 @@ export interface InfoItem { type: string; }; platforms: { id: string; name: string }[]; + primaryDataSource: DataSource; statistics: Statistics; stripePublicKey?: string; subscriptions: Subscription[]; diff --git a/package.json b/package.json index bb631c356..1910f4106 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "@simplewebauthn/server": "4.1.0", "@simplewebauthn/typescript-types": "4.0.0", "@stripe/stripe-js": "1.15.0", + "@types/papaparse": "5.2.6", "alphavantage": "2.2.0", "angular-material-css-vars": "2.1.2", "bent": "7.3.12", @@ -99,6 +100,7 @@ "ngx-markdown": "12.0.1", "ngx-skeleton-loader": "2.9.1", "ngx-stripe": "12.0.2", + "papaparse": "5.3.1", "passport": "0.4.1", "passport-google-oauth20": "2.0.0", "passport-jwt": "4.0.0", diff --git a/yarn.lock b/yarn.lock index eae139db3..8ff8413c7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3920,6 +3920,13 @@ resolved "https://registry.yarnpkg.com/@types/overlayscrollbars/-/overlayscrollbars-1.12.1.tgz#fb637071b545834fb12aea94ee309a2ff4cdc0a8" integrity sha512-V25YHbSoKQN35UasHf0EKD9U2vcmexRSp78qa8UglxFH8H3D+adEa9zGZwrqpH4TdvqeMrgMqVqsLB4woAryrQ== +"@types/papaparse@5.2.6": + version "5.2.6" + resolved "https://registry.yarnpkg.com/@types/papaparse/-/papaparse-5.2.6.tgz#0bba18de4d15eff65883bc7c0794e0134de9e7c7" + integrity sha512-xGKSd0UTn58N1h0+zf8mW863Rv8BvXcGibEgKFtBIXZlcDXAmX/T4RdDO2mwmrmOypUDt5vRgo2v32a78JdqUA== + dependencies: + "@types/node" "*" + "@types/parse-json@^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" @@ -13146,6 +13153,11 @@ pako@^1.0.3, pako@~1.0.5: resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== +papaparse@5.3.1: + version "5.3.1" + resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.3.1.tgz#770b7a9124d821d4b2132132b7bd7dce7194b5b1" + integrity sha512-Dbt2yjLJrCwH2sRqKFFJaN5XgIASO9YOFeFP8rIBRG2Ain8mqk5r1M6DkfvqEVozVcz3r3HaUGw253hA1nLIcA== + parallel-transform@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/parallel-transform/-/parallel-transform-1.2.0.tgz#9049ca37d6cb2182c3b1d2c720be94d14a5814fc"