Browse Source

Support import of csv files

pull/419/head
Thomas 4 years ago
parent
commit
cacd0a7e79
  1. 71
      apps/client/src/app/pages/portfolio/transactions/transactions-page.component.ts
  2. 3
      apps/client/src/app/pages/portfolio/transactions/transactions-page.module.ts
  3. 4
      apps/client/src/app/services/data.service.ts
  4. 193
      apps/client/src/app/services/import-transactions.service.ts
  5. 2
      package.json
  6. 12
      yarn.lock

71
apps/client/src/app/pages/portfolio/transactions/transactions-page.component.ts

@ -6,14 +6,15 @@ import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
import { UpdateOrderDto } from '@ghostfolio/api/app/order/update-order.dto'; import { UpdateOrderDto } from '@ghostfolio/api/app/order/update-order.dto';
import { DataService } from '@ghostfolio/client/services/data.service'; import { DataService } from '@ghostfolio/client/services/data.service';
import { ImpersonationStorageService } from '@ghostfolio/client/services/impersonation-storage.service'; import { ImpersonationStorageService } from '@ghostfolio/client/services/impersonation-storage.service';
import { ImportTransactionsService } from '@ghostfolio/client/services/import-transactions.service';
import { UserService } from '@ghostfolio/client/services/user/user.service'; import { UserService } from '@ghostfolio/client/services/user/user.service';
import { User } from '@ghostfolio/common/interfaces'; import { User } from '@ghostfolio/common/interfaces';
import { hasPermission, permissions } from '@ghostfolio/common/permissions'; import { hasPermission, permissions } from '@ghostfolio/common/permissions';
import { Order as OrderModel } from '@prisma/client'; import { Order as OrderModel } from '@prisma/client';
import { format, parseISO } from 'date-fns'; import { format, parseISO } from 'date-fns';
import { DeviceDetectorService } from 'ngx-device-detector'; import { DeviceDetectorService } from 'ngx-device-detector';
import { EMPTY, Subject, Subscription } from 'rxjs'; import { Subject, Subscription } from 'rxjs';
import { catchError, takeUntil } from 'rxjs/operators'; import { takeUntil } from 'rxjs/operators';
import { CreateOrUpdateTransactionDialog } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.component'; import { CreateOrUpdateTransactionDialog } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.component';
import { ImportTransactionDialog } from './import-transaction-dialog/import-transaction-dialog.component'; import { ImportTransactionDialog } from './import-transaction-dialog/import-transaction-dialog.component';
@ -46,6 +47,7 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
private deviceService: DeviceDetectorService, private deviceService: DeviceDetectorService,
private dialog: MatDialog, private dialog: MatDialog,
private impersonationStorageService: ImpersonationStorageService, private impersonationStorageService: ImpersonationStorageService,
private importTransactionsService: ImportTransactionsService,
private route: ActivatedRoute, private route: ActivatedRoute,
private router: Router, private router: Router,
private snackBar: MatSnackBar, private snackBar: MatSnackBar,
@ -58,8 +60,8 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
this.openCreateTransactionDialog(); this.openCreateTransactionDialog();
} else if (params['editDialog']) { } else if (params['editDialog']) {
if (this.transactions) { if (this.transactions) {
const transaction = this.transactions.find((transaction) => { const transaction = this.transactions.find(({ id }) => {
return transaction.id === params['transactionId']; return id === params['transactionId'];
}); });
this.openUpdateTransactionDialog(transaction); this.openUpdateTransactionDialog(transaction);
@ -167,6 +169,8 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
input.type = 'file'; input.type = 'file';
input.onchange = (event) => { input.onchange = (event) => {
this.snackBar.open('⏳ Importing data...');
// Getting the file reference // Getting the file reference
const file = (event.target as HTMLInputElement).files[0]; const file = (event.target as HTMLInputElement).files[0];
@ -174,35 +178,42 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
const reader = new FileReader(); const reader = new FileReader();
reader.readAsText(file, 'UTF-8'); reader.readAsText(file, 'UTF-8');
reader.onload = (readerEvent) => { reader.onload = async (readerEvent) => {
try { const fileContent = readerEvent.target.result as string;
const content = JSON.parse(readerEvent.target.result as string);
this.snackBar.open('⏳ Importing data...'); try {
if (file.type === 'application/json') {
const content = JSON.parse(fileContent);
try {
await this.importTransactionsService.importJson({
content: content.orders,
defaultAccountId: this.defaultAccountId
});
this.dataService this.handleImportSuccess();
.postImport({ } catch (error) {
orders: content.orders.map((order) => {
return { ...order, accountId: this.defaultAccountId };
})
})
.pipe(
catchError((error) => {
this.handleImportError(error); this.handleImportError(error);
}
return EMPTY; return;
}), } else if (file.type === 'text/csv') {
takeUntil(this.unsubscribeSubject) try {
) await this.importTransactionsService.importCsv({
.subscribe({ fileContent,
next: () => { defaultAccountId: this.defaultAccountId
this.fetchOrders(); });
this.snackBar.open('✅ Import has been completed', undefined, { this.handleImportSuccess();
duration: 3000 } catch (error) {
this.handleImportError({
error: { message: error?.error?.message ?? [error?.message] }
}); });
} }
});
return;
}
throw new Error();
} catch (error) { } catch (error) {
this.handleImportError({ error: { message: ['Unexpected format'] } }); this.handleImportError({ error: { message: ['Unexpected format'] } });
} }
@ -302,6 +313,14 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
}); });
} }
private handleImportSuccess() {
this.fetchOrders();
this.snackBar.open('✅ Import has been completed', undefined, {
duration: 3000
});
}
private openCreateTransactionDialog(aTransaction?: OrderModel): void { private openCreateTransactionDialog(aTransaction?: OrderModel): void {
const dialogRef = this.dialog.open(CreateOrUpdateTransactionDialog, { const dialogRef = this.dialog.open(CreateOrUpdateTransactionDialog, {
data: { data: {

3
apps/client/src/app/pages/portfolio/transactions/transactions-page.module.ts

@ -4,6 +4,7 @@ import { MatButtonModule } from '@angular/material/button';
import { MatSnackBarModule } from '@angular/material/snack-bar'; import { MatSnackBarModule } from '@angular/material/snack-bar';
import { RouterModule } from '@angular/router'; import { RouterModule } from '@angular/router';
import { GfTransactionsTableModule } from '@ghostfolio/client/components/transactions-table/transactions-table.module'; import { GfTransactionsTableModule } from '@ghostfolio/client/components/transactions-table/transactions-table.module';
import { ImportTransactionsService } from '@ghostfolio/client/services/import-transactions.service';
import { GfCreateOrUpdateTransactionDialogModule } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.module'; import { GfCreateOrUpdateTransactionDialogModule } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.module';
import { GfImportTransactionDialogModule } from './import-transaction-dialog/import-transaction-dialog.module'; import { GfImportTransactionDialogModule } from './import-transaction-dialog/import-transaction-dialog.module';
@ -23,7 +24,7 @@ import { TransactionsPageComponent } from './transactions-page.component';
RouterModule, RouterModule,
TransactionsPageRoutingModule TransactionsPageRoutingModule
], ],
providers: [], providers: [ImportTransactionsService],
schemas: [CUSTOM_ELEMENTS_SCHEMA] schemas: [CUSTOM_ELEMENTS_SCHEMA]
}) })
export class TransactionsPageModule {} export class TransactionsPageModule {}

4
apps/client/src/app/services/data.service.ts

@ -194,10 +194,6 @@ export class DataService {
return this.http.post<OrderModel>(`/api/account`, aAccount); return this.http.post<OrderModel>(`/api/account`, aAccount);
} }
public postImport(aImportData: ImportDataDto) {
return this.http.post<void>('/api/import', aImportData);
}
public postOrder(aOrder: CreateOrderDto) { public postOrder(aOrder: CreateOrderDto) {
return this.http.post<OrderModel>(`/api/order`, aOrder); return this.http.post<OrderModel>(`/api/order`, aOrder);
} }

193
apps/client/src/app/services/import-transactions.service.ts

@ -0,0 +1,193 @@
import { HttpClient } from '@angular/common/http';
import { Injectable } from '@angular/core';
import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
import { Type } from '@prisma/client';
import { parse } from 'date-fns';
import { isNumber } from 'lodash';
import { parse as csvToJson } from 'papaparse';
import { EMPTY } from 'rxjs';
import { catchError } from 'rxjs/operators';
@Injectable({
providedIn: 'root'
})
export class ImportTransactionsService {
private static CURRENCY_KEYS = ['ccy', 'currency'];
private static DATE_KEYS = ['date'];
private static FEE_KEYS = ['fee'];
private static QUANTITY_KEYS = ['qty', 'quantity'];
private static SYMBOL_KEYS = ['code', 'symbol'];
private static TYPE_KEYS = ['action', 'type'];
private static UNIT_PRICE_KEYS = ['price', 'unitprice'];
public constructor(private http: HttpClient) {}
public async importCsv({
defaultAccountId,
fileContent
}: {
defaultAccountId: string;
fileContent: string;
}) {
const content = csvToJson(fileContent, {
dynamicTyping: true,
header: true,
skipEmptyLines: true
}).data;
const orders: CreateOrderDto[] = [];
for (const item of content) {
orders.push({
accountId: defaultAccountId,
currency: this.parseCurrency(item),
dataSource: 'YAHOO',
date: this.parseDate(item),
fee: this.parseFee(item),
quantity: this.parseQuantity(item),
symbol: this.parseSymbol(item),
type: this.parseType(item),
unitPrice: this.parseUnitPrice(item)
});
}
await this.importJson({ defaultAccountId, content: orders });
}
public importJson({
content,
defaultAccountId
}: {
content: CreateOrderDto[];
defaultAccountId: string;
}): Promise<void> {
return new Promise((resolve, reject) => {
this.postImport({
orders: content.map((order) => {
return { ...order, accountId: defaultAccountId };
})
})
.pipe(
catchError((error) => {
reject(error);
return EMPTY;
})
)
.subscribe({
next: () => {
resolve();
}
});
});
}
private lowercaseKeys(aObject: any) {
return Object.keys(aObject).reduce((acc, key) => {
acc[key.toLowerCase()] = aObject[key];
return acc;
}, {});
}
private parseCurrency(aItem: any) {
const item = this.lowercaseKeys(aItem);
for (const key of ImportTransactionsService.CURRENCY_KEYS) {
if (item[key]) {
return item[key];
}
}
throw new Error('Could not parse currency');
}
private parseDate(aItem: any) {
const item = this.lowercaseKeys(aItem);
let date: string;
for (const key of ImportTransactionsService.DATE_KEYS) {
if (item[key]) {
try {
date = parse(item[key], 'dd-MM-yyyy', new Date()).toISOString();
} catch {}
try {
date = parse(item[key], 'dd/MM/yyyy', new Date()).toISOString();
} catch {}
if (date) {
return date;
}
}
}
throw new Error('Could not parse date');
}
private parseFee(aItem: any) {
const item = this.lowercaseKeys(aItem);
for (const key of ImportTransactionsService.FEE_KEYS) {
if ((item[key] || item[key] === 0) && isNumber(item[key])) {
return item[key];
}
}
throw new Error('Could not parse fee');
}
private parseQuantity(aItem: any) {
const item = this.lowercaseKeys(aItem);
for (const key of ImportTransactionsService.QUANTITY_KEYS) {
if (item[key] && isNumber(item[key])) {
return item[key];
}
}
throw new Error('Could not parse quantity');
}
private parseSymbol(aItem: any) {
const item = this.lowercaseKeys(aItem);
for (const key of ImportTransactionsService.SYMBOL_KEYS) {
if (item[key]) {
return item[key];
}
}
throw new Error('Could not parse symbol');
}
private parseType(aItem: any) {
const item = this.lowercaseKeys(aItem);
for (const key of ImportTransactionsService.TYPE_KEYS) {
if (item[key]) {
if (item[key].toLowerCase() === 'buy') {
return Type.BUY;
} else if (item[key].toLowerCase() === 'sell') {
return Type.SELL;
}
}
}
throw new Error('Could not parse type');
}
private parseUnitPrice(aItem: any) {
const item = this.lowercaseKeys(aItem);
for (const key of ImportTransactionsService.UNIT_PRICE_KEYS) {
if (item[key] && isNumber(item[key])) {
return item[key];
}
}
throw new Error('Could not parse unit price (unitPrice)');
}
private postImport(aImportData: { orders: CreateOrderDto[] }) {
return this.http.post<void>('/api/import', aImportData);
}
}

2
package.json

@ -73,6 +73,7 @@
"@simplewebauthn/server": "4.1.0", "@simplewebauthn/server": "4.1.0",
"@simplewebauthn/typescript-types": "4.0.0", "@simplewebauthn/typescript-types": "4.0.0",
"@stripe/stripe-js": "1.15.0", "@stripe/stripe-js": "1.15.0",
"@types/papaparse": "5.2.6",
"alphavantage": "2.2.0", "alphavantage": "2.2.0",
"angular-material-css-vars": "2.1.2", "angular-material-css-vars": "2.1.2",
"bent": "7.3.12", "bent": "7.3.12",
@ -99,6 +100,7 @@
"ngx-markdown": "12.0.1", "ngx-markdown": "12.0.1",
"ngx-skeleton-loader": "2.9.1", "ngx-skeleton-loader": "2.9.1",
"ngx-stripe": "12.0.2", "ngx-stripe": "12.0.2",
"papaparse": "5.3.1",
"passport": "0.4.1", "passport": "0.4.1",
"passport-google-oauth20": "2.0.0", "passport-google-oauth20": "2.0.0",
"passport-jwt": "4.0.0", "passport-jwt": "4.0.0",

12
yarn.lock

@ -3920,6 +3920,13 @@
resolved "https://registry.yarnpkg.com/@types/overlayscrollbars/-/overlayscrollbars-1.12.1.tgz#fb637071b545834fb12aea94ee309a2ff4cdc0a8" resolved "https://registry.yarnpkg.com/@types/overlayscrollbars/-/overlayscrollbars-1.12.1.tgz#fb637071b545834fb12aea94ee309a2ff4cdc0a8"
integrity sha512-V25YHbSoKQN35UasHf0EKD9U2vcmexRSp78qa8UglxFH8H3D+adEa9zGZwrqpH4TdvqeMrgMqVqsLB4woAryrQ== integrity sha512-V25YHbSoKQN35UasHf0EKD9U2vcmexRSp78qa8UglxFH8H3D+adEa9zGZwrqpH4TdvqeMrgMqVqsLB4woAryrQ==
"@types/papaparse@5.2.6":
version "5.2.6"
resolved "https://registry.yarnpkg.com/@types/papaparse/-/papaparse-5.2.6.tgz#0bba18de4d15eff65883bc7c0794e0134de9e7c7"
integrity sha512-xGKSd0UTn58N1h0+zf8mW863Rv8BvXcGibEgKFtBIXZlcDXAmX/T4RdDO2mwmrmOypUDt5vRgo2v32a78JdqUA==
dependencies:
"@types/node" "*"
"@types/parse-json@^4.0.0": "@types/parse-json@^4.0.0":
version "4.0.0" version "4.0.0"
resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0"
@ -13146,6 +13153,11 @@ pako@^1.0.3, pako@~1.0.5:
resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
papaparse@5.3.1:
version "5.3.1"
resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.3.1.tgz#770b7a9124d821d4b2132132b7bd7dce7194b5b1"
integrity sha512-Dbt2yjLJrCwH2sRqKFFJaN5XgIASO9YOFeFP8rIBRG2Ain8mqk5r1M6DkfvqEVozVcz3r3HaUGw253hA1nLIcA==
parallel-transform@^1.1.0: parallel-transform@^1.1.0:
version "1.2.0" version "1.2.0"
resolved "https://registry.yarnpkg.com/parallel-transform/-/parallel-transform-1.2.0.tgz#9049ca37d6cb2182c3b1d2c720be94d14a5814fc" resolved "https://registry.yarnpkg.com/parallel-transform/-/parallel-transform-1.2.0.tgz#9049ca37d6cb2182c3b1d2c720be94d14a5814fc"

Loading…
Cancel
Save