From 222c76364a574fe8834acd2d8de889753650d62d Mon Sep 17 00:00:00 2001
From: Thomas <4159106+dtslvr@users.noreply.github.com>
Date: Sun, 18 Apr 2021 17:09:48 +0200
Subject: [PATCH] Optimize data gathering and exchange rate calculation
---
CHANGELOG.md | 5 ++
.../src/services/data-gathering.service.ts | 6 ++-
.../services/exchange-rate-data.service.ts | 52 +++++++++++++------
.../app/pages/admin/admin-page.component.ts | 16 ++++--
.../src/app/pages/admin/admin-page.html | 11 ++--
libs/helper/src/lib/config.ts | 5 +-
6 files changed, 66 insertions(+), 29 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 79422434d..8e54030b0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added a validation for environment variables
- Added support for feature flags to simplify the initial project setup
+### Changed
+
+- Optimized the data management for historical data
+- Optimized the exchange rate service
+
## 0.85.0 - 16.04.2021
### Changed
diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts
index dcf3d0fa5..d403b83d2 100644
--- a/apps/api/src/services/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering.service.ts
@@ -66,9 +66,11 @@ export class DataGatheringService {
}
public async gatherMax() {
- const isDataGatheringNeeded = await this.isDataGatheringNeeded();
+ const isDataGatheringLocked = await this.prisma.property.findUnique({
+ where: { key: 'LOCKED_DATA_GATHERING' }
+ });
- if (isDataGatheringNeeded) {
+ if (!isDataGatheringLocked) {
console.log('Max data gathering has been started.');
console.time('data-gathering');
diff --git a/apps/api/src/services/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data.service.ts
index dbca06336..ee63e6729 100644
--- a/apps/api/src/services/exchange-rate-data.service.ts
+++ b/apps/api/src/services/exchange-rate-data.service.ts
@@ -1,7 +1,7 @@
+import { getYesterday } from '@ghostfolio/helper';
import { Injectable } from '@nestjs/common';
import { Currency } from '@prisma/client';
import { format } from 'date-fns';
-import { getYesterday } from 'libs/helper/src';
import { DataProviderService } from './data-provider.service';
@@ -15,6 +15,8 @@ export class ExchangeRateDataService {
}
public async initialize() {
+ this.pairs = [];
+
this.addPairs(Currency.CHF, Currency.EUR);
this.addPairs(Currency.CHF, Currency.GBP);
this.addPairs(Currency.CHF, Currency.USD);
@@ -25,11 +27,6 @@ export class ExchangeRateDataService {
await this.loadCurrencies();
}
- private addPairs(aCurrency1: Currency, aCurrency2: Currency) {
- this.pairs.push(`${aCurrency1}${aCurrency2}`);
- this.pairs.push(`${aCurrency2}${aCurrency1}`);
- }
-
public async loadCurrencies() {
const result = await this.dataProviderService.getHistorical(
this.pairs,
@@ -38,19 +35,34 @@ export class ExchangeRateDataService {
getYesterday()
);
+ const resultExtended = result;
+
+ Object.keys(result).forEach((pair) => {
+ const [currency1, currency2] = pair.match(/.{1,3}/g);
+ const [date] = Object.keys(result[pair]);
+
+ // Calculate the opposite direction
+ resultExtended[`${currency2}${currency1}`] = {
+ [date]: {
+ marketPrice: 1 / result[pair][date].marketPrice
+ }
+ };
+ });
+
this.pairs.forEach((pair) => {
- this.currencies[pair] =
- result[pair]?.[format(getYesterday(), 'yyyy-MM-dd')]?.marketPrice || 1;
+ const [currency1, currency2] = pair.match(/.{1,3}/g);
+ const date = format(getYesterday(), 'yyyy-MM-dd');
- if (this.currencies[pair] === 1) {
- // Calculate the other direction
- const [currency1, currency2] = pair.match(/.{1,3}/g);
+ this.currencies[pair] = resultExtended[pair]?.[date]?.marketPrice;
+ if (!this.currencies[pair]) {
+ // Not found, calculate indirectly via USD
this.currencies[pair] =
- 1 /
- result[`${currency2}${currency1}`]?.[
- format(getYesterday(), 'yyyy-MM-dd')
- ]?.marketPrice;
+ resultExtended[`${currency1}${Currency.USD}`][date].marketPrice *
+ resultExtended[`${Currency.USD}${currency2}`][date].marketPrice;
+
+ // Calculate the opposite direction
+ this.currencies[`${currency2}${currency1}`] = 1 / this.currencies[pair];
}
});
}
@@ -60,6 +72,11 @@ export class ExchangeRateDataService {
aFromCurrency: Currency,
aToCurrency: Currency
) {
+ if (isNaN(this.currencies[`${Currency.USD}${Currency.CHF}`])) {
+ // Reinitialize if data is not loaded correctly
+ this.initialize();
+ }
+
let factor = 1;
if (aFromCurrency !== aToCurrency) {
@@ -68,4 +85,9 @@ export class ExchangeRateDataService {
return factor * aValue;
}
+
+ private addPairs(aCurrency1: Currency, aCurrency2: Currency) {
+ this.pairs.push(`${aCurrency1}${aCurrency2}`);
+ this.pairs.push(`${aCurrency2}${aCurrency1}`);
+ }
}
diff --git a/apps/client/src/app/pages/admin/admin-page.component.ts b/apps/client/src/app/pages/admin/admin-page.component.ts
index 0bbfc10fb..accce45e8 100644
--- a/apps/client/src/app/pages/admin/admin-page.component.ts
+++ b/apps/client/src/app/pages/admin/admin-page.component.ts
@@ -83,11 +83,17 @@ export class AdminPageComponent implements OnInit {
}
public onGatherMax() {
- this.adminService.gatherMax().subscribe(() => {
- setTimeout(() => {
- window.location.reload();
- }, 300);
- });
+ const confirmation = confirm(
+ 'This action may take some time. Do you want to proceed?'
+ );
+
+ if (confirmation === true) {
+ this.adminService.gatherMax().subscribe(() => {
+ setTimeout(() => {
+ window.location.reload();
+ }, 300);
+ });
+ }
}
public formatDistanceToNow(aDateString: string) {
diff --git a/apps/client/src/app/pages/admin/admin-page.html b/apps/client/src/app/pages/admin/admin-page.html
index be2ed17b3..832d5ef18 100644
--- a/apps/client/src/app/pages/admin/admin-page.html
+++ b/apps/client/src/app/pages/admin/admin-page.html
@@ -34,11 +34,16 @@
(click)="onFlushCache()"
>
- Reset
+ Reset Data Gathering
-