Optimize data gathering and exchange rate calculation

CHANGELOG.md

@@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added a validation for environment variables
 - Added support for feature flags to simplify the initial project setup
 
+### Changed
+
+- Optimized the data management for historical data
+- Optimized the exchange rate service
+
 ## 0.85.0 - 16.04.2021
 
 ### Changed

apps/api/src/services/data-gathering.service.ts

@@ -66,9 +66,11 @@ export class DataGatheringService {
   }
 
   public async gatherMax() {
-    const isDataGatheringNeeded = await this.isDataGatheringNeeded();
+    const isDataGatheringLocked = await this.prisma.property.findUnique({
+      where: { key: 'LOCKED_DATA_GATHERING' }
+    });
 
-    if (isDataGatheringNeeded) {
+    if (!isDataGatheringLocked) {
       console.log('Max data gathering has been started.');
       console.time('data-gathering');
 

apps/api/src/services/exchange-rate-data.service.ts

@@ -1,7 +1,7 @@
+import { getYesterday } from '@ghostfolio/helper';
 import { Injectable } from '@nestjs/common';
 import { Currency } from '@prisma/client';
 import { format } from 'date-fns';
-import { getYesterday } from 'libs/helper/src';
 
 import { DataProviderService } from './data-provider.service';
 
@@ -15,6 +15,8 @@ export class ExchangeRateDataService {
   }
 
   public async initialize() {
+    this.pairs = [];
+
     this.addPairs(Currency.CHF, Currency.EUR);
     this.addPairs(Currency.CHF, Currency.GBP);
     this.addPairs(Currency.CHF, Currency.USD);
@@ -25,11 +27,6 @@ export class ExchangeRateDataService {
     await this.loadCurrencies();
   }
 
-  private addPairs(aCurrency1: Currency, aCurrency2: Currency) {
-    this.pairs.push(`${aCurrency1}${aCurrency2}`);
-    this.pairs.push(`${aCurrency2}${aCurrency1}`);
-  }
-
   public async loadCurrencies() {
     const result = await this.dataProviderService.getHistorical(
       this.pairs,
@@ -38,19 +35,34 @@ export class ExchangeRateDataService {
       getYesterday()
     );
 
-    this.pairs.forEach((pair) => {
+    const resultExtended = result;
-      this.currencies[pair] =
+
-        result[pair]?.[format(getYesterday(), 'yyyy-MM-dd')]?.marketPrice || 1;
+    Object.keys(result).forEach((pair) => {
+      const [currency1, currency2] = pair.match(/.{1,3}/g);
+      const [date] = Object.keys(result[pair]);
+
+      // Calculate the opposite direction
+      resultExtended[`${currency2}${currency1}`] = {
+        [date]: {
+          marketPrice: 1 / result[pair][date].marketPrice
+        }
+      };
+    });
 
-      if (this.currencies[pair] === 1) {
+    this.pairs.forEach((pair) => {
-        // Calculate the other direction
       const [currency1, currency2] = pair.match(/.{1,3}/g);
+      const date = format(getYesterday(), 'yyyy-MM-dd');
+
+      this.currencies[pair] = resultExtended[pair]?.[date]?.marketPrice;
 
+      if (!this.currencies[pair]) {
+        // Not found, calculate indirectly via USD
         this.currencies[pair] =
-          1 /
+          resultExtended[`${currency1}${Currency.USD}`][date].marketPrice *
-          result[`${currency2}${currency1}`]?.[
+          resultExtended[`${Currency.USD}${currency2}`][date].marketPrice;
-            format(getYesterday(), 'yyyy-MM-dd')
+
-          ]?.marketPrice;
+        // Calculate the opposite direction
+        this.currencies[`${currency2}${currency1}`] = 1 / this.currencies[pair];
       }
     });
   }
@@ -60,6 +72,11 @@ export class ExchangeRateDataService {
     aFromCurrency: Currency,
     aToCurrency: Currency
   ) {
+    if (isNaN(this.currencies[`${Currency.USD}${Currency.CHF}`])) {
+      // Reinitialize if data is not loaded correctly
+      this.initialize();
+    }
+
     let factor = 1;
 
     if (aFromCurrency !== aToCurrency) {
@@ -68,4 +85,9 @@ export class ExchangeRateDataService {
 
     return factor * aValue;
   }
+
+  private addPairs(aCurrency1: Currency, aCurrency2: Currency) {
+    this.pairs.push(`${aCurrency1}${aCurrency2}`);
+    this.pairs.push(`${aCurrency2}${aCurrency1}`);
+  }
 }

apps/client/src/app/pages/admin/admin-page.component.ts

@@ -83,12 +83,18 @@ export class AdminPageComponent implements OnInit {
   }
 
   public onGatherMax() {
+    const confirmation = confirm(
+      'This action may take some time. Do you want to proceed?'
+    );
+
+    if (confirmation === true) {
       this.adminService.gatherMax().subscribe(() => {
         setTimeout(() => {
           window.location.reload();
         }, 300);
       });
     }
+  }
 
   public formatDistanceToNow(aDateString: string) {
     const distanceString = formatDistanceToNow(

apps/client/src/app/pages/admin/admin-page.html

@@ -34,11 +34,16 @@
                   (click)="onFlushCache()"
                 >
                   <ion-icon class="mr-1" name="close-circle-outline"></ion-icon>
-                  <span i18n>Reset</span>
+                  <span i18n>Reset Data Gathering</span>
                 </button>
-                <button color="warn" mat-flat-button (click)="onGatherMax()">
+                <button
+                  color="warn"
+                  mat-flat-button
+                  [disabled]="dataGatheringInProgress"
+                  (click)="onGatherMax()"
+                >
                   <ion-icon class="mr-1" name="warning-outline"></ion-icon>
-                  <span i18n>Gather Max</span>
+                  <span i18n>Gather All Data</span>
                 </button>
               </div>
             </div>

libs/helper/src/lib/config.ts

@@ -2,12 +2,9 @@ import { Currency } from '.prisma/client';
 
 export const baseCurrency = Currency.CHF;
 
-export const benchmarks = ['CSSMIM.SW', 'GC=F', 'VOO', 'VTI', 'VWRD.L', 'VXUS'];
+export const benchmarks = ['VOO'];
 
 export const currencyPairs = [
-  `${Currency.EUR}${Currency.CHF}`,
-  `${Currency.GBP}${Currency.CHF}`,
-  `${Currency.GBP}${Currency.EUR}`,
   `${Currency.USD}${Currency.EUR}`,
   `${Currency.USD}${Currency.GBP}`,
   `${Currency.USD}${Currency.CHF}`