Merge branch 'main' into feature/upgrade-prisma-to-version-4.12.0

CHANGELOG.md

@@ -9,10 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Skipped creating queue jobs for asset profiles with `MANUAL` data source not having a scraper configuration
+- Reduced the execution interval of the data gathering to every hour
 - Upgraded `prisma` from version `4.11.0` to `4.12.0`
 
 ### Fixed
 
+- Improved the style of the system message
+
+## 1.254.0 - 2023-04-14
+
+### Changed
+
+- Improved the queue jobs implementation by adding in bulk
+- Improved the queue jobs implementation by introducing unique job ids
+- Reverted the execution interval of the data gathering from every 12 hours to every 4 hours
+
+## 1.253.0 - 2023-04-14
+
+### Changed
+
+- Reduced the execution interval of the data gathering to every 12 hours
+
+### Fixed
+
 - Fixed the background color of dialogs in dark mode
 
 ## 1.252.2 - 2023-04-11

apps/api/src/app/admin/admin.controller.ts

@@ -100,16 +100,21 @@ export class AdminController {
 
     const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    for (const { dataSource, symbol } of uniqueAssets) {
+    await this.dataGatheringService.addJobsToQueue(
-      await this.dataGatheringService.addJobToQueue(
+      uniqueAssets.map(({ dataSource, symbol }) => {
-        GATHER_ASSET_PROFILE_PROCESS,
+        return {
-        {
+          data: {
-          dataSource,
+            dataSource,
-          symbol
+            symbol
-        },
+          },
-        GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+          name: GATHER_ASSET_PROFILE_PROCESS,
-      );
+          opts: {
-    }
+            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}}`
+          }
+        };
+      })
+    );
 
     this.dataGatheringService.gatherMax();
   }
@@ -131,16 +136,21 @@ export class AdminController {
 
     const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    for (const { dataSource, symbol } of uniqueAssets) {
+    await this.dataGatheringService.addJobsToQueue(
-      await this.dataGatheringService.addJobToQueue(
+      uniqueAssets.map(({ dataSource, symbol }) => {
-        GATHER_ASSET_PROFILE_PROCESS,
+        return {
-        {
+          data: {
-          dataSource,
+            dataSource,
-          symbol
+            symbol
-        },
+          },
-        GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+          name: GATHER_ASSET_PROFILE_PROCESS,
-      );
+          opts: {
-    }
+            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}}`
+          }
+        };
+      })
+    );
   }
 
   @Post('gather/profile-data/:dataSource/:symbol')
@@ -161,14 +171,17 @@ export class AdminController {
       );
     }
 
-    await this.dataGatheringService.addJobToQueue(
+    await this.dataGatheringService.addJobToQueue({
-      GATHER_ASSET_PROFILE_PROCESS,
+      data: {
-      {
         dataSource,
         symbol
       },
-      GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+      name: GATHER_ASSET_PROFILE_PROCESS,
-    );
+      opts: {
+        ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+        jobId: `${dataSource}-${symbol}}`
+      }
+    });
   }
 
   @Post('gather/:dataSource/:symbol')

apps/api/src/app/order/order.service.ts

@@ -112,14 +112,17 @@ export class OrderService {
       };
     }
 
-    await this.dataGatheringService.addJobToQueue(
+    await this.dataGatheringService.addJobToQueue({
-      GATHER_ASSET_PROFILE_PROCESS,
+      data: {
-      {
         dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
         symbol: data.SymbolProfile.connectOrCreate.create.symbol
       },
-      GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+      name: GATHER_ASSET_PROFILE_PROCESS,
-    );
+      opts: {
+        ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+        jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}}`
+      }
+    });
 
     const isDraft = isAfter(data.date as Date, endOfToday());
 

apps/api/src/app/portfolio/portfolio-calculator.ts

@@ -722,7 +722,7 @@ export class PortfolioCalculator {
         );
       } else if (!currentPosition.quantity.eq(0)) {
         Logger.warn(
-          `Missing initial value for symbol ${currentPosition.symbol} at ${currentPosition.firstBuyDate}`,
+          `Missing historical market data for symbol ${currentPosition.symbol}`,
           'PortfolioCalculator'
         );
         hasErrors = true;

apps/api/src/services/cron.service.ts

@@ -19,8 +19,8 @@ export class CronService {
     private readonly twitterBotService: TwitterBotService
   ) {}
 
-  @Cron(CronExpression.EVERY_4_HOURS)
+  @Cron(CronExpression.EVERY_HOUR)
-  public async runEveryFourHours() {
+  public async runEveryHour() {
     await this.dataGatheringService.gather7Days();
   }
 
@@ -38,15 +38,20 @@ export class CronService {
   public async runEverySundayAtTwelvePm() {
     const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    for (const { dataSource, symbol } of uniqueAssets) {
+    await this.dataGatheringService.addJobsToQueue(
-      await this.dataGatheringService.addJobToQueue(
+      uniqueAssets.map(({ dataSource, symbol }) => {
-        GATHER_ASSET_PROFILE_PROCESS,
+        return {
-        {
+          data: {
-          dataSource,
+            dataSource,
-          symbol
+            symbol
-        },
+          },
-        GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+          name: GATHER_ASSET_PROFILE_PROCESS,
-      );
+          opts: {
-    }
+            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}}`
+          }
+        };
+      })
+    );
   }
 }

apps/api/src/services/data-gathering.service.ts

@@ -2,8 +2,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.se
 import {
   DATA_GATHERING_QUEUE,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS,
-  GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
+  GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
-  QUEUE_JOB_STATUS_LIST
 } from '@ghostfolio/common/config';
 import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
@@ -12,6 +11,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
 import { JobOptions, Queue } from 'bull';
 import { format, min, subDays, subYears } from 'date-fns';
+import { isEmpty } from 'lodash';
 
 import { DataProviderService } from './data-provider/data-provider.service';
 import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface';
@@ -34,17 +34,22 @@ export class DataGatheringService {
     private readonly symbolProfileService: SymbolProfileService
   ) {}
 
-  public async addJobToQueue(name: string, data: any, options?: JobOptions) {
+  public async addJobToQueue({
-    const hasJob = await this.hasJob(name, data);
+    data,
+    name,
+    opts
+  }: {
+    data: any;
+    name: string;
+    opts?: JobOptions;
+  }) {
+    return this.dataGatheringQueue.add(name, data, opts);
+  }
 
-    if (hasJob) {
+  public async addJobsToQueue(
-      Logger.log(
+    jobs: { data: any; name: string; opts?: JobOptions }[]
-        `Job ${name} with data ${JSON.stringify(data)} already exists.`,
+  ) {
-        'DataGatheringService'
+    return this.dataGatheringQueue.addBulk(jobs);
-      );
-    } else {
-      return this.dataGatheringQueue.add(name, data, options);
-    }
   }
 
   public async gather7Days() {
@@ -209,59 +214,22 @@ export class DataGatheringService {
   }
 
   public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
-    for (const { dataSource, date, symbol } of aSymbolsWithStartDate) {
+    await this.addJobsToQueue(
-      await this.addJobToQueue(
+      aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => {
-        GATHER_HISTORICAL_MARKET_DATA_PROCESS,
-        {
-          dataSource,
-          date,
-          symbol
-        },
-        GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
-      );
-    }
-  }
-
-  public async getSymbolsMax(): Promise<IDataGatheringItem[]> {
-    const startDate =
-      (
-        await this.prismaService.order.findFirst({
-          orderBy: [{ date: 'asc' }]
-        })
-      )?.date ?? new Date();
-
-    const currencyPairsToGather = this.exchangeRateDataService
-      .getCurrencyPairs()
-      .map(({ dataSource, symbol }) => {
         return {
-          dataSource,
+          data: {
-          symbol,
+            dataSource,
-          date: min([startDate, subYears(new Date(), 10)])
+            date,
-        };
+            symbol
-      });
-
-    const symbolProfilesToGather = (
-      await this.prismaService.symbolProfile.findMany({
-        orderBy: [{ symbol: 'asc' }],
-        select: {
-          dataSource: true,
-          Order: {
-            orderBy: [{ date: 'asc' }],
-            select: { date: true },
-            take: 1
           },
-          scraperConfiguration: true,
+          name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
-          symbol: true
+          opts: {
-        }
+            ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}`
+          }
+        };
       })
-    ).map((symbolProfile) => {
+    );
-      return {
-        ...symbolProfile,
-        date: symbolProfile.Order?.[0]?.date ?? startDate
-      };
-    });
-
-    return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 
   public async getUniqueAssets(): Promise<UniqueAsset[]> {
@@ -298,7 +266,7 @@ export class DataGatheringService {
 
     // Only consider symbols with incomplete market data for the last
     // 7 days
-    const symbolsNotToGather = (
+    const symbolsWithCompleteMarketData = (
       await this.prismaService.marketData.groupBy({
         _count: true,
         by: ['symbol'],
@@ -316,8 +284,14 @@ export class DataGatheringService {
       });
 
     const symbolProfilesToGather = symbolProfiles
-      .filter(({ symbol }) => {
+      .filter(({ dataSource, scraperConfiguration, symbol }) => {
-        return !symbolsNotToGather.includes(symbol);
+        const manualDataSourceWithScraperConfiguration =
+          dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);
+
+        return (
+          !symbolsWithCompleteMarketData.includes(symbol) &&
+          (dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration)
+        );
       })
       .map((symbolProfile) => {
         return {
@@ -329,7 +303,7 @@ export class DataGatheringService {
     const currencyPairsToGather = this.exchangeRateDataService
       .getCurrencyPairs()
       .filter(({ symbol }) => {
-        return !symbolsNotToGather.includes(symbol);
+        return !symbolsWithCompleteMarketData.includes(symbol);
       })
       .map(({ dataSource, symbol }) => {
         return {
@@ -342,17 +316,56 @@ export class DataGatheringService {
     return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 
-  private async hasJob(name: string, data: any) {
+  private async getSymbolsMax(): Promise<IDataGatheringItem[]> {
-    const jobs = await this.dataGatheringQueue.getJobs(
+    const startDate =
-      QUEUE_JOB_STATUS_LIST.filter((status) => {
+      (
-        return status !== 'completed';
+        await this.prismaService.order.findFirst({
+          orderBy: [{ date: 'asc' }]
+        })
+      )?.date ?? new Date();
+
+    const currencyPairsToGather = this.exchangeRateDataService
+      .getCurrencyPairs()
+      .map(({ dataSource, symbol }) => {
+        return {
+          dataSource,
+          symbol,
+          date: min([startDate, subYears(new Date(), 10)])
+        };
+      });
+
+    const symbolProfilesToGather = (
+      await this.prismaService.symbolProfile.findMany({
+        orderBy: [{ symbol: 'asc' }],
+        select: {
+          dataSource: true,
+          Order: {
+            orderBy: [{ date: 'asc' }],
+            select: { date: true },
+            take: 1
+          },
+          scraperConfiguration: true,
+          symbol: true
+        }
+      })
+    )
+      .filter((symbolProfile) => {
+        const manualDataSourceWithScraperConfiguration =
+          symbolProfile.dataSource === 'MANUAL' &&
+          !isEmpty(symbolProfile.scraperConfiguration);
+
+        return (
+          symbolProfile.dataSource !== 'MANUAL' ||
+          manualDataSourceWithScraperConfiguration
+        );
       })
-    );
+      .map((symbolProfile) => {
+        return {
+          ...symbolProfile,
+          date: symbolProfile.Order?.[0]?.date ?? startDate
+        };
+      });
 
-    return jobs.some((job) => {
+    return [...currencyPairsToGather, ...symbolProfilesToGather];
-      return (
-        job.name === name && JSON.stringify(job.data) === JSON.stringify(data)
-      );
-    });
   }
 }

apps/client/src/styles.scss

@@ -464,7 +464,7 @@ ngx-skeleton-loader {
 }
 
 .with-info-message {
-  height: calc(100vh - 5rem - 3.5rem) !important;
+  height: calc(100vh - 5rem - 3.5rem + 0.5rem) !important;
 }
 
 .with-placeholder-as-option {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ghostfolio",
-  "version": "1.252.2",
+  "version": "1.254.0",
   "homepage": "https://ghostfol.io",
   "license": "AGPL-3.0",
   "scripts": {