diff --git a/CHANGELOG.md b/CHANGELOG.md index e179023d0..1e572e30c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Skipped creating queue jobs for asset profiles with `MANUAL` data source not having a scraper configuration +- Reduced the execution interval of the data gathering to every hour - Upgraded `prisma` from version `4.11.0` to `4.12.0` ### Fixed +- Improved the style of the system message + +## 1.254.0 - 2023-04-14 + +### Changed + +- Improved the queue jobs implementation by adding in bulk +- Improved the queue jobs implementation by introducing unique job ids +- Reverted the execution interval of the data gathering from every 12 hours to every 4 hours + +## 1.253.0 - 2023-04-14 + +### Changed + +- Reduced the execution interval of the data gathering to every 12 hours + +### Fixed + - Fixed the background color of dialogs in dark mode ## 1.252.2 - 2023-04-11 diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 7f3f17791..c32ecf03b 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -100,16 +100,21 @@ export class AdminController { const uniqueAssets = await this.dataGatheringService.getUniqueAssets(); - for (const { dataSource, symbol } of uniqueAssets) { - await this.dataGatheringService.addJobToQueue( - GATHER_ASSET_PROFILE_PROCESS, - { - dataSource, - symbol - }, - GATHER_ASSET_PROFILE_PROCESS_OPTIONS - ); - } + await this.dataGatheringService.addJobsToQueue( + uniqueAssets.map(({ dataSource, symbol }) => { + return { + data: { + dataSource, + symbol + }, + name: GATHER_ASSET_PROFILE_PROCESS, + opts: { + ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + jobId: `${dataSource}-${symbol}}` + } + }; + }) + ); this.dataGatheringService.gatherMax(); } @@ -131,16 +136,21 @@ export class AdminController { const uniqueAssets = await this.dataGatheringService.getUniqueAssets(); - for (const { dataSource, symbol } of uniqueAssets) { - await this.dataGatheringService.addJobToQueue( - GATHER_ASSET_PROFILE_PROCESS, - { - dataSource, - symbol - }, - GATHER_ASSET_PROFILE_PROCESS_OPTIONS - ); - } + await this.dataGatheringService.addJobsToQueue( + uniqueAssets.map(({ dataSource, symbol }) => { + return { + data: { + dataSource, + symbol + }, + name: GATHER_ASSET_PROFILE_PROCESS, + opts: { + ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + jobId: `${dataSource}-${symbol}}` + } + }; + }) + ); } @Post('gather/profile-data/:dataSource/:symbol') @@ -161,14 +171,17 @@ export class AdminController { ); } - await this.dataGatheringService.addJobToQueue( - GATHER_ASSET_PROFILE_PROCESS, - { + await this.dataGatheringService.addJobToQueue({ + data: { dataSource, symbol }, - GATHER_ASSET_PROFILE_PROCESS_OPTIONS - ); + name: GATHER_ASSET_PROFILE_PROCESS, + opts: { + ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + jobId: `${dataSource}-${symbol}}` + } + }); } @Post('gather/:dataSource/:symbol') diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 2d0cb7376..4080c5c34 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -112,14 +112,17 @@ export class OrderService { }; } - await this.dataGatheringService.addJobToQueue( - GATHER_ASSET_PROFILE_PROCESS, - { + await this.dataGatheringService.addJobToQueue({ + data: { dataSource: data.SymbolProfile.connectOrCreate.create.dataSource, symbol: data.SymbolProfile.connectOrCreate.create.symbol }, - GATHER_ASSET_PROFILE_PROCESS_OPTIONS - ); + name: GATHER_ASSET_PROFILE_PROCESS, + opts: { + ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}}` + } + }); const isDraft = isAfter(data.date as Date, endOfToday()); diff --git a/apps/api/src/app/portfolio/portfolio-calculator.ts b/apps/api/src/app/portfolio/portfolio-calculator.ts index 57835298c..952b81677 100644 --- a/apps/api/src/app/portfolio/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/portfolio-calculator.ts @@ -722,7 +722,7 @@ export class PortfolioCalculator { ); } else if (!currentPosition.quantity.eq(0)) { Logger.warn( - `Missing initial value for symbol ${currentPosition.symbol} at ${currentPosition.firstBuyDate}`, + `Missing historical market data for symbol ${currentPosition.symbol}`, 'PortfolioCalculator' ); hasErrors = true; diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index 8b036c35e..bf186b32f 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -19,8 +19,8 @@ export class CronService { private readonly twitterBotService: TwitterBotService ) {} - @Cron(CronExpression.EVERY_4_HOURS) - public async runEveryFourHours() { + @Cron(CronExpression.EVERY_HOUR) + public async runEveryHour() { await this.dataGatheringService.gather7Days(); } @@ -38,15 +38,20 @@ export class CronService { public async runEverySundayAtTwelvePm() { const uniqueAssets = await this.dataGatheringService.getUniqueAssets(); - for (const { dataSource, symbol } of uniqueAssets) { - await this.dataGatheringService.addJobToQueue( - GATHER_ASSET_PROFILE_PROCESS, - { - dataSource, - symbol - }, - GATHER_ASSET_PROFILE_PROCESS_OPTIONS - ); - } + await this.dataGatheringService.addJobsToQueue( + uniqueAssets.map(({ dataSource, symbol }) => { + return { + data: { + dataSource, + symbol + }, + name: GATHER_ASSET_PROFILE_PROCESS, + opts: { + ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, + jobId: `${dataSource}-${symbol}}` + } + }; + }) + ); } } diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 62209846c..5de70c925 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -2,8 +2,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.se import { DATA_GATHERING_QUEUE, GATHER_HISTORICAL_MARKET_DATA_PROCESS, - GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, - QUEUE_JOB_STATUS_LIST + GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS } from '@ghostfolio/common/config'; import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; @@ -12,6 +11,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common'; import { DataSource } from '@prisma/client'; import { JobOptions, Queue } from 'bull'; import { format, min, subDays, subYears } from 'date-fns'; +import { isEmpty } from 'lodash'; import { DataProviderService } from './data-provider/data-provider.service'; import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface'; @@ -34,17 +34,22 @@ export class DataGatheringService { private readonly symbolProfileService: SymbolProfileService ) {} - public async addJobToQueue(name: string, data: any, options?: JobOptions) { - const hasJob = await this.hasJob(name, data); + public async addJobToQueue({ + data, + name, + opts + }: { + data: any; + name: string; + opts?: JobOptions; + }) { + return this.dataGatheringQueue.add(name, data, opts); + } - if (hasJob) { - Logger.log( - `Job ${name} with data ${JSON.stringify(data)} already exists.`, - 'DataGatheringService' - ); - } else { - return this.dataGatheringQueue.add(name, data, options); - } + public async addJobsToQueue( + jobs: { data: any; name: string; opts?: JobOptions }[] + ) { + return this.dataGatheringQueue.addBulk(jobs); } public async gather7Days() { @@ -209,59 +214,22 @@ export class DataGatheringService { } public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) { - for (const { dataSource, date, symbol } of aSymbolsWithStartDate) { - await this.addJobToQueue( - GATHER_HISTORICAL_MARKET_DATA_PROCESS, - { - dataSource, - date, - symbol - }, - GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS - ); - } - } - - public async getSymbolsMax(): Promise { - const startDate = - ( - await this.prismaService.order.findFirst({ - orderBy: [{ date: 'asc' }] - }) - )?.date ?? new Date(); - - const currencyPairsToGather = this.exchangeRateDataService - .getCurrencyPairs() - .map(({ dataSource, symbol }) => { + await this.addJobsToQueue( + aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => { return { - dataSource, - symbol, - date: min([startDate, subYears(new Date(), 10)]) - }; - }); - - const symbolProfilesToGather = ( - await this.prismaService.symbolProfile.findMany({ - orderBy: [{ symbol: 'asc' }], - select: { - dataSource: true, - Order: { - orderBy: [{ date: 'asc' }], - select: { date: true }, - take: 1 + data: { + dataSource, + date, + symbol }, - scraperConfiguration: true, - symbol: true - } + name: GATHER_HISTORICAL_MARKET_DATA_PROCESS, + opts: { + ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, + jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}` + } + }; }) - ).map((symbolProfile) => { - return { - ...symbolProfile, - date: symbolProfile.Order?.[0]?.date ?? startDate - }; - }); - - return [...currencyPairsToGather, ...symbolProfilesToGather]; + ); } public async getUniqueAssets(): Promise { @@ -298,7 +266,7 @@ export class DataGatheringService { // Only consider symbols with incomplete market data for the last // 7 days - const symbolsNotToGather = ( + const symbolsWithCompleteMarketData = ( await this.prismaService.marketData.groupBy({ _count: true, by: ['symbol'], @@ -316,8 +284,14 @@ export class DataGatheringService { }); const symbolProfilesToGather = symbolProfiles - .filter(({ symbol }) => { - return !symbolsNotToGather.includes(symbol); + .filter(({ dataSource, scraperConfiguration, symbol }) => { + const manualDataSourceWithScraperConfiguration = + dataSource === 'MANUAL' && !isEmpty(scraperConfiguration); + + return ( + !symbolsWithCompleteMarketData.includes(symbol) && + (dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration) + ); }) .map((symbolProfile) => { return { @@ -329,7 +303,7 @@ export class DataGatheringService { const currencyPairsToGather = this.exchangeRateDataService .getCurrencyPairs() .filter(({ symbol }) => { - return !symbolsNotToGather.includes(symbol); + return !symbolsWithCompleteMarketData.includes(symbol); }) .map(({ dataSource, symbol }) => { return { @@ -342,17 +316,56 @@ export class DataGatheringService { return [...currencyPairsToGather, ...symbolProfilesToGather]; } - private async hasJob(name: string, data: any) { - const jobs = await this.dataGatheringQueue.getJobs( - QUEUE_JOB_STATUS_LIST.filter((status) => { - return status !== 'completed'; + private async getSymbolsMax(): Promise { + const startDate = + ( + await this.prismaService.order.findFirst({ + orderBy: [{ date: 'asc' }] + }) + )?.date ?? new Date(); + + const currencyPairsToGather = this.exchangeRateDataService + .getCurrencyPairs() + .map(({ dataSource, symbol }) => { + return { + dataSource, + symbol, + date: min([startDate, subYears(new Date(), 10)]) + }; + }); + + const symbolProfilesToGather = ( + await this.prismaService.symbolProfile.findMany({ + orderBy: [{ symbol: 'asc' }], + select: { + dataSource: true, + Order: { + orderBy: [{ date: 'asc' }], + select: { date: true }, + take: 1 + }, + scraperConfiguration: true, + symbol: true + } + }) + ) + .filter((symbolProfile) => { + const manualDataSourceWithScraperConfiguration = + symbolProfile.dataSource === 'MANUAL' && + !isEmpty(symbolProfile.scraperConfiguration); + + return ( + symbolProfile.dataSource !== 'MANUAL' || + manualDataSourceWithScraperConfiguration + ); }) - ); + .map((symbolProfile) => { + return { + ...symbolProfile, + date: symbolProfile.Order?.[0]?.date ?? startDate + }; + }); - return jobs.some((job) => { - return ( - job.name === name && JSON.stringify(job.data) === JSON.stringify(data) - ); - }); + return [...currencyPairsToGather, ...symbolProfilesToGather]; } } diff --git a/apps/client/src/styles.scss b/apps/client/src/styles.scss index 73d7d4ded..942e75294 100644 --- a/apps/client/src/styles.scss +++ b/apps/client/src/styles.scss @@ -464,7 +464,7 @@ ngx-skeleton-loader { } .with-info-message { - height: calc(100vh - 5rem - 3.5rem) !important; + height: calc(100vh - 5rem - 3.5rem + 0.5rem) !important; } .with-placeholder-as-option { diff --git a/package.json b/package.json index fcaf6c8db..e4cf11477 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ghostfolio", - "version": "1.252.2", + "version": "1.254.0", "homepage": "https://ghostfol.io", "license": "AGPL-3.0", "scripts": {