From 4e7d93db13c6af29bd222c1f93876544d69fdc65 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 13 Apr 2024 09:28:14 +0200 Subject: [PATCH 1/3] Feature/adapt priorities of data gathering jobs (#3262) * Adapt priorities of data gathering jobs * Update changelog --- CHANGELOG.md | 2 + apps/api/src/app/admin/admin.controller.ts | 16 +++---- apps/api/src/app/admin/queue/queue.service.ts | 1 + apps/api/src/app/import/import.service.ts | 13 ++++-- apps/api/src/app/order/order.controller.ts | 22 ++++++---- apps/api/src/app/order/order.service.ts | 22 ++++++---- apps/api/src/services/cron.service.ts | 4 +- .../data-gathering/data-gathering.service.ts | 40 +++++++++++++----- .../admin-jobs/admin-jobs.component.ts | 13 +++++- .../app/components/admin-jobs/admin-jobs.html | 42 ++++++++++++++++--- libs/common/src/lib/config.ts | 7 ++-- .../lib/interfaces/admin-jobs.interface.ts | 1 + 12 files changed, 136 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68a85fd95..73999efb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added support to immediately execute a queue job from the admin control panel +- Added a priority column to the queue jobs view in the admin control panel ### Changed +- Adapted the priorities of queue jobs - Upgraded `angular` from version `17.2.4` to `17.3.3` - Upgraded `Nx` from version `18.1.2` to `18.2.3` - Upgraded `prisma` from version `5.11.0` to `5.12.1` diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index dab8fb8b2..298a471c3 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -7,13 +7,12 @@ import { ManualService } from '@ghostfolio/api/services/data-provider/manual/man import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { PropertyDto } from '@ghostfolio/api/services/property/property.dto'; import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS_OPTIONS } from '@ghostfolio/common/config'; -import { - getAssetProfileIdentifier, - resetHours -} from '@ghostfolio/common/helper'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { AdminData, AdminMarketData, @@ -94,7 +93,8 @@ export class AdminController { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: getAssetProfileIdentifier({ dataSource, symbol }) + jobId: getAssetProfileIdentifier({ dataSource, symbol }), + priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM } }; }) @@ -119,7 +119,8 @@ export class AdminController { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: getAssetProfileIdentifier({ dataSource, symbol }) + jobId: getAssetProfileIdentifier({ dataSource, symbol }), + priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM } }; }) @@ -141,7 +142,8 @@ export class AdminController { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: getAssetProfileIdentifier({ dataSource, symbol }) + jobId: getAssetProfileIdentifier({ dataSource, symbol }), + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH } }); } diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.ts index e4f29bc60..abae3cad1 100644 --- a/apps/api/src/app/admin/queue/queue.service.ts +++ b/apps/api/src/app/admin/queue/queue.service.ts @@ -58,6 +58,7 @@ export class QueueService { finishedOn: job.finishedOn, id: job.id, name: job.name, + opts: job.opts, stacktrace: job.stacktrace, state: await job.getState(), timestamp: job.timestamp diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index cbdff87c0..26df9d069 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -13,6 +13,10 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/da import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; +import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + DATA_GATHERING_QUEUE_PRIORITY_MEDIUM +} from '@ghostfolio/common/config'; import { DATE_FORMAT, getAssetProfileIdentifier, @@ -448,15 +452,16 @@ export class ImportService { }); }); - this.dataGatheringService.gatherSymbols( - uniqueActivities.map(({ date, SymbolProfile }) => { + this.dataGatheringService.gatherSymbols({ + dataGatheringItems: uniqueActivities.map(({ date, SymbolProfile }) => { return { date, dataSource: SymbolProfile.dataSource, symbol: SymbolProfile.symbol }; - }) - ); + }), + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); } return activities; diff --git a/apps/api/src/app/order/order.controller.ts b/apps/api/src/app/order/order.controller.ts index c7fec0dac..3dadedcaf 100644 --- a/apps/api/src/app/order/order.controller.ts +++ b/apps/api/src/app/order/order.controller.ts @@ -7,7 +7,10 @@ import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interc import { ApiService } from '@ghostfolio/api/services/api/api.service'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service'; -import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config'; +import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + HEADER_KEY_IMPERSONATION +} from '@ghostfolio/common/config'; import { hasPermission, permissions } from '@ghostfolio/common/permissions'; import type { DateRange, RequestWithUser } from '@ghostfolio/common/types'; @@ -160,13 +163,16 @@ export class OrderController { if (data.dataSource && !order.isDraft) { // Gather symbol data in the background, if data source is set // (not MANUAL) and not draft - this.dataGatheringService.gatherSymbols([ - { - dataSource: data.dataSource, - date: order.date, - symbol: data.symbol - } - ]); + this.dataGatheringService.gatherSymbols({ + dataGatheringItems: [ + { + dataSource: data.dataSource, + date: order.date, + symbol: data.symbol + } + ], + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); } return order; diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 20b2d5f15..35bfa1bcf 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -4,6 +4,7 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate- import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS_OPTIONS } from '@ghostfolio/common/config'; @@ -101,7 +102,8 @@ export class OrderService { jobId: getAssetProfileIdentifier({ dataSource: data.SymbolProfile.connectOrCreate.create.dataSource, symbol: data.SymbolProfile.connectOrCreate.create.symbol - }) + }), + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH } }); } @@ -427,13 +429,17 @@ export class OrderService { if (!isDraft) { // Gather symbol data of order in the background, if not draft - this.dataGatheringService.gatherSymbols([ - { - dataSource: data.SymbolProfile.connect.dataSource_symbol.dataSource, - date: data.date, - symbol: data.SymbolProfile.connect.dataSource_symbol.symbol - } - ]); + this.dataGatheringService.gatherSymbols({ + dataGatheringItems: [ + { + dataSource: + data.SymbolProfile.connect.dataSource_symbol.dataSource, + date: data.date, + symbol: data.SymbolProfile.connect.dataSource_symbol.symbol + } + ], + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); } } diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index d74ad6a94..fc5d613a2 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -1,4 +1,5 @@ import { + DATA_GATHERING_QUEUE_PRIORITY_LOW, GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS_OPTIONS, PROPERTY_IS_DATA_GATHERING_ENABLED @@ -56,7 +57,8 @@ export class CronService { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: getAssetProfileIdentifier({ dataSource, symbol }) + jobId: getAssetProfileIdentifier({ dataSource, symbol }), + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW } }; }) diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.ts index 6dccd645e..b2b0c371c 100644 --- a/apps/api/src/services/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering/data-gathering.service.ts @@ -8,6 +8,8 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.serv import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DATA_GATHERING_QUEUE, + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + DATA_GATHERING_QUEUE_PRIORITY_LOW, GATHER_HISTORICAL_MARKET_DATA_PROCESS, GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, PROPERTY_BENCHMARKS @@ -61,24 +63,35 @@ export class DataGatheringService { public async gather7Days() { const dataGatheringItems = await this.getSymbols7D(); - await this.gatherSymbols(dataGatheringItems); + await this.gatherSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW + }); } public async gatherMax() { const dataGatheringItems = await this.getSymbolsMax(); - await this.gatherSymbols(dataGatheringItems); + await this.gatherSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW + }); } public async gatherSymbol({ dataSource, symbol }: UniqueAsset) { await this.marketDataService.deleteMany({ dataSource, symbol }); - const symbols = (await this.getSymbolsMax()).filter((dataGatheringItem) => { - return ( - dataGatheringItem.dataSource === dataSource && - dataGatheringItem.symbol === symbol - ); + const dataGatheringItems = (await this.getSymbolsMax()).filter( + (dataGatheringItem) => { + return ( + dataGatheringItem.dataSource === dataSource && + dataGatheringItem.symbol === symbol + ); + } + ); + await this.gatherSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH }); - await this.gatherSymbols(symbols); } public async gatherSymbolForDate({ @@ -230,9 +243,15 @@ export class DataGatheringService { ); } - public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) { + public async gatherSymbols({ + dataGatheringItems, + priority + }: { + dataGatheringItems: IDataGatheringItem[]; + priority: number; + }) { await this.addJobsToQueue( - aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => { + dataGatheringItems.map(({ dataSource, date, symbol }) => { return { data: { dataSource, @@ -242,6 +261,7 @@ export class DataGatheringService { name: GATHER_HISTORICAL_MARKET_DATA_PROCESS, opts: { ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, + priority, jobId: `${getAssetProfileIdentifier({ dataSource, symbol diff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts b/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts index 99d67dea2..23730f3aa 100644 --- a/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts +++ b/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts @@ -1,6 +1,11 @@ import { AdminService } from '@ghostfolio/client/services/admin.service'; import { UserService } from '@ghostfolio/client/services/user/user.service'; -import { QUEUE_JOB_STATUS_LIST } from '@ghostfolio/common/config'; +import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + DATA_GATHERING_QUEUE_PRIORITY_LOW, + DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, + QUEUE_JOB_STATUS_LIST +} from '@ghostfolio/common/config'; import { getDateWithTimeFormatString } from '@ghostfolio/common/helper'; import { AdminJobs, User } from '@ghostfolio/common/interfaces'; @@ -24,6 +29,11 @@ import { takeUntil } from 'rxjs/operators'; templateUrl: './admin-jobs.html' }) export class AdminJobsComponent implements OnDestroy, OnInit { + public DATA_GATHERING_QUEUE_PRIORITY_LOW = DATA_GATHERING_QUEUE_PRIORITY_LOW; + public DATA_GATHERING_QUEUE_PRIORITY_HIGH = + DATA_GATHERING_QUEUE_PRIORITY_HIGH; + public DATA_GATHERING_QUEUE_PRIORITY_MEDIUM = + DATA_GATHERING_QUEUE_PRIORITY_MEDIUM; public defaultDateTimeFormat: string; public filterForm: FormGroup; public dataSource: MatTableDataSource = @@ -33,6 +43,7 @@ export class AdminJobsComponent implements OnDestroy, OnInit { 'type', 'symbol', 'dataSource', + 'priority', 'attempts', 'created', 'finished', diff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.html b/apps/client/src/app/components/admin-jobs/admin-jobs.html index 31dad0ca6..5da929fe1 100644 --- a/apps/client/src/app/components/admin-jobs/admin-jobs.html +++ b/apps/client/src/app/components/admin-jobs/admin-jobs.html @@ -58,6 +58,25 @@ + + + Priority + + + @if (element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_LOW) { + + } @else if ( + element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_MEDIUM + ) { + + } @else if ( + element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_HIGH + ) { + + } + + + Attempts @@ -90,24 +109,37 @@ Status - + - - + + diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 293f77488..5e1366ce2 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -32,8 +32,11 @@ export const warnColorRgb = { }; export const DATA_GATHERING_QUEUE = 'DATA_GATHERING_QUEUE'; -export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER; export const DATA_GATHERING_QUEUE_PRIORITY_HIGH = 1; +export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER; +export const DATA_GATHERING_QUEUE_PRIORITY_MEDIUM = Math.round( + DATA_GATHERING_QUEUE_PRIORITY_LOW / 2 +); export const DEFAULT_CURRENCY = 'USD'; export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy'; @@ -69,7 +72,6 @@ export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = { delay: ms('1 minute'), type: 'exponential' }, - priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH, removeOnComplete: true }; export const GATHER_HISTORICAL_MARKET_DATA_PROCESS = @@ -80,7 +82,6 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = { delay: ms('1 minute'), type: 'exponential' }, - priority: DATA_GATHERING_QUEUE_PRIORITY_LOW, removeOnComplete: true }; diff --git a/libs/common/src/lib/interfaces/admin-jobs.interface.ts b/libs/common/src/lib/interfaces/admin-jobs.interface.ts index 25e937626..b4c91ebc0 100644 --- a/libs/common/src/lib/interfaces/admin-jobs.interface.ts +++ b/libs/common/src/lib/interfaces/admin-jobs.interface.ts @@ -8,6 +8,7 @@ export interface AdminJobs { | 'finishedOn' | 'id' | 'name' + | 'opts' | 'stacktrace' | 'timestamp' > & { From 7d308917ddba1b6f7d1fa40b9005d73b022ec0fb Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 13 Apr 2024 09:28:38 +0200 Subject: [PATCH 2/3] Feature/upgrade yahoo finance2 to version 2.11.1 (#3254) * Upgrade yahoo-finance2 to version 2.11.1 * Update changelog --- CHANGELOG.md | 1 + package.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73999efb7..e48f26c0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Upgraded `angular` from version `17.2.4` to `17.3.3` - Upgraded `Nx` from version `18.1.2` to `18.2.3` - Upgraded `prisma` from version `5.11.0` to `5.12.1` +- Upgraded `yahoo-finance2` from version `2.11.0` to `2.11.1` ### Fixed diff --git a/package.json b/package.json index 1f2e7520a..252fb045f 100644 --- a/package.json +++ b/package.json @@ -132,7 +132,7 @@ "svgmap": "2.6.0", "twitter-api-v2": "1.14.2", "uuid": "9.0.1", - "yahoo-finance2": "2.11.0", + "yahoo-finance2": "2.11.1", "zone.js": "0.14.4" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 9c77b9917..8cbfbccd0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19620,10 +19620,10 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== -yahoo-finance2@2.11.0: - version "2.11.0" - resolved "https://registry.yarnpkg.com/yahoo-finance2/-/yahoo-finance2-2.11.0.tgz#293ade36d2e969218ad38fc308abc11ca02c1fab" - integrity sha512-lQWjnf9cnfAEmpL9ymwabxjQxu1xT5jg03NYOYcpn0ObOx9oGk9hes6JDaCeIq/xT+xgMdPMupsgYxnAxfKyhw== +yahoo-finance2@2.11.1: + version "2.11.1" + resolved "https://registry.yarnpkg.com/yahoo-finance2/-/yahoo-finance2-2.11.1.tgz#97758d4784ef0b4efe4b370a72063929cc4c6342" + integrity sha512-YglgpjIDithq1PG8Je/gy8nzJFqkH214x2ZGfr6Y+HV4ymTDFLluq2W9Hsvvyydv1zTv9/Ykedf0J4YIpmO2Zg== dependencies: "@types/tough-cookie" "^4.0.2" ajv "8.10.0" From b31bbbe2d14eee09b78699f80d8d37c1a03613de Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 13 Apr 2024 09:31:07 +0200 Subject: [PATCH 3/3] Release 2.72.0 (#3270) --- CHANGELOG.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e48f26c0a..344ad73ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## 2.72.0 - 2024-04-13 ### Added diff --git a/package.json b/package.json index 252fb045f..35ee3b1a4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ghostfolio", - "version": "2.71.0", + "version": "2.72.0", "homepage": "https://ghostfol.io", "license": "AGPL-3.0", "repository": "https://github.com/ghostfolio/ghostfolio",