From 7d779d84617af567370f999a0129d7dea385a3e0 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 20 Jun 2026 18:37:44 +0200 Subject: [PATCH] Feature/add data gathering frequency to symbol profile (#7083) * Add data gathering frequency to symbol profile and gather hourly * Update changelog --- CHANGELOG.md | 2 + apps/api/src/app/admin/admin.service.ts | 2 + .../market-data/market-data.controller.ts | 6 +- apps/api/src/app/import/import.service.ts | 2 + apps/api/src/app/symbol/symbol.service.ts | 25 ++++-- apps/api/src/services/cron/cron.service.ts | 1 + .../market-data/market-data.service.ts | 13 +++ .../data-gathering/data-gathering.service.ts | 83 +++++++++++++++++-- .../symbol-profile/symbol-profile.service.ts | 2 + .../asset-profile-dialog.component.ts | 21 +++++ .../asset-profile-dialog.html | 17 +++- .../src/lib/dtos/update-asset-profile.dto.ts | 12 ++- .../enhanced-symbol-profile.interface.ts | 8 +- .../responses/export-response.interface.ts | 1 + libs/ui/src/lib/services/admin.service.ts | 2 + .../migration.sql | 8 ++ prisma/schema.prisma | 7 ++ 17 files changed, 196 insertions(+), 16 deletions(-) create mode 100644 prisma/migrations/20260620163851_added_data_gathering_frequency_to_symbol_profile/migration.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index d4afa84a9..038612a02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added an icon to indicate external links in the page tabs component - Added the Korean (`ko`) language to the footer +- Added a data gathering frequency (`DAILY` or `HOURLY`) to the asset profile to control the market data gathering interval ### Changed +- Changed the _Fear & Greed Index_ (market mood) in the markets overview to use the stored market data instead of a live quote - Moved the endpoint to get the asset profiles from `GET api/v1/admin/market-data` to `GET api/v1/asset-profiles` - Added the selected asset profile count to the delete menu item of the historical market data table in the admin control panel - Added the selected asset profile count to the deletion confirmation dialog of the historical market data table in the admin control panel diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts index f08fb6a37..9a5429655 100644 --- a/apps/api/src/app/admin/admin.service.ts +++ b/apps/api/src/app/admin/admin.service.ts @@ -287,6 +287,7 @@ export class AdminService { comment, countries, currency, + dataGatheringFrequency, dataSource: newDataSource, holdings, isActive, @@ -370,6 +371,7 @@ export class AdminService { const updatedSymbolProfile: Prisma.SymbolProfileUpdateInput = { comment, currency, + dataGatheringFrequency, dataSource, isActive, scraperConfiguration, diff --git a/apps/api/src/app/endpoints/market-data/market-data.controller.ts b/apps/api/src/app/endpoints/market-data/market-data.controller.ts index f6857283b..f60eea904 100644 --- a/apps/api/src/app/endpoints/market-data/market-data.controller.ts +++ b/apps/api/src/app/endpoints/market-data/market-data.controller.ts @@ -64,14 +64,16 @@ export class MarketDataController { dataGatheringItem: { dataSource: ghostfolioFearAndGreedIndexDataSourceCryptocurrencies, symbol: ghostfolioFearAndGreedIndexSymbolCryptocurrencies - } + }, + useIntradayData: true }), this.symbolService.get({ includeHistoricalData, dataGatheringItem: { dataSource: ghostfolioFearAndGreedIndexDataSourceStocks, symbol: ghostfolioFearAndGreedIndexSymbolStocks - } + }, + useIntradayData: true }) ]); diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index b82f763a0..2ecc4d3a5 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -536,6 +536,8 @@ export class ImportService { url, comment: assetProfile.comment, currency: assetProfile.currency, + dataGatheringFrequency: + assetProfile.dataGatheringFrequency ?? 'DAILY', userId: dataSource === 'MANUAL' ? user.id : undefined }, symbolProfileId: undefined, diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index fdbc7f84c..f2bf4beb1 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -24,15 +24,30 @@ export class SymbolService { public async get({ dataGatheringItem, - includeHistoricalData + includeHistoricalData, + useIntradayData = false }: { dataGatheringItem: DataGatheringItem; includeHistoricalData?: number; + useIntradayData?: boolean; }): Promise { - const quotes = await this.dataProviderService.getQuotes({ - items: [dataGatheringItem] - }); - const { currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {}; + let currency: string; + let marketPrice: number; + + if (useIntradayData) { + const latestMarketData = await this.marketDataService.getLatest({ + dataSource: dataGatheringItem.dataSource, + symbol: dataGatheringItem.symbol + }); + + marketPrice = latestMarketData?.marketPrice; + } else { + const quotes = await this.dataProviderService.getQuotes({ + items: [dataGatheringItem] + }); + + ({ currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {}); + } if (dataGatheringItem.dataSource && marketPrice >= 0) { let historicalData: HistoricalDataItem[] = []; diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts index e680f0063..7299cbbf4 100644 --- a/apps/api/src/services/cron/cron.service.ts +++ b/apps/api/src/services/cron/cron.service.ts @@ -42,6 +42,7 @@ export class CronService { public async runEveryHourAtRandomMinute() { if (await this.isDataGatheringEnabled()) { await this.dataGatheringService.gather7Days(); + await this.dataGatheringService.gatherHourlySymbols(); } } diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 87b08e1bd..27c741055 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -40,6 +40,19 @@ export class MarketDataService { }); } + public async getLatest({ + dataSource, + symbol + }: AssetProfileIdentifier): Promise { + return this.prismaService.marketData.findFirst({ + orderBy: [{ date: 'desc' }], + where: { + dataSource, + symbol + } + }); + } + public async getMax({ dataSource, symbol }: AssetProfileIdentifier) { return this.prismaService.marketData.findFirst({ select: { diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index b5b701fe4..dfd371a13 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -2,6 +2,7 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; +import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; @@ -17,6 +18,7 @@ import { import { DATE_FORMAT, getAssetProfileIdentifier, + getStartOfUtcDate, resetHours } from '@ghostfolio/common/helper'; import { @@ -26,7 +28,7 @@ import { import { InjectQueue } from '@nestjs/bull'; import { Inject, Injectable, Logger } from '@nestjs/common'; -import { DataSource } from '@prisma/client'; +import { DataSource, Prisma } from '@prisma/client'; import { JobOptions, Queue } from 'bull'; import { format, min, subDays, subMilliseconds, subYears } from 'date-fns'; import { isEmpty } from 'lodash'; @@ -43,6 +45,7 @@ export class DataGatheringService { private readonly dataGatheringQueue: Queue, private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, + private readonly marketDataService: MarketDataService, private readonly prismaService: PrismaService, private readonly propertyService: PropertyService, private readonly symbolProfileService: SymbolProfileService @@ -279,6 +282,46 @@ export class DataGatheringService { } } + public async gatherHourlySymbols() { + const assetProfileIdentifiers = + await this.getHourlyAssetProfileIdentifiers(); + + if (assetProfileIdentifiers.length <= 0) { + return; + } + + const date = getStartOfUtcDate(new Date()); + + try { + const quotes = await this.dataProviderService.getQuotes({ + items: assetProfileIdentifiers, + useCache: false + }); + + const data: Prisma.MarketDataUpdateInput[] = []; + + for (const { dataSource, symbol } of assetProfileIdentifiers) { + const quote = quotes[symbol]; + + if (quote?.dataSource !== dataSource || !quote.marketPrice) { + continue; + } + + data.push({ + dataSource, + date, + symbol, + marketPrice: quote.marketPrice, + state: 'INTRADAY' + }); + } + + await this.marketDataService.updateMany({ data }); + } catch (error) { + this.logger.error('Could not gather hourly market data', error); + } + } + public async gatherSymbols({ dataGatheringItems, force = false, @@ -389,6 +432,36 @@ export class DataGatheringService { return min([aStartDate, subYears(new Date(), 10)]); } + private async getHourlyAssetProfileIdentifiers(): Promise< + AssetProfileIdentifier[] + > { + const symbolProfiles = await this.prismaService.symbolProfile.findMany({ + orderBy: [{ symbol: 'asc' }, { dataSource: 'asc' }], + select: { + dataSource: true, + scraperConfiguration: true, + symbol: true + }, + where: { + dataGatheringFrequency: 'HOURLY', + isActive: true + } + }); + + return symbolProfiles + .filter(({ dataSource, scraperConfiguration }) => { + const manualDataSourceWithScraperConfiguration = + dataSource === 'MANUAL' && !isEmpty(scraperConfiguration); + + return ( + dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration + ); + }) + .map(({ dataSource, symbol }) => { + return { dataSource, symbol }; + }); + } + private async getSymbols7D({ withUserSubscription = false }: { @@ -469,14 +542,12 @@ export class DataGatheringService { } }) ) - .filter((symbolProfile) => { + .filter(({ dataSource, scraperConfiguration }) => { const manualDataSourceWithScraperConfiguration = - symbolProfile.dataSource === 'MANUAL' && - !isEmpty(symbolProfile.scraperConfiguration); + dataSource === 'MANUAL' && !isEmpty(scraperConfiguration); return ( - symbolProfile.dataSource !== 'MANUAL' || - manualDataSourceWithScraperConfiguration + dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration ); }) .map((symbolProfile) => { diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.ts index 2d5116274..5d0968c5c 100644 --- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts +++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts @@ -178,6 +178,7 @@ export class SymbolProfileService { comment, countries, currency, + dataGatheringFrequency, holdings, isActive, name, @@ -195,6 +196,7 @@ export class SymbolProfileService { comment, countries, currency, + dataGatheringFrequency, holdings, isActive, name, diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts index 829129b38..5ceb5808a 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts @@ -74,6 +74,7 @@ import { IonIcon } from '@ionic/angular/standalone'; import { AssetClass, AssetSubClass, + DataGatheringFrequency, MarketData, Prisma, SymbolProfile @@ -155,6 +156,7 @@ export class GfAssetProfileDialogComponent implements OnInit { comment: '', countries: ['', jsonValidator()], currency: '', + dataGatheringFrequency: new FormControl('DAILY'), historicalData: this.formBuilder.group({ csvString: '' }), @@ -199,6 +201,20 @@ export class GfAssetProfileDialogComponent implements OnInit { protected currencies: string[] = []; + protected readonly dataGatheringFrequencyValues: { + value: DataGatheringFrequency; + viewValue: string; + }[] = [ + { + value: 'DAILY', + viewValue: $localize`Daily` + }, + { + value: 'HOURLY', + viewValue: $localize`Hourly` + } + ]; + protected readonly dateRangeOptions = [ { label: $localize`Current week` + ' (' + $localize`WTD` + ')', @@ -401,6 +417,8 @@ export class GfAssetProfileDialogComponent implements OnInit { }) ?? [] ), currency: this.assetProfile?.currency ?? null, + dataGatheringFrequency: + this.assetProfile?.dataGatheringFrequency ?? 'DAILY', historicalData: { csvString: GfAssetProfileDialogComponent.HISTORICAL_DATA_TEMPLATE }, @@ -583,6 +601,9 @@ export class GfAssetProfileDialogComponent implements OnInit { this.assetProfileForm.controls.assetSubClass.value ?? undefined, comment: this.assetProfileForm.controls.comment.value || undefined, currency: this.assetProfileForm.controls.currency.value ?? undefined, + dataGatheringFrequency: + this.assetProfileForm.controls.dataGatheringFrequency.value ?? + undefined, isActive: isBoolean(this.assetProfileForm.controls.isActive.value) ? this.assetProfileForm.controls.isActive.value : undefined, diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html index 453b8cb6a..f00c279a0 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html @@ -113,7 +113,7 @@
Overview
-
+
@if (isEditAssetProfileIdentifierMode) {
@@ -444,6 +444,21 @@ >
+
+ + Data Gathering Frequency + + @for ( + dataGatheringFrequencyValue of dataGatheringFrequencyValues; + track dataGatheringFrequencyValue.value + ) { + {{ + dataGatheringFrequencyValue.viewValue + }} + } + + +
diff --git a/libs/common/src/lib/dtos/update-asset-profile.dto.ts b/libs/common/src/lib/dtos/update-asset-profile.dto.ts index 1c8af3e72..ba3868482 100644 --- a/libs/common/src/lib/dtos/update-asset-profile.dto.ts +++ b/libs/common/src/lib/dtos/update-asset-profile.dto.ts @@ -1,6 +1,12 @@ import { IsCurrencyCode } from '@ghostfolio/common/validators/is-currency-code'; -import { AssetClass, AssetSubClass, DataSource, Prisma } from '@prisma/client'; +import { + AssetClass, + AssetSubClass, + DataGatheringFrequency, + DataSource, + Prisma +} from '@prisma/client'; import { IsArray, IsBoolean, @@ -32,6 +38,10 @@ export class UpdateAssetProfileDto { @IsOptional() currency?: string; + @IsEnum(DataGatheringFrequency) + @IsOptional() + dataGatheringFrequency?: DataGatheringFrequency; + @IsEnum(DataSource) @IsOptional() dataSource?: DataSource; diff --git a/libs/common/src/lib/interfaces/enhanced-symbol-profile.interface.ts b/libs/common/src/lib/interfaces/enhanced-symbol-profile.interface.ts index 8426916c9..69fdf9f18 100644 --- a/libs/common/src/lib/interfaces/enhanced-symbol-profile.interface.ts +++ b/libs/common/src/lib/interfaces/enhanced-symbol-profile.interface.ts @@ -1,4 +1,9 @@ -import { AssetClass, AssetSubClass, DataSource } from '@prisma/client'; +import { + AssetClass, + AssetSubClass, + DataGatheringFrequency, + DataSource +} from '@prisma/client'; import { Country } from './country.interface'; import { DataProviderInfo } from './data-provider-info.interface'; @@ -15,6 +20,7 @@ export interface EnhancedSymbolProfile { createdAt: Date; currency?: string; cusip?: string; + dataGatheringFrequency?: DataGatheringFrequency; dataProviderInfo?: DataProviderInfo; dataSource: DataSource; dateOfFirstActivity?: Date; diff --git a/libs/common/src/lib/interfaces/responses/export-response.interface.ts b/libs/common/src/lib/interfaces/responses/export-response.interface.ts index fa592faf2..0e8ba1574 100644 --- a/libs/common/src/lib/interfaces/responses/export-response.interface.ts +++ b/libs/common/src/lib/interfaces/responses/export-response.interface.ts @@ -28,6 +28,7 @@ export interface ExportResponse { assetProfiles: (Omit< SymbolProfile, | 'createdAt' + | 'dataGatheringFrequency' | 'id' | 'scraperConfiguration' | 'symbolMapping' diff --git a/libs/ui/src/lib/services/admin.service.ts b/libs/ui/src/lib/services/admin.service.ts index 0eea768d6..90e477f28 100644 --- a/libs/ui/src/lib/services/admin.service.ts +++ b/libs/ui/src/lib/services/admin.service.ts @@ -189,6 +189,7 @@ export class AdminService { comment, countries, currency, + dataGatheringFrequency, dataSource: newDataSource, isActive, name, @@ -207,6 +208,7 @@ export class AdminService { comment, countries, currency, + dataGatheringFrequency, dataSource: newDataSource, isActive, name, diff --git a/prisma/migrations/20260620163851_added_data_gathering_frequency_to_symbol_profile/migration.sql b/prisma/migrations/20260620163851_added_data_gathering_frequency_to_symbol_profile/migration.sql new file mode 100644 index 000000000..33ea3f732 --- /dev/null +++ b/prisma/migrations/20260620163851_added_data_gathering_frequency_to_symbol_profile/migration.sql @@ -0,0 +1,8 @@ +-- CreateEnum +CREATE TYPE "DataGatheringFrequency" AS ENUM ('DAILY', 'HOURLY'); + +-- AlterTable +ALTER TABLE "SymbolProfile" ADD COLUMN "dataGatheringFrequency" "DataGatheringFrequency" NOT NULL DEFAULT 'DAILY'; + +-- CreateIndex +CREATE INDEX "SymbolProfile_dataGatheringFrequency_idx" ON "SymbolProfile"("dataGatheringFrequency"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 024ab7aa0..2c39667ee 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -191,6 +191,7 @@ model SymbolProfile { createdAt DateTime @default(now()) currency String cusip String? + dataGatheringFrequency DataGatheringFrequency @default(DAILY) dataSource DataSource figi String? figiComposite String? @@ -215,6 +216,7 @@ model SymbolProfile { @@index([assetClass]) @@index([currency]) @@index([cusip]) + @@index([dataGatheringFrequency]) @@index([dataSource]) @@index([isActive]) @@index([isin]) @@ -316,6 +318,11 @@ enum AssetSubClass { STOCK } +enum DataGatheringFrequency { + DAILY + HOURLY +} + enum DataSource { ALPHA_VANTAGE COINGECKO