Feature/add data gathering frequency to symbol profile (#7083)

* Add data gathering frequency to symbol profile and gather hourly

* Update changelog

CHANGELOG.md

@@ -11,9 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added an icon to indicate external links in the page tabs component
 - Added the Korean (`ko`) language to the footer
+- Added a data gathering frequency (`DAILY` or `HOURLY`) to the asset profile to control the market data gathering interval
 
 ### Changed
 
+- Changed the _Fear & Greed Index_ (market mood) in the markets overview to use the stored market data instead of a live quote
 - Moved the endpoint to get the asset profiles from `GET api/v1/admin/market-data` to `GET api/v1/asset-profiles`
 - Added the selected asset profile count to the delete menu item of the historical market data table in the admin control panel
 - Added the selected asset profile count to the deletion confirmation dialog of the historical market data table in the admin control panel

apps/api/src/app/admin/admin.service.ts

@@ -287,6 +287,7 @@ export class AdminService {
       comment,
       countries,
       currency,
+      dataGatheringFrequency,
       dataSource: newDataSource,
       holdings,
       isActive,
@@ -370,6 +371,7 @@ export class AdminService {
       const updatedSymbolProfile: Prisma.SymbolProfileUpdateInput = {
         comment,
         currency,
+        dataGatheringFrequency,
         dataSource,
         isActive,
         scraperConfiguration,

apps/api/src/app/endpoints/market-data/market-data.controller.ts

@@ -64,14 +64,16 @@ export class MarketDataController {
         dataGatheringItem: {
           dataSource: ghostfolioFearAndGreedIndexDataSourceCryptocurrencies,
           symbol: ghostfolioFearAndGreedIndexSymbolCryptocurrencies
-        }
+        },
+        useIntradayData: true
       }),
       this.symbolService.get({
         includeHistoricalData,
         dataGatheringItem: {
           dataSource: ghostfolioFearAndGreedIndexDataSourceStocks,
           symbol: ghostfolioFearAndGreedIndexSymbolStocks
-        }
+        },
+        useIntradayData: true
       })
     ]);
 

apps/api/src/app/import/import.service.ts

@@ -536,6 +536,8 @@ export class ImportService {
             url,
             comment: assetProfile.comment,
             currency: assetProfile.currency,
+            dataGatheringFrequency:
+              assetProfile.dataGatheringFrequency ?? 'DAILY',
             userId: dataSource === 'MANUAL' ? user.id : undefined
           },
           symbolProfileId: undefined,

apps/api/src/app/symbol/symbol.service.ts

@@ -24,15 +24,30 @@ export class SymbolService {
 
   public async get({
     dataGatheringItem,
-    includeHistoricalData
+    includeHistoricalData,
+    useIntradayData = false
   }: {
     dataGatheringItem: DataGatheringItem;
     includeHistoricalData?: number;
+    useIntradayData?: boolean;
   }): Promise<SymbolItem> {
-    const quotes = await this.dataProviderService.getQuotes({
+    let currency: string;
-      items: [dataGatheringItem]
+    let marketPrice: number;
-    });
+
-    const { currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {};
+    if (useIntradayData) {
+      const latestMarketData = await this.marketDataService.getLatest({
+        dataSource: dataGatheringItem.dataSource,
+        symbol: dataGatheringItem.symbol
+      });
+
+      marketPrice = latestMarketData?.marketPrice;
+    } else {
+      const quotes = await this.dataProviderService.getQuotes({
+        items: [dataGatheringItem]
+      });
+
+      ({ currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {});
+    }
 
     if (dataGatheringItem.dataSource && marketPrice >= 0) {
       let historicalData: HistoricalDataItem[] = [];

apps/api/src/services/cron/cron.service.ts

@@ -42,6 +42,7 @@ export class CronService {
   public async runEveryHourAtRandomMinute() {
     if (await this.isDataGatheringEnabled()) {
       await this.dataGatheringService.gather7Days();
+      await this.dataGatheringService.gatherHourlySymbols();
     }
   }
 

apps/api/src/services/market-data/market-data.service.ts

@@ -40,6 +40,19 @@ export class MarketDataService {
     });
   }
 
+  public async getLatest({
+    dataSource,
+    symbol
+  }: AssetProfileIdentifier): Promise<MarketData> {
+    return this.prismaService.marketData.findFirst({
+      orderBy: [{ date: 'desc' }],
+      where: {
+        dataSource,
+        symbol
+      }
+    });
+  }
+
   public async getMax({ dataSource, symbol }: AssetProfileIdentifier) {
     return this.prismaService.marketData.findFirst({
       select: {

apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@@ -2,6 +2,7 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data
 import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
@@ -17,6 +18,7 @@ import {
 import {
   DATE_FORMAT,
   getAssetProfileIdentifier,
+  getStartOfUtcDate,
   resetHours
 } from '@ghostfolio/common/helper';
 import {
@@ -26,7 +28,7 @@ import {
 
 import { InjectQueue } from '@nestjs/bull';
 import { Inject, Injectable, Logger } from '@nestjs/common';
-import { DataSource } from '@prisma/client';
+import { DataSource, Prisma } from '@prisma/client';
 import { JobOptions, Queue } from 'bull';
 import { format, min, subDays, subMilliseconds, subYears } from 'date-fns';
 import { isEmpty } from 'lodash';
@@ -43,6 +45,7 @@ export class DataGatheringService {
     private readonly dataGatheringQueue: Queue,
     private readonly dataProviderService: DataProviderService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
+    private readonly marketDataService: MarketDataService,
     private readonly prismaService: PrismaService,
     private readonly propertyService: PropertyService,
     private readonly symbolProfileService: SymbolProfileService
@@ -279,6 +282,46 @@ export class DataGatheringService {
     }
   }
 
+  public async gatherHourlySymbols() {
+    const assetProfileIdentifiers =
+      await this.getHourlyAssetProfileIdentifiers();
+
+    if (assetProfileIdentifiers.length <= 0) {
+      return;
+    }
+
+    const date = getStartOfUtcDate(new Date());
+
+    try {
+      const quotes = await this.dataProviderService.getQuotes({
+        items: assetProfileIdentifiers,
+        useCache: false
+      });
+
+      const data: Prisma.MarketDataUpdateInput[] = [];
+
+      for (const { dataSource, symbol } of assetProfileIdentifiers) {
+        const quote = quotes[symbol];
+
+        if (quote?.dataSource !== dataSource || !quote.marketPrice) {
+          continue;
+        }
+
+        data.push({
+          dataSource,
+          date,
+          symbol,
+          marketPrice: quote.marketPrice,
+          state: 'INTRADAY'
+        });
+      }
+
+      await this.marketDataService.updateMany({ data });
+    } catch (error) {
+      this.logger.error('Could not gather hourly market data', error);
+    }
+  }
+
   public async gatherSymbols({
     dataGatheringItems,
     force = false,
@@ -389,6 +432,36 @@ export class DataGatheringService {
     return min([aStartDate, subYears(new Date(), 10)]);
   }
 
+  private async getHourlyAssetProfileIdentifiers(): Promise<
+    AssetProfileIdentifier[]
+  > {
+    const symbolProfiles = await this.prismaService.symbolProfile.findMany({
+      orderBy: [{ symbol: 'asc' }, { dataSource: 'asc' }],
+      select: {
+        dataSource: true,
+        scraperConfiguration: true,
+        symbol: true
+      },
+      where: {
+        dataGatheringFrequency: 'HOURLY',
+        isActive: true
+      }
+    });
+
+    return symbolProfiles
+      .filter(({ dataSource, scraperConfiguration }) => {
+        const manualDataSourceWithScraperConfiguration =
+          dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);
+
+        return (
+          dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration
+        );
+      })
+      .map(({ dataSource, symbol }) => {
+        return { dataSource, symbol };
+      });
+  }
+
   private async getSymbols7D({
     withUserSubscription = false
   }: {
@@ -469,14 +542,12 @@ export class DataGatheringService {
         }
       })
     )
-      .filter((symbolProfile) => {
+      .filter(({ dataSource, scraperConfiguration }) => {
         const manualDataSourceWithScraperConfiguration =
-          symbolProfile.dataSource === 'MANUAL' &&
+          dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);
-          !isEmpty(symbolProfile.scraperConfiguration);
 
         return (
-          symbolProfile.dataSource !== 'MANUAL' ||
+          dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration
-          manualDataSourceWithScraperConfiguration
         );
       })
       .map((symbolProfile) => {

apps/api/src/services/symbol-profile/symbol-profile.service.ts

@@ -178,6 +178,7 @@ export class SymbolProfileService {
       comment,
       countries,
       currency,
+      dataGatheringFrequency,
       holdings,
       isActive,
       name,
@@ -195,6 +196,7 @@ export class SymbolProfileService {
         comment,
         countries,
         currency,
+        dataGatheringFrequency,
         holdings,
         isActive,
         name,

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts

@@ -74,6 +74,7 @@ import { IonIcon } from '@ionic/angular/standalone';
 import {
   AssetClass,
   AssetSubClass,
+  DataGatheringFrequency,
   MarketData,
   Prisma,
   SymbolProfile
@@ -155,6 +156,7 @@ export class GfAssetProfileDialogComponent implements OnInit {
     comment: '',
     countries: ['', jsonValidator()],
     currency: '',
+    dataGatheringFrequency: new FormControl<DataGatheringFrequency>('DAILY'),
     historicalData: this.formBuilder.group({
       csvString: ''
     }),
@@ -199,6 +201,20 @@ export class GfAssetProfileDialogComponent implements OnInit {
 
   protected currencies: string[] = [];
 
+  protected readonly dataGatheringFrequencyValues: {
+    value: DataGatheringFrequency;
+    viewValue: string;
+  }[] = [
+    {
+      value: 'DAILY',
+      viewValue: $localize`Daily`
+    },
+    {
+      value: 'HOURLY',
+      viewValue: $localize`Hourly`
+    }
+  ];
+
   protected readonly dateRangeOptions = [
     {
       label: $localize`Current week` + ' (' + $localize`WTD` + ')',
@@ -401,6 +417,8 @@ export class GfAssetProfileDialogComponent implements OnInit {
             }) ?? []
           ),
           currency: this.assetProfile?.currency ?? null,
+          dataGatheringFrequency:
+            this.assetProfile?.dataGatheringFrequency ?? 'DAILY',
           historicalData: {
             csvString: GfAssetProfileDialogComponent.HISTORICAL_DATA_TEMPLATE
           },
@@ -583,6 +601,9 @@ export class GfAssetProfileDialogComponent implements OnInit {
         this.assetProfileForm.controls.assetSubClass.value ?? undefined,
       comment: this.assetProfileForm.controls.comment.value || undefined,
       currency: this.assetProfileForm.controls.currency.value ?? undefined,
+      dataGatheringFrequency:
+        this.assetProfileForm.controls.dataGatheringFrequency.value ??
+        undefined,
       isActive: isBoolean(this.assetProfileForm.controls.isActive.value)
         ? this.assetProfileForm.controls.isActive.value
         : undefined,

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html

@@ -113,7 +113,7 @@
           <ion-icon name="reader-outline" />
           <div class="d-none d-sm-block ml-2" i18n>Overview</div>
         </ng-template>
-        <div class="container mt-3 p-0">
+        <div class="container px-0 py-3">
           <div class="row w-100">
             @if (isEditAssetProfileIdentifierMode) {
               <div class="col-12 mb-4">
@@ -444,6 +444,21 @@
                 ></textarea>
               </mat-form-field>
             </div>
+            <div>
+              <mat-form-field appearance="outline" class="w-100 without-hint">
+                <mat-label i18n>Data Gathering Frequency</mat-label>
+                <mat-select formControlName="dataGatheringFrequency">
+                  @for (
+                    dataGatheringFrequencyValue of dataGatheringFrequencyValues;
+                    track dataGatheringFrequencyValue.value
+                  ) {
+                    <mat-option [value]="dataGatheringFrequencyValue.value">{{
+                      dataGatheringFrequencyValue.viewValue
+                    }}</mat-option>
+                  }
+                </mat-select>
+              </mat-form-field>
+            </div>
           </form>
         </div>
       </mat-tab>

libs/common/src/lib/dtos/update-asset-profile.dto.ts

@@ -1,6 +1,12 @@
 import { IsCurrencyCode } from '@ghostfolio/common/validators/is-currency-code';
 
-import { AssetClass, AssetSubClass, DataSource, Prisma } from '@prisma/client';
+import {
+  AssetClass,
+  AssetSubClass,
+  DataGatheringFrequency,
+  DataSource,
+  Prisma
+} from '@prisma/client';
 import {
   IsArray,
   IsBoolean,
@@ -32,6 +38,10 @@ export class UpdateAssetProfileDto {
   @IsOptional()
   currency?: string;
 
+  @IsEnum(DataGatheringFrequency)
+  @IsOptional()
+  dataGatheringFrequency?: DataGatheringFrequency;
+
   @IsEnum(DataSource)
   @IsOptional()
   dataSource?: DataSource;

libs/common/src/lib/interfaces/enhanced-symbol-profile.interface.ts

@@ -1,4 +1,9 @@
-import { AssetClass, AssetSubClass, DataSource } from '@prisma/client';
+import {
+  AssetClass,
+  AssetSubClass,
+  DataGatheringFrequency,
+  DataSource
+} from '@prisma/client';
 
 import { Country } from './country.interface';
 import { DataProviderInfo } from './data-provider-info.interface';
@@ -15,6 +20,7 @@ export interface EnhancedSymbolProfile {
   createdAt: Date;
   currency?: string;
   cusip?: string;
+  dataGatheringFrequency?: DataGatheringFrequency;
   dataProviderInfo?: DataProviderInfo;
   dataSource: DataSource;
   dateOfFirstActivity?: Date;

libs/common/src/lib/interfaces/responses/export-response.interface.ts

@@ -28,6 +28,7 @@ export interface ExportResponse {
   assetProfiles: (Omit<
     SymbolProfile,
     | 'createdAt'
+    | 'dataGatheringFrequency'
     | 'id'
     | 'scraperConfiguration'
     | 'symbolMapping'

libs/ui/src/lib/services/admin.service.ts

@@ -189,6 +189,7 @@ export class AdminService {
       comment,
       countries,
       currency,
+      dataGatheringFrequency,
       dataSource: newDataSource,
       isActive,
       name,
@@ -207,6 +208,7 @@ export class AdminService {
         comment,
         countries,
         currency,
+        dataGatheringFrequency,
         dataSource: newDataSource,
         isActive,
         name,

prisma/migrations/20260620163851_added_data_gathering_frequency_to_symbol_profile/migration.sql

@@ -0,0 +1,8 @@
+-- CreateEnum
+CREATE TYPE "DataGatheringFrequency" AS ENUM ('DAILY', 'HOURLY');
+
+-- AlterTable
+ALTER TABLE "SymbolProfile" ADD COLUMN "dataGatheringFrequency" "DataGatheringFrequency" NOT NULL DEFAULT 'DAILY';
+
+-- CreateIndex
+CREATE INDEX "SymbolProfile_dataGatheringFrequency_idx" ON "SymbolProfile"("dataGatheringFrequency");

prisma/schema.prisma

@@ -191,6 +191,7 @@ model SymbolProfile {
   createdAt              DateTime                @default(now())
   currency               String
   cusip                  String?
+  dataGatheringFrequency DataGatheringFrequency  @default(DAILY)
   dataSource             DataSource
   figi                   String?
   figiComposite          String?
@@ -215,6 +216,7 @@ model SymbolProfile {
   @@index([assetClass])
   @@index([currency])
   @@index([cusip])
+  @@index([dataGatheringFrequency])
   @@index([dataSource])
   @@index([isActive])
   @@index([isin])
@@ -316,6 +318,11 @@ enum AssetSubClass {
   STOCK
 }
 
+enum DataGatheringFrequency {
+  DAILY
+  HOURLY
+}
+
 enum DataSource {
   ALPHA_VANTAGE
   COINGECKO