Add data gathering mode to symbol profile and gather hourly

4 weeks ago · 3fb5c7fea1
13 changed files with 157 additions and 9 deletions
--- a/apps/api/src/app/admin/admin.service.ts
+++ b/apps/api/src/app/admin/admin.service.ts
@ -287,6 +287,7 @@ export class AdminService {
      comment,
      countries,
      currency,
+      dataGatheringFrequency,
      dataSource: newDataSource,
      holdings,
      isActive,
@ -370,6 +371,7 @@ export class AdminService {
      const updatedSymbolProfile: Prisma.SymbolProfileUpdateInput = {
        comment,
        currency,
+        dataGatheringFrequency,
        dataSource,
        isActive,
        scraperConfiguration,
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@ -536,6 +536,8 @@ export class ImportService {
            url,
            comment: assetProfile.comment,
            currency: assetProfile.currency,
+            dataGatheringFrequency:
+              assetProfile.dataGatheringFrequency ?? 'DAILY',
            userId: dataSource === 'MANUAL' ? user.id : undefined
          },
          symbolProfileId: undefined,
--- a/apps/api/src/services/cron/cron.service.ts
+++ b/apps/api/src/services/cron/cron.service.ts
@ -42,6 +42,7 @@ export class CronService {
  public async runEveryHourAtRandomMinute() {
    if (await this.isDataGatheringEnabled()) {
      await this.dataGatheringService.gather7Days();
+      await this.dataGatheringService.gatherHourlySymbols();
    }
  }

--- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
+++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
@ -2,6 +2,7 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data
 import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
@ -17,6 +18,7 @@ import {
 import {
  DATE_FORMAT,
  getAssetProfileIdentifier,
+  getStartOfUtcDate,
  resetHours
 } from '@ghostfolio/common/helper';
 import {
@ -26,7 +28,7 @@ import {

 import { InjectQueue } from '@nestjs/bull';
 import { Inject, Injectable, Logger } from '@nestjs/common';
-import { DataSource } from '@prisma/client';
+import { DataSource, Prisma } from '@prisma/client';
 import { JobOptions, Queue } from 'bull';
 import { format, min, subDays, subMilliseconds, subYears } from 'date-fns';
 import { isEmpty } from 'lodash';
@ -43,6 +45,7 @@ export class DataGatheringService {
    private readonly dataGatheringQueue: Queue,
    private readonly dataProviderService: DataProviderService,
    private readonly exchangeRateDataService: ExchangeRateDataService,
+    private readonly marketDataService: MarketDataService,
    private readonly prismaService: PrismaService,
    private readonly propertyService: PropertyService,
    private readonly symbolProfileService: SymbolProfileService
@ -153,6 +156,46 @@ export class DataGatheringService {
    }
  }

+  public async gatherHourlySymbols() {
+    const assetProfileIdentifiers =
+      await this.getHourlyAssetProfileIdentifiers();
+
+    if (assetProfileIdentifiers.length <= 0) {
+      return;
+    }
+
+    const date = getStartOfUtcDate(new Date());
+
+    try {
+      const quotes = await this.dataProviderService.getQuotes({
+        items: assetProfileIdentifiers,
+        useCache: false
+      });
+
+      const data: Prisma.MarketDataUpdateInput[] = [];
+
+      for (const { dataSource, symbol } of assetProfileIdentifiers) {
+        const quote = quotes[symbol];
+
+        if (quote?.dataSource !== dataSource || !quote.marketPrice) {
+          continue;
+        }
+
+        data.push({
+          dataSource,
+          date,
+          symbol,
+          marketPrice: quote.marketPrice,
+          state: 'INTRADAY'
+        });
+      }
+
+      await this.marketDataService.updateMany({ data });
+    } catch (error) {
+      this.logger.error('Could not gather hourly market data', error);
+    }
+  }
+
  public async gatherAssetProfiles(
    aAssetProfileIdentifiers?: AssetProfileIdentifier[]
  ) {
@ -389,6 +432,36 @@ export class DataGatheringService {
    return min([aStartDate, subYears(new Date(), 10)]);
  }

+  private async getHourlyAssetProfileIdentifiers(): Promise<
+    AssetProfileIdentifier[]
+  > {
+    const symbolProfiles = await this.prismaService.symbolProfile.findMany({
+      orderBy: [{ symbol: 'asc' }, { dataSource: 'asc' }],
+      select: {
+        dataSource: true,
+        scraperConfiguration: true,
+        symbol: true
+      },
+      where: {
+        dataGatheringFrequency: 'HOURLY',
+        isActive: true
+      }
+    });
+
+    return symbolProfiles
+      .filter(({ dataSource, scraperConfiguration }) => {
+        const manualDataSourceWithScraperConfiguration =
+          dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);
+
+        return (
+          dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration
+        );
+      })
+      .map(({ dataSource, symbol }) => {
+        return { dataSource, symbol };
+      });
+  }
+
  private async getSymbols7D({
    withUserSubscription = false
  }: {
@ -469,14 +542,12 @@ export class DataGatheringService {
        }
      })
    )
-      .filter((symbolProfile) => {
+      .filter(({ dataSource, scraperConfiguration }) => {
        const manualDataSourceWithScraperConfiguration =
-          symbolProfile.dataSource === 'MANUAL' &&
-          !isEmpty(symbolProfile.scraperConfiguration);
+          dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);

        return (
-          symbolProfile.dataSource !== 'MANUAL' ||
-          manualDataSourceWithScraperConfiguration
+          dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration
        );
      })
      .map((symbolProfile) => {
--- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts
+++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts
@ -178,6 +178,7 @@ export class SymbolProfileService {
      comment,
      countries,
      currency,
+      dataGatheringFrequency,
      holdings,
      isActive,
      name,
@ -195,6 +196,7 @@ export class SymbolProfileService {
        comment,
        countries,
        currency,
+        dataGatheringFrequency,
        holdings,
        isActive,
        name,
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
@ -74,6 +74,7 @@ import { IonIcon } from '@ionic/angular/standalone';
 import {
  AssetClass,
  AssetSubClass,
+  DataGatheringFrequency,
  MarketData,
  Prisma,
  SymbolProfile
@ -155,6 +156,7 @@ export class GfAssetProfileDialogComponent implements OnInit {
    comment: '',
    countries: ['', jsonValidator()],
    currency: '',
+    dataGatheringFrequency: new FormControl<DataGatheringFrequency>('DAILY'),
    historicalData: this.formBuilder.group({
      csvString: ''
    }),
@ -199,6 +201,20 @@ export class GfAssetProfileDialogComponent implements OnInit {

  protected currencies: string[] = [];

+  protected readonly dataGatheringFrequencyValues: {
+    value: DataGatheringFrequency;
+    viewValue: string;
+  }[] = [
+    {
+      value: 'DAILY',
+      viewValue: $localize`Daily`
+    },
+    {
+      value: 'HOURLY',
+      viewValue: $localize`Hourly`
+    }
+  ];
+
  protected readonly dateRangeOptions = [
    {
      label: $localize`Current week` + ' (' + $localize`WTD` + ')',
@ -401,6 +417,8 @@ export class GfAssetProfileDialogComponent implements OnInit {
            }) ?? []
          ),
          currency: this.assetProfile?.currency ?? null,
+          dataGatheringFrequency:
+            this.assetProfile?.dataGatheringFrequency ?? 'DAILY',
          historicalData: {
            csvString: GfAssetProfileDialogComponent.HISTORICAL_DATA_TEMPLATE
          },
@ -583,6 +601,9 @@ export class GfAssetProfileDialogComponent implements OnInit {
        this.assetProfileForm.controls.assetSubClass.value ?? undefined,
      comment: this.assetProfileForm.controls.comment.value || undefined,
      currency: this.assetProfileForm.controls.currency.value ?? undefined,
+      dataGatheringFrequency:
+        this.assetProfileForm.controls.dataGatheringFrequency.value ??
+        undefined,
      isActive: isBoolean(this.assetProfileForm.controls.isActive.value)
        ? this.assetProfileForm.controls.isActive.value
        : undefined,
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
@ -113,7 +113,7 @@
          <ion-icon name="reader-outline" />
          <div class="d-none d-sm-block ml-2" i18n>Overview</div>
        </ng-template>
-        <div class="container mt-3 p-0">
+        <div class="container px-0 py-3">
          <div class="row w-100">
            @if (isEditAssetProfileIdentifierMode) {
              <div class="col-12 mb-4">
@ -444,6 +444,21 @@
                ></textarea>
              </mat-form-field>
            </div>
+            <div>
+              <mat-form-field appearance="outline" class="w-100 without-hint">
+                <mat-label i18n>Data Gathering Frequency</mat-label>
+                <mat-select formControlName="dataGatheringFrequency">
+                  @for (
+                    dataGatheringFrequencyValue of dataGatheringFrequencyValues;
+                    track dataGatheringFrequencyValue.value
+                  ) {
+                    <mat-option [value]="dataGatheringFrequencyValue.value">{{
+                      dataGatheringFrequencyValue.viewValue
+                    }}</mat-option>
+                  }
+                </mat-select>
+              </mat-form-field>
+            </div>
          </form>
        </div>
      </mat-tab>
--- a/libs/common/src/lib/dtos/update-asset-profile.dto.ts
+++ b/libs/common/src/lib/dtos/update-asset-profile.dto.ts
@ -1,6 +1,12 @@
 import { IsCurrencyCode } from '@ghostfolio/common/validators/is-currency-code';

-import { AssetClass, AssetSubClass, DataSource, Prisma } from '@prisma/client';
+import {
+  AssetClass,
+  AssetSubClass,
+  DataGatheringFrequency,
+  DataSource,
+  Prisma
+} from '@prisma/client';
 import {
  IsArray,
  IsBoolean,
@ -32,6 +38,10 @@ export class UpdateAssetProfileDto {
  @IsOptional()
  currency?: string;

+  @IsEnum(DataGatheringFrequency)
+  @IsOptional()
+  dataGatheringFrequency?: DataGatheringFrequency;
+
  @IsEnum(DataSource)
  @IsOptional()
  dataSource?: DataSource;
--- a/libs/common/src/lib/interfaces/enhanced-symbol-profile.interface.ts
+++ b/libs/common/src/lib/interfaces/enhanced-symbol-profile.interface.ts
@ -1,4 +1,9 @@
-import { AssetClass, AssetSubClass, DataSource } from '@prisma/client';
+import {
+  AssetClass,
+  AssetSubClass,
+  DataGatheringFrequency,
+  DataSource
+} from '@prisma/client';

 import { Country } from './country.interface';
 import { DataProviderInfo } from './data-provider-info.interface';
@ -15,6 +20,7 @@ export interface EnhancedSymbolProfile {
  createdAt: Date;
  currency?: string;
  cusip?: string;
+  dataGatheringFrequency?: DataGatheringFrequency;
  dataProviderInfo?: DataProviderInfo;
  dataSource: DataSource;
  dateOfFirstActivity?: Date;
--- a/libs/common/src/lib/interfaces/responses/export-response.interface.ts
+++ b/libs/common/src/lib/interfaces/responses/export-response.interface.ts
@ -28,6 +28,7 @@ export interface ExportResponse {
  assetProfiles: (Omit<
    SymbolProfile,
    | 'createdAt'
+    | 'dataGatheringFrequency'
    | 'id'
    | 'scraperConfiguration'
    | 'symbolMapping'
--- a/libs/ui/src/lib/services/admin.service.ts
+++ b/libs/ui/src/lib/services/admin.service.ts
@ -189,6 +189,7 @@ export class AdminService {
      comment,
      countries,
      currency,
+      dataGatheringFrequency,
      dataSource: newDataSource,
      isActive,
      name,
@ -207,6 +208,7 @@ export class AdminService {
        comment,
        countries,
        currency,
+        dataGatheringFrequency,
        dataSource: newDataSource,
        isActive,
        name,
--- a/prisma/migrations/20260620163851_added_data_gathering_frequency_to_symbol_profile/migration.sql
+++ b/prisma/migrations/20260620163851_added_data_gathering_frequency_to_symbol_profile/migration.sql
@ -0,0 +1,8 @@
+-- CreateEnum
+CREATE TYPE "DataGatheringFrequency" AS ENUM ('DAILY', 'HOURLY');
+
+-- AlterTable
+ALTER TABLE "SymbolProfile" ADD COLUMN "dataGatheringFrequency" "DataGatheringFrequency" NOT NULL DEFAULT 'DAILY';
+
+-- CreateIndex
+CREATE INDEX "SymbolProfile_dataGatheringFrequency_idx" ON "SymbolProfile"("dataGatheringFrequency");
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@ -191,6 +191,7 @@ model SymbolProfile {
  createdAt              DateTime                @default(now())
  currency               String
  cusip                  String?
+  dataGatheringFrequency DataGatheringFrequency  @default(DAILY)
  dataSource             DataSource
  figi                   String?
  figiComposite          String?
@ -215,6 +216,7 @@ model SymbolProfile {
  @@index([assetClass])
  @@index([currency])
  @@index([cusip])
+  @@index([dataGatheringFrequency])
  @@index([dataSource])
  @@index([isActive])
  @@index([isin])
@ -316,6 +318,11 @@ enum AssetSubClass {
  STOCK
 }

+enum DataGatheringFrequency {
+  DAILY
+  HOURLY
+}
+
 enum DataSource {
  ALPHA_VANTAGE
  COINGECKO