Task/consolidate exchange rates gathering together with hourly market data (#7095)

* Consolidate exchange rates gathering with hourly market data

* Update changelog

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Consolidated the exchange rates to be gathered with hourly market data
 - Improved the language localization for German (`de`)
 - Upgraded `@openrouter/ai-sdk-provider` from version `2.9.0` to `2.9.1`
 - Upgraded `undici` from version `7.24.4` to `8.5.0`

apps/api/src/app/admin/admin.controller.ts

@@ -86,8 +86,8 @@ export class AdminController {
   @HasPermission(permissions.accessAdminControl)
   @Post('gather')
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
-  public async gather7Days(): Promise<void> {
+  public async gatherRecentMarketData(): Promise<void> {
-    this.dataGatheringService.gather7Days();
+    this.dataGatheringService.gatherRecentMarketData();
   }
 
   @HasPermission(permissions.accessAdminControl)

apps/api/src/services/cron/cron.module.ts

@@ -2,8 +2,6 @@ import { UserModule } from '@ghostfolio/api/app/user/user.module';
 import { UserService } from '@ghostfolio/api/app/user/user.service';
 import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
@@ -21,7 +19,6 @@ import { CronService } from './cron.service';
   imports: [
     ConfigurationModule,
     DataGatheringQueueModule,
-    ExchangeRateDataModule,
     PropertyModule,
     StatisticsGatheringQueueModule,
     TwitterBotModule,
@@ -32,7 +29,6 @@ import { CronService } from './cron.service';
       inject: [
         ConfigurationService,
         DataGatheringService,
-        ExchangeRateDataService,
         PropertyService,
         StatisticsGatheringService,
         TwitterBotService,
@@ -42,7 +38,6 @@ import { CronService } from './cron.service';
       useFactory: (
         configurationService: ConfigurationService,
         dataGatheringService: DataGatheringService,
-        exchangeRateDataService: ExchangeRateDataService,
         propertyService: PropertyService,
         statisticsGatheringService: StatisticsGatheringService,
         twitterBotService: TwitterBotService,
@@ -57,7 +52,6 @@ import { CronService } from './cron.service';
         return new CronService(
           configurationService,
           dataGatheringService,
-          exchangeRateDataService,
           propertyService,
           statisticsGatheringService,
           twitterBotService,

apps/api/src/services/cron/cron.service.ts

@@ -1,6 +1,5 @@
 import { UserService } from '@ghostfolio/api/app/user/user.service';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service';
 import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service';
@@ -24,7 +23,6 @@ export class CronService {
   public constructor(
     private readonly configurationService: ConfigurationService,
     private readonly dataGatheringService: DataGatheringService,
-    private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly propertyService: PropertyService,
     private readonly statisticsGatheringService: StatisticsGatheringService,
     private readonly twitterBotService: TwitterBotService,
@@ -41,16 +39,11 @@ export class CronService {
   @Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE)
   public async runEveryHourAtRandomMinute() {
     if (await this.isDataGatheringEnabled()) {
-      await this.dataGatheringService.gather7Days();
+      await this.dataGatheringService.gatherHourlyMarketData();
-      await this.dataGatheringService.gatherHourlySymbols();
+      await this.dataGatheringService.gatherRecentMarketData();
     }
   }
 
-  @Cron(CronExpression.EVERY_12_HOURS)
-  public async runEveryTwelveHours() {
-    await this.exchangeRateDataService.loadCurrencies();
-  }
-
   @Cron(CronExpression.EVERY_DAY_AT_5PM)
   public async runEveryDayAtFivePm() {
     if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) {

apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@@ -69,91 +69,6 @@ export class DataGatheringService {
     return this.dataGatheringQueue.addBulk(jobs);
   }
 
-  public async gather7Days() {
-    await this.gatherSymbols({
-      dataGatheringItems: await this.getCurrencies7D(),
-      priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
-    });
-
-    await this.gatherSymbols({
-      dataGatheringItems: await this.getSymbols7D({
-        withUserSubscription: true
-      }),
-      priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
-    });
-
-    await this.gatherSymbols({
-      dataGatheringItems: await this.getSymbols7D({
-        withUserSubscription: false
-      }),
-      priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
-    });
-  }
-
-  public async gatherMax() {
-    const dataGatheringItems = await this.getSymbolsMax();
-    await this.gatherSymbols({
-      dataGatheringItems,
-      priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
-    });
-  }
-
-  public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) {
-    const dataGatheringItems = (await this.getSymbolsMax())
-      .filter((dataGatheringItem) => {
-        return (
-          dataGatheringItem.dataSource === dataSource &&
-          dataGatheringItem.symbol === symbol
-        );
-      })
-      .map((item) => ({
-        ...item,
-        date: date ?? item.date
-      }));
-
-    await this.gatherSymbols({
-      dataGatheringItems,
-      force: true,
-      priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
-    });
-  }
-
-  public async gatherSymbolForDate({
-    dataSource,
-    date,
-    symbol
-  }: { date: Date } & AssetProfileIdentifier) {
-    try {
-      const historicalData = await this.dataProviderService.getHistoricalRaw({
-        assetProfileIdentifiers: [{ dataSource, symbol }],
-        from: date,
-        to: date
-      });
-
-      const marketPrice =
-        historicalData[getAssetProfileIdentifier({ dataSource, symbol })][
-          format(date, DATE_FORMAT)
-        ].marketPrice;
-
-      if (marketPrice) {
-        return await this.prismaService.marketData.upsert({
-          create: {
-            dataSource,
-            date,
-            marketPrice,
-            symbol
-          },
-          update: { marketPrice },
-          where: { dataSource_date_symbol: { dataSource, date, symbol } }
-        });
-      }
-    } catch (error) {
-      this.logger.error(error);
-    } finally {
-      return undefined;
-    }
-  }
-
   public async gatherAssetProfiles(
     aAssetProfileIdentifiers?: AssetProfileIdentifier[]
   ) {
@@ -282,7 +197,13 @@ export class DataGatheringService {
     }
   }
 
-  public async gatherHourlySymbols() {
+  public async gatherHourlyMarketData() {
+    try {
+      await this.exchangeRateDataService.loadCurrencies();
+    } catch (error) {
+      this.logger.error('Could not gather exchange rates', error);
+    }
+
     const assetProfileIdentifiers =
       await this.getHourlyAssetProfileIdentifiers();
 
@@ -322,6 +243,91 @@ export class DataGatheringService {
     }
   }
 
+  public async gatherMax() {
+    const dataGatheringItems = await this.getSymbolsMax();
+    await this.gatherSymbols({
+      dataGatheringItems,
+      priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
+    });
+  }
+
+  public async gatherRecentMarketData() {
+    await this.gatherSymbols({
+      dataGatheringItems: await this.getCurrencies7D(),
+      priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+    });
+
+    await this.gatherSymbols({
+      dataGatheringItems: await this.getSymbols7D({
+        withUserSubscription: true
+      }),
+      priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
+    });
+
+    await this.gatherSymbols({
+      dataGatheringItems: await this.getSymbols7D({
+        withUserSubscription: false
+      }),
+      priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
+    });
+  }
+
+  public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) {
+    const dataGatheringItems = (await this.getSymbolsMax())
+      .filter((dataGatheringItem) => {
+        return (
+          dataGatheringItem.dataSource === dataSource &&
+          dataGatheringItem.symbol === symbol
+        );
+      })
+      .map((item) => ({
+        ...item,
+        date: date ?? item.date
+      }));
+
+    await this.gatherSymbols({
+      dataGatheringItems,
+      force: true,
+      priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+    });
+  }
+
+  public async gatherSymbolForDate({
+    dataSource,
+    date,
+    symbol
+  }: { date: Date } & AssetProfileIdentifier) {
+    try {
+      const historicalData = await this.dataProviderService.getHistoricalRaw({
+        assetProfileIdentifiers: [{ dataSource, symbol }],
+        from: date,
+        to: date
+      });
+
+      const marketPrice =
+        historicalData[getAssetProfileIdentifier({ dataSource, symbol })][
+          format(date, DATE_FORMAT)
+        ].marketPrice;
+
+      if (marketPrice) {
+        return await this.prismaService.marketData.upsert({
+          create: {
+            dataSource,
+            date,
+            marketPrice,
+            symbol
+          },
+          update: { marketPrice },
+          where: { dataSource_date_symbol: { dataSource, date, symbol } }
+        });
+      }
+    } catch (error) {
+      this.logger.error(error);
+    } finally {
+      return undefined;
+    }
+  }
+
   public async gatherSymbols({
     dataGatheringItems,
     force = false,

apps/client/src/app/components/admin-market-data/admin-market-data.component.ts

@@ -305,17 +305,6 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit {
     );
   }
 
-  protected onGather7Days() {
-    this.adminService
-      .gather7Days()
-      .pipe(takeUntilDestroyed(this.destroyRef))
-      .subscribe(() => {
-        setTimeout(() => {
-          window.location.reload();
-        }, 300);
-      });
-  }
-
   protected onGatherMax() {
     this.adminService
       .gatherMax()
@@ -334,6 +323,17 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit {
       .subscribe();
   }
 
+  protected onGatherRecentMarketData() {
+    this.adminService
+      .gatherRecentMarketData()
+      .pipe(takeUntilDestroyed(this.destroyRef))
+      .subscribe(() => {
+        setTimeout(() => {
+          window.location.reload();
+        }, 300);
+      });
+  }
+
   protected onOpenAssetProfileDialog({
     dataSource,
     symbol

apps/client/src/app/components/admin-market-data/admin-market-data.html

@@ -220,7 +220,7 @@
                 class="no-max-width"
                 xPosition="before"
               >
-                <button mat-menu-item (click)="onGather7Days()">
+                <button mat-menu-item (click)="onGatherRecentMarketData()">
                   <ng-container i18n
                     >Gather Recent Historical Market Data</ng-container
                   >

libs/ui/src/lib/services/admin.service.ts

@@ -127,10 +127,6 @@ export class AdminService {
     return this.http.get<AdminUsersResponse>('/api/v1/admin/user', { params });
   }
 
-  public gather7Days() {
-    return this.http.post<void>('/api/v1/admin/gather', {});
-  }
-
   public gatherMax() {
     return this.http.post<void>('/api/v1/admin/gather/max', {});
   }
@@ -149,6 +145,10 @@ export class AdminService {
     );
   }
 
+  public gatherRecentMarketData() {
+    return this.http.post<void>('/api/v1/admin/gather', {});
+  }
+
   public gatherSymbol({
     dataSource,
     range,