From d341c4804abab279484f21909ab018a5761a8013 Mon Sep 17 00:00:00 2001
From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com>
Date: Sat, 29 Nov 2025 13:57:57 +0100
Subject: [PATCH] Feature/improve asset profile data gathering (#5997)

* Improve asset profile data gathering

* Update changelog
---
 CHANGELOG.md                                  |  1 +
 apps/api/src/app/admin/admin.controller.ts    |  4 +-
 apps/api/src/services/cron/cron.service.ts    |  4 +-
 .../data-gathering/data-gathering.service.ts  | 53 +++++++++++--------
 4 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index aa78ea5c0..5acae5a13 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Restricted the asset profile data gathering on Sundays to only process outdated asset profiles
 - Eliminated `uuid` in favor of using `randomUUID` from `node:crypto`
 - Upgraded `color` from version `5.0.0` to `5.0.3`
 
diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts
index 8b5da4965..24467c732 100644
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@@ -93,7 +93,7 @@ export class AdminController {
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   public async gatherMax(): Promise<void> {
     const assetProfileIdentifiers =
-      await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
+      await this.dataGatheringService.getActiveAssetProfileIdentifiers();
 
     await this.dataGatheringService.addJobsToQueue(
       assetProfileIdentifiers.map(({ dataSource, symbol }) => {
@@ -120,7 +120,7 @@ export class AdminController {
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   public async gatherProfileData(): Promise<void> {
     const assetProfileIdentifiers =
-      await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
+      await this.dataGatheringService.getActiveAssetProfileIdentifiers();
 
     await this.dataGatheringService.addJobsToQueue(
       assetProfileIdentifiers.map(({ dataSource, symbol }) => {
diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts
index 88fcabce2..ee91a811e 100644
--- a/apps/api/src/services/cron/cron.service.ts
+++ b/apps/api/src/services/cron/cron.service.ts
@@ -59,7 +59,9 @@ export class CronService {
   public async runEverySundayAtTwelvePm() {
     if (await this.isDataGatheringEnabled()) {
       const assetProfileIdentifiers =
-        await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
+        await this.dataGatheringService.getActiveAssetProfileIdentifiers({
+          maxAge: '60 days'
+        });
 
       await this.dataGatheringService.addJobsToQueue(
         assetProfileIdentifiers.map(({ dataSource, symbol }) => {
diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
index c433f692f..cec63c3eb 100644
--- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
+++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
@@ -28,8 +28,9 @@ import { InjectQueue } from '@nestjs/bull';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
 import { JobOptions, Queue } from 'bull';
-import { format, min, subDays, subYears } from 'date-fns';
+import { format, min, subDays, subMilliseconds, subYears } from 'date-fns';
 import { isEmpty } from 'lodash';
+import ms, { StringValue } from 'ms';
 
 @Injectable()
 export class DataGatheringService {
@@ -160,8 +161,7 @@ export class DataGatheringService {
     );
 
     if (!assetProfileIdentifiers) {
-      assetProfileIdentifiers =
-        await this.getAllActiveAssetProfileIdentifiers();
+      assetProfileIdentifiers = await this.getActiveAssetProfileIdentifiers();
     }
 
     if (assetProfileIdentifiers.length <= 0) {
@@ -301,29 +301,36 @@ export class DataGatheringService {
     );
   }
 
-  public async getAllActiveAssetProfileIdentifiers(): Promise<
-    AssetProfileIdentifier[]
-  > {
-    const symbolProfiles = await this.prismaService.symbolProfile.findMany({
-      orderBy: [{ symbol: 'asc' }],
+  /**
+   * Returns active asset profile identifiers
+   *
+   * @param {StringValue} maxAge - Optional. Specifies the maximum allowed age
+   * of a profile’s last update timestamp. Only asset profiles considered stale
+   * are returned.
+   */
+  public async getActiveAssetProfileIdentifiers({
+    maxAge
+  }: {
+    maxAge?: StringValue;
+  } = {}): Promise<AssetProfileIdentifier[]> {
+    return this.prismaService.symbolProfile.findMany({
+      orderBy: [{ symbol: 'asc' }, { dataSource: 'asc' }],
+      select: {
+        dataSource: true,
+        symbol: true
+      },
       where: {
-        isActive: true
+        dataSource: {
+          notIn: ['MANUAL', 'RAPID_API']
+        },
+        isActive: true,
+        ...(maxAge && {
+          updatedAt: {
+            lt: subMilliseconds(new Date(), ms(maxAge))
+          }
+        })
       }
     });
-
-    return symbolProfiles
-      .filter(({ dataSource }) => {
-        return (
-          dataSource !== DataSource.MANUAL &&
-          dataSource !== DataSource.RAPID_API
-        );
-      })
-      .map(({ dataSource, symbol }) => {
-        return {
-          dataSource,
-          symbol
-        };
-      });
   }
 
   private async getAssetProfileIdentifiersWithCompleteMarketData(): Promise<