Introduce maxAge

apps/api/src/app/admin/admin.controller.ts

@@ -93,7 +93,7 @@ export class AdminController {
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   public async gatherMax(): Promise<void> {
     const assetProfileIdentifiers =
-      await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
+      await this.dataGatheringService.getActiveAssetProfileIdentifiers();
 
     await this.dataGatheringService.addJobsToQueue(
       assetProfileIdentifiers.map(({ dataSource, symbol }) => {
@@ -120,7 +120,7 @@ export class AdminController {
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   public async gatherProfileData(): Promise<void> {
     const assetProfileIdentifiers =
-      await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
+      await this.dataGatheringService.getActiveAssetProfileIdentifiers();
 
     await this.dataGatheringService.addJobsToQueue(
       assetProfileIdentifiers.map(({ dataSource, symbol }) => {

apps/api/src/services/cron/cron.service.ts

@@ -59,7 +59,9 @@ export class CronService {
   public async runEverySundayAtTwelvePm() {
     if (await this.isDataGatheringEnabled()) {
       const assetProfileIdentifiers =
-        await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
+        await this.dataGatheringService.getActiveAssetProfileIdentifiers({
+          maxAge: '60 days'
+        });
 
       await this.dataGatheringService.addJobsToQueue(
         assetProfileIdentifiers.map(({ dataSource, symbol }) => {

apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@@ -28,8 +28,9 @@ import { InjectQueue } from '@nestjs/bull';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
 import { JobOptions, Queue } from 'bull';
-import { format, min, subDays, subYears } from 'date-fns';
+import { format, min, subDays, subMilliseconds, subYears } from 'date-fns';
 import { isEmpty } from 'lodash';
+import ms, { StringValue } from 'ms';
 
 @Injectable()
 export class DataGatheringService {
@@ -160,8 +161,7 @@ export class DataGatheringService {
     );
 
     if (!assetProfileIdentifiers) {
-      assetProfileIdentifiers =
-        await this.getAllActiveAssetProfileIdentifiers();
+      assetProfileIdentifiers = await this.getActiveAssetProfileIdentifiers();
     }
 
     if (assetProfileIdentifiers.length <= 0) {
@@ -301,25 +301,36 @@ export class DataGatheringService {
     );
   }
 
-  public async getAllActiveAssetProfileIdentifiers(): Promise<
-    AssetProfileIdentifier[]
-  > {
-    const symbolProfiles = await this.prismaService.symbolProfile.findMany({
-      orderBy: [{ symbol: 'asc' }],
+  /**
+   * Returns active asset profile identifiers
+   *
+   * @param {StringValue} maxAge - Optional. Specifies the maximum allowed age
+   * of a profile’s last update timestamp. Only asset profiles considered stale
+   * are returned.
+   */
+  public async getActiveAssetProfileIdentifiers({
+    maxAge
+  }: {
+    maxAge?: StringValue;
+  } = {}): Promise<AssetProfileIdentifier[]> {
+    return this.prismaService.symbolProfile.findMany({
+      orderBy: [{ symbol: 'asc' }, { dataSource: 'asc' }],
+      select: {
+        dataSource: true,
+        symbol: true
+      },
       where: {
         dataSource: {
           notIn: ['MANUAL', 'RAPID_API']
         },
-        isActive: true
+        isActive: true,
+        ...(maxAge && {
+          updatedAt: {
+            lt: subMilliseconds(new Date(), ms(maxAge))
+          }
+        })
       }
     });
-
-    return symbolProfiles.map(({ dataSource, symbol }) => {
-      return {
-        dataSource,
-        symbol
-      };
-    });
   }
 
   private async getAssetProfileIdentifiersWithCompleteMarketData(): Promise<