Fix data gathering blocked by failed jobs

apps/api/src/app/admin/admin.controller.ts

@@ -158,6 +158,11 @@ export class AdminController {
     @Param('dataSource') dataSource: DataSource,
     @Param('symbol') symbol: string
   ): Promise<void> {
+    await this.dataGatheringService.removeAssetProfileJobFromQueue({
+      dataSource,
+      symbol
+    });
+
     await this.dataGatheringService.addJobToQueue({
       data: {
         dataSource,

apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@@ -421,6 +421,20 @@ export class DataGatheringService {
     });
   }
 
+  /**
+   * Removes the asset profile job of a symbol from the queue
+   */
+  public async removeAssetProfileJobFromQueue({
+    dataSource,
+    symbol
+  }: AssetProfileIdentifier) {
+    const job = await this.dataGatheringQueue.getJob(
+      `${getAssetProfileIdentifier({ dataSource, symbol })}:${dataSource}`
+    );
+
+    return job?.remove();
+  }
+
   private async getAssetProfileIdentifiersWithRecentMarketData(): Promise<
     AssetProfileIdentifier[]
   > {

libs/common/src/lib/config.ts

@@ -200,7 +200,9 @@ export const GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS: JobOptions = {
     delay: ms('1 minute'),
     type: 'exponential'
   },
-  removeOnComplete: true
+  removeOnComplete: true,
+  removeOnFail: true,
+  timeout: ms('5 minutes')
 };
 
 export const GATHER_HISTORICAL_MARKET_DATA_COOLDOWN_IN_MS = ms('12 hours');
@@ -212,7 +214,9 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = {
     delay: ms('1 minute'),
     type: 'exponential'
   },
-  removeOnComplete: true
+  removeOnComplete: true,
+  removeOnFail: true,
+  timeout: ms('5 minutes')
 };
 
 export const GATHER_STATISTICS_PROCESS_JOB_OPTIONS: JobOptions = {