From 39527299ae4393fc71317ee0680c740cda1b7567 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Tue, 16 Jul 2024 20:37:30 +0200 Subject: [PATCH] Refactoring --- apps/api/src/app/admin/admin.controller.ts | 10 ++- apps/api/src/services/cron.service.ts | 5 +- .../data-gathering/data-gathering.service.ts | 74 +++++++++---------- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 4494fef7a..69e6955c1 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -81,10 +81,11 @@ export class AdminController { @Post('gather/max') @UseGuards(AuthGuard('jwt'), HasPermissionGuard) public async gatherMax(): Promise { - const uniqueAssets = await this.dataGatheringService.getUniqueAssets(); + const assetProfileIdentifiers = + await this.dataGatheringService.getAllAssetProfileIdentifiers(); await this.dataGatheringService.addJobsToQueue( - uniqueAssets.map(({ dataSource, symbol }) => { + assetProfileIdentifiers.map(({ dataSource, symbol }) => { return { data: { dataSource, @@ -107,10 +108,11 @@ export class AdminController { @Post('gather/profile-data') @UseGuards(AuthGuard('jwt'), HasPermissionGuard) public async gatherProfileData(): Promise { - const uniqueAssets = await this.dataGatheringService.getUniqueAssets(); + const assetProfileIdentifiers = + await this.dataGatheringService.getAllAssetProfileIdentifiers(); await this.dataGatheringService.addJobsToQueue( - uniqueAssets.map(({ dataSource, symbol }) => { + assetProfileIdentifiers.map(({ dataSource, symbol }) => { return { data: { dataSource, diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index fc5d613a2..864891c6a 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -45,10 +45,11 @@ export class CronService { @Cron(CronService.EVERY_SUNDAY_AT_LUNCH_TIME) public async runEverySundayAtTwelvePm() { if (await this.isDataGatheringEnabled()) { - const uniqueAssets = await this.dataGatheringService.getUniqueAssets(); + const assetProfileIdentifiers = + await this.dataGatheringService.getAllAssetProfileIdentifiers(); await this.dataGatheringService.addJobsToQueue( - uniqueAssets.map(({ dataSource, symbol }) => { + assetProfileIdentifiers.map(({ dataSource, symbol }) => { return { data: { dataSource, diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.ts index a4a35445a..2bf6cc1b2 100644 --- a/apps/api/src/services/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering/data-gathering.service.ts @@ -152,7 +152,7 @@ export class DataGatheringService { }); if (!uniqueAssets) { - uniqueAssets = await this.getUniqueAssets(); + uniqueAssets = await this.getAllAssetProfileIdentifiers(); } if (uniqueAssets.length <= 0) { @@ -284,7 +284,7 @@ export class DataGatheringService { ); } - public async getUniqueAssets(): Promise { + public async getAllAssetProfileIdentifiers(): Promise { const symbolProfiles = await this.prismaService.symbolProfile.findMany({ orderBy: [{ symbol: 'asc' }] }); @@ -304,22 +304,44 @@ export class DataGatheringService { }); } - private async getCurrencies7D(): Promise { - const startDate = subDays(resetHours(new Date()), 7); + private async getAssetProfileIdentifiersWithCompleteMarketData(): Promise< + UniqueAsset[] + > { + return ( + await this.prismaService.marketData.groupBy({ + _count: true, + by: ['dataSource', 'symbol'], + orderBy: [{ symbol: 'asc' }], + where: { + date: { gt: subDays(resetHours(new Date()), 7) }, + state: 'CLOSE' + } + }) + ) + .filter(({ _count }) => { + return _count >= 6; + }) + .map(({ dataSource, symbol }) => { + return { dataSource, symbol }; + }); + } - const symbolsWithCompleteMarketData = - await this.getSymbolsWithCompleteMarketData(); + private async getCurrencies7D(): Promise { + const assetProfileIdentifiersWithCompleteMarketData = + await this.getAssetProfileIdentifiersWithCompleteMarketData(); return this.exchangeRateDataService .getCurrencyPairs() - .filter(({ symbol }) => { - return !symbolsWithCompleteMarketData.includes(symbol); + .filter(({ dataSource, symbol }) => { + return !assetProfileIdentifiersWithCompleteMarketData.some((item) => { + return item.dataSource === dataSource && item.symbol === symbol; + }); }) .map(({ dataSource, symbol }) => { return { dataSource, symbol, - date: startDate + date: subDays(resetHours(new Date()), 7) }; }); } @@ -333,15 +355,13 @@ export class DataGatheringService { }: { withUserSubscription?: boolean; }): Promise { - const startDate = subDays(resetHours(new Date()), 7); - const symbolProfiles = await this.symbolProfileService.getSymbolProfilesByUserSubscription({ withUserSubscription }); - const symbolsWithCompleteMarketData = - await this.getSymbolsWithCompleteMarketData(); + const assetProfileIdentifiersWithCompleteMarketData = + await this.getAssetProfileIdentifiersWithCompleteMarketData(); return symbolProfiles .filter(({ dataSource, scraperConfiguration, symbol }) => { @@ -349,14 +369,16 @@ export class DataGatheringService { dataSource === 'MANUAL' && !isEmpty(scraperConfiguration); return ( - !symbolsWithCompleteMarketData.includes(symbol) && + !assetProfileIdentifiersWithCompleteMarketData.some((item) => { + return item.dataSource === dataSource && item.symbol === symbol; + }) && (dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration) ); }) .map((symbolProfile) => { return { ...symbolProfile, - date: startDate + date: subDays(resetHours(new Date()), 7) }; }); } @@ -428,26 +450,4 @@ export class DataGatheringService { return [...currencyPairsToGather, ...symbolProfilesToGather]; } - - private async getSymbolsWithCompleteMarketData() { - const startDate = subDays(resetHours(new Date()), 7); - - return ( - await this.prismaService.marketData.groupBy({ - _count: true, - by: ['symbol'], - orderBy: [{ symbol: 'asc' }], - where: { - date: { gt: startDate }, - state: 'CLOSE' - } - }) - ) - .filter(({ _count }) => { - return _count >= 6; - }) - .map(({ symbol }) => { - return symbol; - }); - } }