From 0fbd959d337d24a902805e8a8b3827a3d7e9b679 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Mon, 15 Jul 2024 21:06:46 +0200 Subject: [PATCH] Optimize 7d data gathering by prioritization --- .../data-gathering/data-gathering.service.ts | 116 +++++++++++------- .../data-provider/data-provider.service.ts | 24 +--- .../symbol-profile/symbol-profile.service.ts | 9 +- 3 files changed, 78 insertions(+), 71 deletions(-) diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.ts index a80d68d6b..a4a35445a 100644 --- a/apps/api/src/services/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering/data-gathering.service.ts @@ -10,6 +10,7 @@ import { DATA_GATHERING_QUEUE, DATA_GATHERING_QUEUE_PRIORITY_HIGH, DATA_GATHERING_QUEUE_PRIORITY_LOW, + DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, GATHER_HISTORICAL_MARKET_DATA_PROCESS, GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, PROPERTY_BENCHMARKS @@ -62,9 +63,22 @@ export class DataGatheringService { } public async gather7Days() { - const dataGatheringItems = await this.getSymbols7D(); await this.gatherSymbols({ - dataGatheringItems, + dataGatheringItems: await this.getCurrencies7D(), + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); + + await this.gatherSymbols({ + dataGatheringItems: await this.getSymbols7D({ + withUserSubscription: true + }), + priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM + }); + + await this.gatherSymbols({ + dataGatheringItems: await this.getSymbols7D({ + withUserSubscription: false + }), priority: DATA_GATHERING_QUEUE_PRIORITY_LOW }); } @@ -290,43 +304,46 @@ export class DataGatheringService { }); } + private async getCurrencies7D(): Promise { + const startDate = subDays(resetHours(new Date()), 7); + + const symbolsWithCompleteMarketData = + await this.getSymbolsWithCompleteMarketData(); + + return this.exchangeRateDataService + .getCurrencyPairs() + .filter(({ symbol }) => { + return !symbolsWithCompleteMarketData.includes(symbol); + }) + .map(({ dataSource, symbol }) => { + return { + dataSource, + symbol, + date: startDate + }; + }); + } + private getEarliestDate(aStartDate: Date) { return min([aStartDate, subYears(new Date(), 10)]); } - private async getSymbols7D(): Promise { + private async getSymbols7D({ + withUserSubscription = false + }: { + withUserSubscription?: boolean; + }): Promise { const startDate = subDays(resetHours(new Date()), 7); - const symbolProfiles = await this.prismaService.symbolProfile.findMany({ - orderBy: [{ symbol: 'asc' }], - select: { - dataSource: true, - scraperConfiguration: true, - symbol: true - } - }); - - // Only consider symbols with incomplete market data for the last - // 7 days - const symbolsWithCompleteMarketData = ( - await this.prismaService.marketData.groupBy({ - _count: true, - by: ['symbol'], - orderBy: [{ symbol: 'asc' }], - where: { - date: { gt: startDate }, - state: 'CLOSE' - } - }) - ) - .filter((group) => { - return group._count >= 6; - }) - .map((group) => { - return group.symbol; + const symbolProfiles = + await this.symbolProfileService.getSymbolProfilesByUserSubscription({ + withUserSubscription }); - const symbolProfilesToGather = symbolProfiles + const symbolsWithCompleteMarketData = + await this.getSymbolsWithCompleteMarketData(); + + return symbolProfiles .filter(({ dataSource, scraperConfiguration, symbol }) => { const manualDataSourceWithScraperConfiguration = dataSource === 'MANUAL' && !isEmpty(scraperConfiguration); @@ -342,21 +359,6 @@ export class DataGatheringService { date: startDate }; }); - - const currencyPairsToGather = this.exchangeRateDataService - .getCurrencyPairs() - .filter(({ symbol }) => { - return !symbolsWithCompleteMarketData.includes(symbol); - }) - .map(({ dataSource, symbol }) => { - return { - dataSource, - symbol, - date: startDate - }; - }); - - return [...currencyPairsToGather, ...symbolProfilesToGather]; } private async getSymbolsMax(): Promise { @@ -426,4 +428,26 @@ export class DataGatheringService { return [...currencyPairsToGather, ...symbolProfilesToGather]; } + + private async getSymbolsWithCompleteMarketData() { + const startDate = subDays(resetHours(new Date()), 7); + + return ( + await this.prismaService.marketData.groupBy({ + _count: true, + by: ['symbol'], + orderBy: [{ symbol: 'asc' }], + where: { + date: { gt: startDate }, + state: 'CLOSE' + } + }) + ) + .filter(({ _count }) => { + return _count >= 6; + }) + .map(({ symbol }) => { + return symbol; + }); + } } diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index c493ac4d1..9a468f10e 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -9,7 +9,6 @@ import { import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; -import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DEFAULT_CURRENCY, DERIVED_CURRENCIES, @@ -37,8 +36,7 @@ export class DataProviderService { private readonly marketDataService: MarketDataService, private readonly prismaService: PrismaService, private readonly propertyService: PropertyService, - private readonly redisCacheService: RedisCacheService, - private readonly symbolProfileService: SymbolProfileService + private readonly redisCacheService: RedisCacheService ) { this.initialize(); } @@ -48,26 +46,6 @@ export class DataProviderService { ((await this.propertyService.getByKey(PROPERTY_DATA_SOURCE_MAPPING)) as { [dataProviderName: string]: string; }) ?? {}; - - const withSubscription = - await this.symbolProfileService.getSymbolProfilesWithSubscription(true); - console.log('withSubscription', withSubscription.length); - - // const vwrl = withSubscription.find(({ symbol }) => { - // return symbol === 'VWRL.SW'; - // }); - - // console.log({ vwrl }); - - const withoutSubscription = - await this.symbolProfileService.getSymbolProfilesWithSubscription(false); - console.log('withoutSubscription', withoutSubscription.length); - - console.log( - 'Total: ', - withSubscription.length + withoutSubscription.length, - await this.prismaService.symbolProfile.count() - ); } public async checkQuote(dataSource: DataSource) { diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.ts index ea66c388e..e0cfed292 100644 --- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts +++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts @@ -91,7 +91,11 @@ export class SymbolProfileService { }); } - public async getSymbolProfilesWithSubscription(withSubscription = true) { + public async getSymbolProfilesByUserSubscription({ + withUserSubscription = false + }: { + withUserSubscription?: boolean; + }) { return this.prismaService.symbolProfile.findMany({ include: { Order: { @@ -100,8 +104,9 @@ export class SymbolProfileService { } } }, + orderBy: [{ symbol: 'asc' }], where: { - Order: withSubscription + Order: withUserSubscription ? { some: { User: {