From 96c7465f1249c655e0b45a3a236cff0084a9b907 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Thu, 10 Apr 2025 20:27:17 +0200 Subject: [PATCH 1/6] Optimize query in getRange() --- .../services/market-data/market-data.service.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 390586b37..0f658ced3 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -75,17 +75,13 @@ export class MarketDataService { } ], where: { - dataSource: { - in: assetProfileIdentifiers.map(({ dataSource }) => { - return dataSource; - }) - }, date: dateQuery, - symbol: { - in: assetProfileIdentifiers.map(({ symbol }) => { - return symbol; - }) - } + OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => { + return { + dataSource, + symbol + }; + }) } }); } From 18253e68f7ee37fafed4671445c47a614d9031ae Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Thu, 10 Apr 2025 20:28:30 +0200 Subject: [PATCH 2/6] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd591b934..af4ba15cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Optimized the query of the data range functionality (`getRange()`) in the market data service - Improved the language localization for Enlish (`en`) - Upgraded `eslint` dependencies - Upgraded `Nx` from version `20.6.4` to `20.7.1` From 659fe44f1221b5211e0a9f754cef1dd0a7cc59c5 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 12 Apr 2025 20:56:42 +0200 Subject: [PATCH 3/6] 2.152.0-beta.1 --- CHANGELOG.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a03bea515..9a7073adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## 2.152.0-beta.1 - 2025-04-12 ### Changed diff --git a/package-lock.json b/package-lock.json index 2b3b33c90..378df1a3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ghostfolio", - "version": "2.151.0", + "version": "2.152.0-beta.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ghostfolio", - "version": "2.151.0", + "version": "2.152.0-beta.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/package.json b/package.json index b657d6610..744b6119f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ghostfolio", - "version": "2.151.0", + "version": "2.152.0-beta.1", "homepage": "https://ghostfol.io", "license": "AGPL-3.0", "repository": "https://github.com/ghostfolio/ghostfolio", From e112a6ed9d6a85afe4a8e102a74f258f5b11cd88 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 13 Apr 2025 08:17:36 +0200 Subject: [PATCH 4/6] Introduce market data page size --- .../src/app/portfolio/current-rate.service.ts | 107 +++++++++--------- .../market-data/market-data.service.ts | 28 ++++- 2 files changed, 83 insertions(+), 52 deletions(-) diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index 058bf1dd5..1f60ca16f 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -21,6 +21,8 @@ import { GetValuesParams } from './interfaces/get-values-params.interface'; @Injectable() export class CurrentRateService { + private static readonly MARKET_DATA_PAGE_SIZE = 50000; + public constructor( private readonly dataProviderService: DataProviderService, private readonly marketDataService: MarketDataService, @@ -41,42 +43,37 @@ export class CurrentRateService { (!dateQuery.gte || isBefore(dateQuery.gte, new Date())) && (!dateQuery.in || this.containsToday(dateQuery.in)); - const promises: Promise[] = []; const quoteErrors: ResponseError['errors'] = []; const today = resetHours(new Date()); + const values: GetValueObject[] = []; if (includesToday) { - promises.push( - this.dataProviderService - .getQuotes({ items: dataGatheringItems, user: this.request?.user }) - .then((dataResultProvider) => { - const result: GetValueObject[] = []; - - for (const { dataSource, symbol } of dataGatheringItems) { - if (dataResultProvider?.[symbol]?.dataProviderInfo) { - dataProviderInfos.push( - dataResultProvider[symbol].dataProviderInfo - ); - } + const quotesBySymbol = await this.dataProviderService.getQuotes({ + items: dataGatheringItems, + user: this.request?.user + }); - if (dataResultProvider?.[symbol]?.marketPrice) { - result.push({ - dataSource, - symbol, - date: today, - marketPrice: dataResultProvider?.[symbol]?.marketPrice - }); - } else { - quoteErrors.push({ - dataSource, - symbol - }); - } - } + for (const { dataSource, symbol } of dataGatheringItems) { + const quote = quotesBySymbol[symbol]; - return result; - }) - ); + if (quote?.dataProviderInfo) { + dataProviderInfos.push(quote.dataProviderInfo); + } + + if (quote?.marketPrice) { + values.push({ + dataSource, + symbol, + date: today, + marketPrice: quote.marketPrice + }); + } else { + quoteErrors.push({ + dataSource, + symbol + }); + } + } } const assetProfileIdentifiers: AssetProfileIdentifier[] = @@ -84,34 +81,42 @@ export class CurrentRateService { return { dataSource, symbol }; }); - promises.push( - this.marketDataService - .getRange({ - assetProfileIdentifiers, - dateQuery - }) - .then((data) => { - return data.map(({ dataSource, date, marketPrice, symbol }) => { - return { - dataSource, - date, - marketPrice, - symbol - }; - }); - }) - ); - - const values = await Promise.all(promises).then((array) => { - return array.flat(); + const marketDataCount = await this.marketDataService.getRangeCount({ + assetProfileIdentifiers, + dateQuery }); + for ( + let i = 0; + i < marketDataCount; + i += CurrentRateService.MARKET_DATA_PAGE_SIZE + ) { + // Use pageSize to limit the number of records fetched at once + const data = await this.marketDataService.getRange({ + assetProfileIdentifiers, + dateQuery, + skip: i, + take: CurrentRateService.MARKET_DATA_PAGE_SIZE + }); + + values.push( + ...data.map(({ dataSource, date, marketPrice, symbol }) => ({ + dataSource, + date, + marketPrice, + symbol + })) + ); + } + const response: GetValuesObject = { dataProviderInfos, errors: quoteErrors.map(({ dataSource, symbol }) => { return { dataSource, symbol }; }), - values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`) + values: uniqBy(values, ({ date, symbol }) => { + return `${date}-${symbol}`; + }) }; if (!isEmpty(quoteErrors)) { diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 0f658ced3..58b9b09ec 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -60,12 +60,18 @@ export class MarketDataService { public async getRange({ assetProfileIdentifiers, - dateQuery + dateQuery, + skip, + take }: { assetProfileIdentifiers: AssetProfileIdentifier[]; dateQuery: DateQuery; + skip?: number; + take?: number; }): Promise { return this.prismaService.marketData.findMany({ + skip, + take, orderBy: [ { date: 'asc' @@ -86,6 +92,26 @@ export class MarketDataService { }); } + public async getRangeCount({ + assetProfileIdentifiers, + dateQuery + }: { + assetProfileIdentifiers: AssetProfileIdentifier[]; + dateQuery: DateQuery; + }): Promise { + return this.prismaService.marketData.count({ + where: { + date: dateQuery, + OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => { + return { + dataSource, + symbol + }; + }) + } + }); + } + public async marketDataItems(params: { select?: Prisma.MarketDataSelectScalar; skip?: number; From 6da0e9fd88239f53403c03187328c996509c505e Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 13 Apr 2025 08:18:37 +0200 Subject: [PATCH 5/6] 2.152.0-beta.2 --- CHANGELOG.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a7073adc..a078f676f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## 2.152.0-beta.1 - 2025-04-12 +## 2.152.0-beta.2 - 2025-04-13 ### Changed diff --git a/package-lock.json b/package-lock.json index 378df1a3f..3f755618b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ghostfolio", - "version": "2.152.0-beta.1", + "version": "2.152.0-beta.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ghostfolio", - "version": "2.152.0-beta.1", + "version": "2.152.0-beta.2", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/package.json b/package.json index 744b6119f..c71a4a47b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ghostfolio", - "version": "2.152.0-beta.1", + "version": "2.152.0-beta.2", "homepage": "https://ghostfol.io", "license": "AGPL-3.0", "repository": "https://github.com/ghostfolio/ghostfolio", From 2e1349f6a3cc76ff77035078a07eddcbde1c3793 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 13 Apr 2025 08:33:00 +0200 Subject: [PATCH 6/6] Update tests --- .../portfolio/current-rate.service.spec.ts | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts index d0e61c8ce..d8b7482e7 100644 --- a/apps/api/src/app/portfolio/current-rate.service.spec.ts +++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts @@ -6,6 +6,7 @@ import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { DataSource, MarketData } from '@prisma/client'; import { CurrentRateService } from './current-rate.service'; +import { DateQuery } from './interfaces/date-query.interface'; import { GetValuesObject } from './interfaces/get-values-object.interface'; jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => { @@ -25,33 +26,40 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => { }, getRange: ({ assetProfileIdentifiers, - dateRangeEnd, - dateRangeStart + dateQuery }: { assetProfileIdentifiers: AssetProfileIdentifier[]; - dateRangeEnd: Date; - dateRangeStart: Date; + dateQuery: DateQuery; + skip?: number; + take?: number; }) => { return Promise.resolve([ { - createdAt: dateRangeStart, + createdAt: dateQuery.gte, dataSource: assetProfileIdentifiers[0].dataSource, - date: dateRangeStart, + date: dateQuery.gte, id: '8fa48fde-f397-4b0d-adbc-fb940e830e6d', marketPrice: 1841.823902, state: 'CLOSE', symbol: assetProfileIdentifiers[0].symbol }, { - createdAt: dateRangeEnd, + createdAt: dateQuery.lt, dataSource: assetProfileIdentifiers[0].dataSource, - date: dateRangeEnd, + date: dateQuery.lt, id: '082d6893-df27-4c91-8a5d-092e84315b56', marketPrice: 1847.839966, state: 'CLOSE', symbol: assetProfileIdentifiers[0].symbol } ]); + }, + getRangeCount: ({}: { + assetProfileIdentifiers: AssetProfileIdentifier[]; + dateRangeEnd: Date; + dateRangeStart: Date; + }) => { + return Promise.resolve(2); } }; }) @@ -128,9 +136,15 @@ describe('CurrentRateService', () => { values: [ { dataSource: 'YAHOO', - date: undefined, + date: new Date('2020-01-01T00:00:00.000Z'), marketPrice: 1841.823902, symbol: 'AMZN' + }, + { + dataSource: 'YAHOO', + date: new Date('2020-01-02T00:00:00.000Z'), + marketPrice: 1847.839966, + symbol: 'AMZN' } ] });