From 9468841ae15a29c7c0335a5c6a2b2e89e86dad72 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:45:06 +0200 Subject: [PATCH] Simplify recent market data gathering --- apps/api/jest-environment-tz.js | 3 +- apps/api/src/app/admin/admin.controller.ts | 3 +- .../data-provider/manual/manual.service.ts | 4 +- .../rapid-api/rapid-api.service.ts | 9 +- .../data-gathering.processor.ts | 3 +- .../data-gathering/data-gathering.service.ts | 28 ++- libs/common/src/lib/helper.ts | 5 + package-lock.json | 234 +----------------- package.json | 2 +- 9 files changed, 48 insertions(+), 243 deletions(-) diff --git a/apps/api/jest-environment-tz.js b/apps/api/jest-environment-tz.js index 61f5e7708..ef0d72576 100644 --- a/apps/api/jest-environment-tz.js +++ b/apps/api/jest-environment-tz.js @@ -7,12 +7,11 @@ class TimeZoneEnvironment extends NodeEnvironment { constructor(config, context) { super(config, context); + this.previousTimeZone = process.env.TZ; this.timeZone = config.projectConfig.testEnvironmentOptions?.timeZone; } async setup() { - this.previousTimeZone = process.env.TZ; - if (this.timeZone) { process.env.TZ = this.timeZone; } diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index f47d216f4..6d3f7cfeb 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -37,6 +37,7 @@ import type { RequestWithUser } from '@ghostfolio/common/types'; +import { utc } from '@date-fns/utc'; import { Body, Controller, @@ -204,7 +205,7 @@ export class AdminController { @Param('dateString') dateString: string, @Param('symbol') symbol: string ): Promise { - const date = parseISO(dateString); + const date = parseISO(dateString, { in: utc }); if (!isDate(date)) { throw new HttpException( diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index f13b42bbb..a187335a0 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -14,7 +14,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/sy import { DATE_FORMAT, extractNumberFromString, - getYesterday + getStartOfUtcDateOfYesterday } from '@ghostfolio/common/helper'; import { DataProviderHistoricalResponse, @@ -115,7 +115,7 @@ export class ManualService implements DataProviderInterface { }); return { - [format(getYesterday(), DATE_FORMAT, { in: utc })]: { + [format(getStartOfUtcDateOfYesterday(), DATE_FORMAT, { in: utc })]: { marketPrice: value } }; diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts index 6d9ae1aa1..07dc62393 100644 --- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts @@ -9,7 +9,10 @@ import { } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service'; import { ghostfolioFearAndGreedIndexSymbolStocks } from '@ghostfolio/common/config'; -import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper'; +import { + DATE_FORMAT, + getStartOfUtcDateOfYesterday +} from '@ghostfolio/common/helper'; import { DataProviderHistoricalResponse, DataProviderInfo, @@ -67,7 +70,9 @@ export class RapidApiService implements DataProviderInterface { if (fgi) { return { - [format(getYesterday(), DATE_FORMAT, { in: utc })]: { + [format(getStartOfUtcDateOfYesterday(), DATE_FORMAT, { + in: utc + })]: { marketPrice: fgi.previousClose.value } }; diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts index 3d31ba7bf..ce9c6fe4b 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts @@ -118,9 +118,10 @@ export class DataGatheringProcessor { }); const data: Prisma.MarketDataUpdateInput[] = []; + const startOfUtcDateOfToday = getStartOfUtcDate(new Date()); let lastMarketPrice: number; - while (isBefore(currentDate, getStartOfUtcDate(new Date()))) { + while (isBefore(currentDate, startOfUtcDateOfToday)) { const marketPriceOfDataProvider = historicalData[assetProfileIdentifier]?.[ format(currentDate, DATE_FORMAT, { in: utc }) diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index 03d5e09c3..6f280f5c9 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -20,7 +20,7 @@ import { DATE_FORMAT, getAssetProfileIdentifier, getStartOfUtcDate, - resetHours + getStartOfUtcDateOfYesterday } from '@ghostfolio/common/helper'; import { AssetProfileIdentifier, @@ -323,29 +323,37 @@ export class DataGatheringService { date, symbol }: { date: Date } & AssetProfileIdentifier) { + const startOfUtcDate = getStartOfUtcDate(date); + try { const historicalData = await this.dataProviderService.getHistoricalRaw({ assetProfileIdentifiers: [{ dataSource, symbol }], - from: date, - to: date + from: startOfUtcDate, + to: startOfUtcDate }); const marketPrice = historicalData[getAssetProfileIdentifier({ dataSource, symbol })]?.[ - format(date, DATE_FORMAT) + format(startOfUtcDate, DATE_FORMAT, { in: utc }) ]?.marketPrice; if (marketPrice) { return await this.prismaService.marketData.upsert({ create: { dataSource, - date, marketPrice, symbol, + date: startOfUtcDate, isCarriedForward: false }, update: { marketPrice, isCarriedForward: false }, - where: { dataSource_date_symbol: { dataSource, date, symbol } } + where: { + dataSource_date_symbol: { + dataSource, + symbol, + date: startOfUtcDate + } + } }); } } catch (error) { @@ -383,7 +391,7 @@ export class DataGatheringService { jobId: `${getAssetProfileIdentifier({ dataSource, symbol - })}-${format(date, DATE_FORMAT)}` + })}-${format(date, DATE_FORMAT, { in: utc })}` } }; }) @@ -430,7 +438,7 @@ export class DataGatheringService { by: ['dataSource', 'symbol'], where: { date: { - gte: getStartOfUtcDate(subDays(new Date(), 1, { in: utc })) + gte: getStartOfUtcDateOfYesterday() }, isCarriedForward: false, state: 'CLOSE' @@ -457,7 +465,7 @@ export class DataGatheringService { return { dataSource, symbol, - date: subDays(resetHours(new Date()), 7) + date: subDays(getStartOfUtcDate(new Date()), 7, { in: utc }) }; }); } @@ -525,7 +533,7 @@ export class DataGatheringService { .map((symbolProfile) => { return { ...symbolProfile, - date: subDays(resetHours(new Date()), 7) + date: subDays(getStartOfUtcDate(new Date()), 7, { in: utc }) }; }); } diff --git a/libs/common/src/lib/helper.ts b/libs/common/src/lib/helper.ts index eb1693b9f..260b0f343 100644 --- a/libs/common/src/lib/helper.ts +++ b/libs/common/src/lib/helper.ts @@ -1,3 +1,4 @@ +import { utc } from '@date-fns/utc'; import { NumberParser } from '@internationalized/number'; import { AccessType, @@ -465,6 +466,10 @@ export function getStartOfUtcDate(aDate: Date) { return date; } +export function getStartOfUtcDateOfYesterday() { + return subDays(getStartOfUtcDate(new Date()), 1, { in: utc }); +} + export function getStringOrNull(aString: string | null | undefined) { const trimmedString = aString?.trim(); diff --git a/package-lock.json b/package-lock.json index f9fb959bb..14957e0f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -160,7 +160,7 @@ "husky": "9.1.7", "jest": "30.3.0", "jest-environment-jsdom": "30.2.0", - "jest-environment-node": "30.2.0", + "jest-environment-node": "30.3.0", "jest-preset-angular": "17.0.0", "nx": "23.1.1", "prettier": "3.9.6", @@ -25013,25 +25013,6 @@ } } }, - "node_modules/jest-config/node_modules/jest-environment-node": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz", - "integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.3.0", - "@jest/fake-timers": "30.3.0", - "@jest/types": "30.3.0", - "@types/node": "*", - "jest-mock": "30.3.0", - "jest-util": "30.3.0", - "jest-validate": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, "node_modules/jest-diff": { "version": "30.3.0", "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.3.0.tgz", @@ -25412,195 +25393,19 @@ } }, "node_modules/jest-environment-node": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.2.0.tgz", - "integrity": "sha512-ElU8v92QJ9UrYsKrxDIKCxu6PfNj4Hdcktcn0JX12zqNdqWHB0N+hwOnnBBXvjLd2vApZtuLUGs1QSY+MsXoNA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.2.0", - "@jest/fake-timers": "30.2.0", - "@jest/types": "30.2.0", - "@types/node": "*", - "jest-mock": "30.2.0", - "jest-util": "30.2.0", - "jest-validate": "30.2.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/@jest/environment": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.2.0.tgz", - "integrity": "sha512-/QPTL7OBJQ5ac09UDRa3EQes4gt1FTEG/8jZ/4v5IVzx+Cv7dLxlVIvfvSVRiiX2drWyXeBjkMSR8hvOWSog5g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/fake-timers": "30.2.0", - "@jest/types": "30.2.0", - "@types/node": "*", - "jest-mock": "30.2.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/@jest/fake-timers": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.2.0.tgz", - "integrity": "sha512-HI3tRLjRxAbBy0VO8dqqm7Hb2mIa8d5bg/NJkyQcOk7V118ObQML8RC5luTF/Zsg4474a+gDvhce7eTnP4GhYw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.2.0", - "@sinonjs/fake-timers": "^13.0.0", - "@types/node": "*", - "jest-message-util": "30.2.0", - "jest-mock": "30.2.0", - "jest-util": "30.2.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/@jest/types": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.2.0.tgz", - "integrity": "sha512-H9xg1/sfVvyfU7o3zMfBEjQ1gcsdeTMgqHoYdN79tuLqfTtuu7WckRA1R5whDwOzxaZAeMKTYWqP+WCAi0CHsg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.1", - "@jest/schemas": "30.0.5", - "@types/istanbul-lib-coverage": "^2.0.6", - "@types/istanbul-reports": "^3.0.4", - "@types/node": "*", - "@types/yargs": "^17.0.33", - "chalk": "^4.1.2" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/@sinonjs/fake-timers": { - "version": "13.0.5", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz", - "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@sinonjs/commons": "^3.0.1" - } - }, - "node_modules/jest-environment-node/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-environment-node/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-environment-node/node_modules/jest-message-util": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.2.0.tgz", - "integrity": "sha512-y4DKFLZ2y6DxTWD4cDe07RglV88ZiNEdlRfGtqahfbIjfsw1nMCPx49Uev4IA/hWn3sDKyAnSPwoYSsAEdcimw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.27.1", - "@jest/types": "30.2.0", - "@types/stack-utils": "^2.0.3", - "chalk": "^4.1.2", - "graceful-fs": "^4.2.11", - "micromatch": "^4.0.8", - "pretty-format": "30.2.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.6" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/jest-mock": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.2.0.tgz", - "integrity": "sha512-JNNNl2rj4b5ICpmAcq+WbLH83XswjPbjH4T7yvGzfAGCPh1rw+xVNbtk+FnRslvt9lkCcdn9i1oAoKUuFsOxRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.2.0", - "@types/node": "*", - "jest-util": "30.2.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/jest-util": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.2.0.tgz", - "integrity": "sha512-QKNsM0o3Xe6ISQU869e+DhG+4CK/48aHYdJZGlFQVTjnbvgpcKyxpzk29fGiO7i/J8VENZ+d2iGnSsvmuHywlA==", + "version": "30.3.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz", + "integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.2.0", + "@jest/environment": "30.3.0", + "@jest/fake-timers": "30.3.0", + "@jest/types": "30.3.0", "@types/node": "*", - "chalk": "^4.1.2", - "ci-info": "^4.2.0", - "graceful-fs": "^4.2.11", - "picomatch": "^4.0.2" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/jest-validate": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.2.0.tgz", - "integrity": "sha512-FBGWi7dP2hpdi8nBoWxSsLvBFewKAg0+uSQwBaof4Y4DPgBabXgpSYC5/lR7VmnIlSpASmCi/ntRWPbv7089Pw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/get-type": "30.1.0", - "@jest/types": "30.2.0", - "camelcase": "^6.3.0", - "chalk": "^4.1.2", - "leven": "^3.1.0", - "pretty-format": "30.2.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/pretty-format": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", - "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.5", - "ansi-styles": "^5.2.0", - "react-is": "^18.3.1" + "jest-mock": "30.3.0", + "jest-util": "30.3.0", + "jest-validate": "30.3.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" @@ -26024,25 +25829,6 @@ "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, - "node_modules/jest-runner/node_modules/jest-environment-node": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz", - "integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.3.0", - "@jest/fake-timers": "30.3.0", - "@jest/types": "30.3.0", - "@types/node": "*", - "jest-mock": "30.3.0", - "jest-util": "30.3.0", - "jest-validate": "30.3.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, "node_modules/jest-runner/node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index 746c91e68..c1399e15f 100644 --- a/package.json +++ b/package.json @@ -204,7 +204,7 @@ "husky": "9.1.7", "jest": "30.3.0", "jest-environment-jsdom": "30.2.0", - "jest-environment-node": "30.2.0", + "jest-environment-node": "30.3.0", "jest-preset-angular": "17.0.0", "nx": "23.1.1", "prettier": "3.9.6",