From 24c117720f334ceb7ff8f1739edea4d54a7c116d Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:53:21 +0200 Subject: [PATCH] Simplify recent market data gathering --- .../coingecko/coingecko.service.ts | 5 +- .../data-provider/data-provider.service.ts | 8 +- .../data-provider/manual/manual.service.ts | 7 +- .../rapid-api/rapid-api.service.ts | 3 +- .../yahoo-finance/yahoo-finance.service.ts | 5 +- .../data-gathering.processor.ts | 2 +- .../data-gathering/data-gathering.service.ts | 5 +- package-lock.json | 233 +++++++++++++++++- package.json | 1 + 9 files changed, 249 insertions(+), 20 deletions(-) diff --git a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts index 96bc00561..208e23269 100644 --- a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts +++ b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts @@ -18,6 +18,7 @@ import { LookupResponse } from '@ghostfolio/common/interfaces'; +import { utc } from '@date-fns/utc'; import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { AssetClass, @@ -147,7 +148,9 @@ export class CoinGeckoService implements DataProviderInterface, OnModuleInit { } = {}; for (const [timestamp, marketPrice] of prices) { - result[format(fromUnixTime(timestamp / 1000), DATE_FORMAT)] = { + result[ + format(fromUnixTime(timestamp / 1000), DATE_FORMAT, { in: utc }) + ] = { marketPrice }; } diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 85b4a068f..e9449be9e 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -35,6 +35,7 @@ import { } from '@ghostfolio/common/interfaces'; import type { Granularity, UserWithSettings } from '@ghostfolio/common/types'; +import { utc } from '@date-fns/utc'; import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { DataSource, MarketData, Prisma, SymbolProfile } from '@prisma/client'; import { Big } from 'big.js'; @@ -494,8 +495,11 @@ export class DataProviderService implements OnModuleInit { [date: string]: DataProviderHistoricalResponse; } = {}; - for (const date of eachDayOfInterval({ end: to, start: from })) { - data[format(date, DATE_FORMAT)] = { marketPrice: 100 }; + for (const date of eachDayOfInterval( + { end: to, start: from }, + { in: utc } + )) { + data[format(date, DATE_FORMAT, { in: utc })] = { marketPrice: 100 }; } promises.push( diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 1d09c91fe..f13b42bbb 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -24,6 +24,7 @@ import { ScraperConfiguration } from '@ghostfolio/common/interfaces'; +import { utc } from '@date-fns/utc'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; import * as cheerio from 'cheerio'; @@ -96,11 +97,11 @@ export class ManualService implements DataProviderInterface { let date = from; while (isBefore(date, to)) { - historical[format(date, DATE_FORMAT)] = { + historical[format(date, DATE_FORMAT, { in: utc })] = { marketPrice: defaultMarketPrice }; - date = addDays(date, 1); + date = addDays(date, 1, { in: utc }); } return historical; @@ -114,7 +115,7 @@ export class ManualService implements DataProviderInterface { }); return { - [format(getYesterday(), DATE_FORMAT)]: { + [format(getYesterday(), DATE_FORMAT, { in: utc })]: { marketPrice: value } }; diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts index f14bfec14..6d9ae1aa1 100644 --- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts @@ -17,6 +17,7 @@ import { LookupResponse } from '@ghostfolio/common/interfaces'; +import { utc } from '@date-fns/utc'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; import { format } from 'date-fns'; @@ -66,7 +67,7 @@ export class RapidApiService implements DataProviderInterface { if (fgi) { return { - [format(getYesterday(), DATE_FORMAT)]: { + [format(getYesterday(), DATE_FORMAT, { in: utc })]: { marketPrice: fgi.previousClose.value } }; diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index d0c7a204a..ca6357e25 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -19,6 +19,7 @@ import { LookupResponse } from '@ghostfolio/common/interfaces'; +import { utc } from '@date-fns/utc'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; import { addDays, format, isSameDay } from 'date-fns'; @@ -103,7 +104,7 @@ export class YahooFinanceService implements DataProviderInterface { } = {}; for (const historicalItem of historicalResult) { - response[format(historicalItem.date, DATE_FORMAT)] = { + response[format(historicalItem.date, DATE_FORMAT, { in: utc })] = { marketPrice: historicalItem.dividends }; } @@ -174,7 +175,7 @@ export class YahooFinanceService implements DataProviderInterface { : undefined); if (marketPrice) { - response[format(date, DATE_FORMAT)] = { marketPrice }; + response[format(date, DATE_FORMAT, { in: utc })] = { marketPrice }; } } diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts index 533195134..3d31ba7bf 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts @@ -141,7 +141,7 @@ export class DataGatheringProcessor { }); } - currentDate = addDays(currentDate, 1); + currentDate = addDays(currentDate, 1, { in: utc }); } if (force) { diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index 8e7478df8..03d5e09c3 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -27,6 +27,7 @@ import { BenchmarkProperty } from '@ghostfolio/common/interfaces'; +import { utc } from '@date-fns/utc'; import { InjectQueue } from '@nestjs/bull'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { Prisma } from '@prisma/client'; @@ -428,7 +429,9 @@ export class DataGatheringService { await this.prismaService.marketData.groupBy({ by: ['dataSource', 'symbol'], where: { - date: { gte: getStartOfUtcDate(subDays(new Date(), 1)) }, + date: { + gte: getStartOfUtcDate(subDays(new Date(), 1, { in: utc })) + }, isCarriedForward: false, state: 'CLOSE' } diff --git a/package-lock.json b/package-lock.json index 79917d90e..f9fb959bb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -160,6 +160,7 @@ "husky": "9.1.7", "jest": "30.3.0", "jest-environment-jsdom": "30.2.0", + "jest-environment-node": "30.2.0", "jest-preset-angular": "17.0.0", "nx": "23.1.1", "prettier": "3.9.6", @@ -25012,6 +25013,25 @@ } } }, + "node_modules/jest-config/node_modules/jest-environment-node": { + "version": "30.3.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz", + "integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "30.3.0", + "@jest/fake-timers": "30.3.0", + "@jest/types": "30.3.0", + "@types/node": "*", + "jest-mock": "30.3.0", + "jest-util": "30.3.0", + "jest-validate": "30.3.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/jest-diff": { "version": "30.3.0", "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.3.0.tgz", @@ -25392,19 +25412,195 @@ } }, "node_modules/jest-environment-node": { - "version": "30.3.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz", - "integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==", + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.2.0.tgz", + "integrity": "sha512-ElU8v92QJ9UrYsKrxDIKCxu6PfNj4Hdcktcn0JX12zqNdqWHB0N+hwOnnBBXvjLd2vApZtuLUGs1QSY+MsXoNA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.3.0", - "@jest/fake-timers": "30.3.0", - "@jest/types": "30.3.0", + "@jest/environment": "30.2.0", + "@jest/fake-timers": "30.2.0", + "@jest/types": "30.2.0", "@types/node": "*", - "jest-mock": "30.3.0", - "jest-util": "30.3.0", - "jest-validate": "30.3.0" + "jest-mock": "30.2.0", + "jest-util": "30.2.0", + "jest-validate": "30.2.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/@jest/environment": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.2.0.tgz", + "integrity": "sha512-/QPTL7OBJQ5ac09UDRa3EQes4gt1FTEG/8jZ/4v5IVzx+Cv7dLxlVIvfvSVRiiX2drWyXeBjkMSR8hvOWSog5g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/fake-timers": "30.2.0", + "@jest/types": "30.2.0", + "@types/node": "*", + "jest-mock": "30.2.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/@jest/fake-timers": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.2.0.tgz", + "integrity": "sha512-HI3tRLjRxAbBy0VO8dqqm7Hb2mIa8d5bg/NJkyQcOk7V118ObQML8RC5luTF/Zsg4474a+gDvhce7eTnP4GhYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.2.0", + "@sinonjs/fake-timers": "^13.0.0", + "@types/node": "*", + "jest-message-util": "30.2.0", + "jest-mock": "30.2.0", + "jest-util": "30.2.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/@jest/types": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.2.0.tgz", + "integrity": "sha512-H9xg1/sfVvyfU7o3zMfBEjQ1gcsdeTMgqHoYdN79tuLqfTtuu7WckRA1R5whDwOzxaZAeMKTYWqP+WCAi0CHsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.1", + "@jest/schemas": "30.0.5", + "@types/istanbul-lib-coverage": "^2.0.6", + "@types/istanbul-reports": "^3.0.4", + "@types/node": "*", + "@types/yargs": "^17.0.33", + "chalk": "^4.1.2" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/@sinonjs/fake-timers": { + "version": "13.0.5", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz", + "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1" + } + }, + "node_modules/jest-environment-node/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-environment-node/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-environment-node/node_modules/jest-message-util": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.2.0.tgz", + "integrity": "sha512-y4DKFLZ2y6DxTWD4cDe07RglV88ZiNEdlRfGtqahfbIjfsw1nMCPx49Uev4IA/hWn3sDKyAnSPwoYSsAEdcimw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@jest/types": "30.2.0", + "@types/stack-utils": "^2.0.3", + "chalk": "^4.1.2", + "graceful-fs": "^4.2.11", + "micromatch": "^4.0.8", + "pretty-format": "30.2.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.6" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/jest-mock": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.2.0.tgz", + "integrity": "sha512-JNNNl2rj4b5ICpmAcq+WbLH83XswjPbjH4T7yvGzfAGCPh1rw+xVNbtk+FnRslvt9lkCcdn9i1oAoKUuFsOxRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.2.0", + "@types/node": "*", + "jest-util": "30.2.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/jest-util": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.2.0.tgz", + "integrity": "sha512-QKNsM0o3Xe6ISQU869e+DhG+4CK/48aHYdJZGlFQVTjnbvgpcKyxpzk29fGiO7i/J8VENZ+d2iGnSsvmuHywlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.2.0", + "@types/node": "*", + "chalk": "^4.1.2", + "ci-info": "^4.2.0", + "graceful-fs": "^4.2.11", + "picomatch": "^4.0.2" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/jest-validate": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.2.0.tgz", + "integrity": "sha512-FBGWi7dP2hpdi8nBoWxSsLvBFewKAg0+uSQwBaof4Y4DPgBabXgpSYC5/lR7VmnIlSpASmCi/ntRWPbv7089Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/get-type": "30.1.0", + "@jest/types": "30.2.0", + "camelcase": "^6.3.0", + "chalk": "^4.1.2", + "leven": "^3.1.0", + "pretty-format": "30.2.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/pretty-format": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", + "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.5", + "ansi-styles": "^5.2.0", + "react-is": "^18.3.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" @@ -25828,6 +26024,25 @@ "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, + "node_modules/jest-runner/node_modules/jest-environment-node": { + "version": "30.3.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz", + "integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "30.3.0", + "@jest/fake-timers": "30.3.0", + "@jest/types": "30.3.0", + "@types/node": "*", + "jest-mock": "30.3.0", + "jest-util": "30.3.0", + "jest-validate": "30.3.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/jest-runner/node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index db37399d6..746c91e68 100644 --- a/package.json +++ b/package.json @@ -204,6 +204,7 @@ "husky": "9.1.7", "jest": "30.3.0", "jest-environment-jsdom": "30.2.0", + "jest-environment-node": "30.2.0", "jest-preset-angular": "17.0.0", "nx": "23.1.1", "prettier": "3.9.6",