topaLE
/
ghostfolio
mirror of https://github.com/ghostfolio/ghostfolio
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Simplify recent market data gathering

task/simplify-recent-market-data-gathering
Thomas Kaul 4 days ago
parent
d8add6aada
commit
24c117720f
9 changed files with 249 additions and 20 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      apps/api/src/services/data-provider/coingecko/coingecko.service.ts
  2. 8
      apps/api/src/services/data-provider/data-provider.service.ts
  3. 7
      apps/api/src/services/data-provider/manual/manual.service.ts
  4. 3
      apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
  5. 5
      apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
  6. 2
      apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
  7. 5
      apps/api/src/services/queues/data-gathering/data-gathering.service.ts
  8. 233
      package-lock.json
  9. 1
      package.json

5
apps/api/src/services/data-provider/coingecko/coingecko.service.ts
View File

@ -18,6 +18,7 @@ import {
LookupResponse
} from '@ghostfolio/common/interfaces';
import { utc } from '@date-fns/utc';
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import {
AssetClass,
@ -147,7 +148,9 @@ export class CoinGeckoService implements DataProviderInterface, OnModuleInit {
} = {};
for (const [timestamp, marketPrice] of prices) {
result[format(fromUnixTime(timestamp / 1000), DATE_FORMAT)] = {
result[
format(fromUnixTime(timestamp / 1000), DATE_FORMAT, { in: utc })
] = {
marketPrice
};
}

8
apps/api/src/services/data-provider/data-provider.service.ts
View File

@ -35,6 +35,7 @@ import {
} from '@ghostfolio/common/interfaces';
import type { Granularity, UserWithSettings } from '@ghostfolio/common/types';
import { utc } from '@date-fns/utc';
import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { DataSource, MarketData, Prisma, SymbolProfile } from '@prisma/client';
import { Big } from 'big.js';
@ -494,8 +495,11 @@ export class DataProviderService implements OnModuleInit {
[date: string]: DataProviderHistoricalResponse;
} = {};
for (const date of eachDayOfInterval({ end: to, start: from })) {
data[format(date, DATE_FORMAT)] = { marketPrice: 100 };
for (const date of eachDayOfInterval(
{ end: to, start: from },
{ in: utc }
)) {
data[format(date, DATE_FORMAT, { in: utc })] = { marketPrice: 100 };
}
promises.push(

7
apps/api/src/services/data-provider/manual/manual.service.ts
View File

@ -24,6 +24,7 @@ import {
ScraperConfiguration
} from '@ghostfolio/common/interfaces';
import { utc } from '@date-fns/utc';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import * as cheerio from 'cheerio';
@ -96,11 +97,11 @@ export class ManualService implements DataProviderInterface {
let date = from;
while (isBefore(date, to)) {
historical[format(date, DATE_FORMAT)] = {
historical[format(date, DATE_FORMAT, { in: utc })] = {
marketPrice: defaultMarketPrice
};
date = addDays(date, 1);
date = addDays(date, 1, { in: utc });
}
return historical;
@ -114,7 +115,7 @@ export class ManualService implements DataProviderInterface {
});
return {
[format(getYesterday(), DATE_FORMAT)]: {
[format(getYesterday(), DATE_FORMAT, { in: utc })]: {
marketPrice: value
}
};

3
apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
View File

@ -17,6 +17,7 @@ import {
LookupResponse
} from '@ghostfolio/common/interfaces';
import { utc } from '@date-fns/utc';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import { format } from 'date-fns';
@ -66,7 +67,7 @@ export class RapidApiService implements DataProviderInterface {
if (fgi) {
return {
[format(getYesterday(), DATE_FORMAT)]: {
[format(getYesterday(), DATE_FORMAT, { in: utc })]: {
marketPrice: fgi.previousClose.value
}
};

5
apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
View File

@ -19,6 +19,7 @@ import {
LookupResponse
} from '@ghostfolio/common/interfaces';
import { utc } from '@date-fns/utc';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import { addDays, format, isSameDay } from 'date-fns';
@ -103,7 +104,7 @@ export class YahooFinanceService implements DataProviderInterface {
} = {};
for (const historicalItem of historicalResult) {
response[format(historicalItem.date, DATE_FORMAT)] = {
response[format(historicalItem.date, DATE_FORMAT, { in: utc })] = {
marketPrice: historicalItem.dividends
};
}
@ -174,7 +175,7 @@ export class YahooFinanceService implements DataProviderInterface {
: undefined);
if (marketPrice) {
response[format(date, DATE_FORMAT)] = { marketPrice };
response[format(date, DATE_FORMAT, { in: utc })] = { marketPrice };
}
}

2
apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
View File

@ -141,7 +141,7 @@ export class DataGatheringProcessor {
});
}
currentDate = addDays(currentDate, 1);
currentDate = addDays(currentDate, 1, { in: utc });
}
if (force) {

5
apps/api/src/services/queues/data-gathering/data-gathering.service.ts
View File

@ -27,6 +27,7 @@ import {
BenchmarkProperty
} from '@ghostfolio/common/interfaces';
import { utc } from '@date-fns/utc';
import { InjectQueue } from '@nestjs/bull';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { Prisma } from '@prisma/client';
@ -428,7 +429,9 @@ export class DataGatheringService {
await this.prismaService.marketData.groupBy({
by: ['dataSource', 'symbol'],
where: {
date: { gte: getStartOfUtcDate(subDays(new Date(), 1)) },
date: {
gte: getStartOfUtcDate(subDays(new Date(), 1, { in: utc }))
},
isCarriedForward: false,
state: 'CLOSE'
}

233
package-lock.json
View File

@ -160,6 +160,7 @@
"husky": "9.1.7",
"jest": "30.3.0",
"jest-environment-jsdom": "30.2.0",
"jest-environment-node": "30.2.0",
"jest-preset-angular": "17.0.0",
"nx": "23.1.1",
"prettier": "3.9.6",
@ -25012,6 +25013,25 @@
}
}
},
"node_modules/jest-config/node_modules/jest-environment-node": {
"version": "30.3.0",
"resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz",
"integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/environment": "30.3.0",
"@jest/fake-timers": "30.3.0",
"@jest/types": "30.3.0",
"@types/node": "*",
"jest-mock": "30.3.0",
"jest-util": "30.3.0",
"jest-validate": "30.3.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-diff": {
"version": "30.3.0",
"resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.3.0.tgz",
@ -25392,19 +25412,195 @@
}
},
"node_modules/jest-environment-node": {
"version": "30.3.0",
"resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz",
"integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==",
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.2.0.tgz",
"integrity": "sha512-ElU8v92QJ9UrYsKrxDIKCxu6PfNj4Hdcktcn0JX12zqNdqWHB0N+hwOnnBBXvjLd2vApZtuLUGs1QSY+MsXoNA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/environment": "30.3.0",
"@jest/fake-timers": "30.3.0",
"@jest/types": "30.3.0",
"@jest/environment": "30.2.0",
"@jest/fake-timers": "30.2.0",
"@jest/types": "30.2.0",
"@types/node": "*",
"jest-mock": "30.3.0",
"jest-util": "30.3.0",
"jest-validate": "30.3.0"
"jest-mock": "30.2.0",
"jest-util": "30.2.0",
"jest-validate": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/@jest/environment": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.2.0.tgz",
"integrity": "sha512-/QPTL7OBJQ5ac09UDRa3EQes4gt1FTEG/8jZ/4v5IVzx+Cv7dLxlVIvfvSVRiiX2drWyXeBjkMSR8hvOWSog5g==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/fake-timers": "30.2.0",
"@jest/types": "30.2.0",
"@types/node": "*",
"jest-mock": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/@jest/fake-timers": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.2.0.tgz",
"integrity": "sha512-HI3tRLjRxAbBy0VO8dqqm7Hb2mIa8d5bg/NJkyQcOk7V118ObQML8RC5luTF/Zsg4474a+gDvhce7eTnP4GhYw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/types": "30.2.0",
"@sinonjs/fake-timers": "^13.0.0",
"@types/node": "*",
"jest-message-util": "30.2.0",
"jest-mock": "30.2.0",
"jest-util": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/@jest/types": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/@jest/types/-/types-30.2.0.tgz",
"integrity": "sha512-H9xg1/sfVvyfU7o3zMfBEjQ1gcsdeTMgqHoYdN79tuLqfTtuu7WckRA1R5whDwOzxaZAeMKTYWqP+WCAi0CHsg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/pattern": "30.0.1",
"@jest/schemas": "30.0.5",
"@types/istanbul-lib-coverage": "^2.0.6",
"@types/istanbul-reports": "^3.0.4",
"@types/node": "*",
"@types/yargs": "^17.0.33",
"chalk": "^4.1.2"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/@sinonjs/fake-timers": {
"version": "13.0.5",
"resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz",
"integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==",
"dev": true,
"license": "BSD-3-Clause",
"dependencies": {
"@sinonjs/commons": "^3.0.1"
}
},
"node_modules/jest-environment-node/node_modules/ansi-styles": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
"integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/jest-environment-node/node_modules/camelcase": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
"integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/jest-environment-node/node_modules/jest-message-util": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.2.0.tgz",
"integrity": "sha512-y4DKFLZ2y6DxTWD4cDe07RglV88ZiNEdlRfGtqahfbIjfsw1nMCPx49Uev4IA/hWn3sDKyAnSPwoYSsAEdcimw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/code-frame": "^7.27.1",
"@jest/types": "30.2.0",
"@types/stack-utils": "^2.0.3",
"chalk": "^4.1.2",
"graceful-fs": "^4.2.11",
"micromatch": "^4.0.8",
"pretty-format": "30.2.0",
"slash": "^3.0.0",
"stack-utils": "^2.0.6"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/jest-mock": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.2.0.tgz",
"integrity": "sha512-JNNNl2rj4b5ICpmAcq+WbLH83XswjPbjH4T7yvGzfAGCPh1rw+xVNbtk+FnRslvt9lkCcdn9i1oAoKUuFsOxRw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/types": "30.2.0",
"@types/node": "*",
"jest-util": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/jest-util": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.2.0.tgz",
"integrity": "sha512-QKNsM0o3Xe6ISQU869e+DhG+4CK/48aHYdJZGlFQVTjnbvgpcKyxpzk29fGiO7i/J8VENZ+d2iGnSsvmuHywlA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/types": "30.2.0",
"@types/node": "*",
"chalk": "^4.1.2",
"ci-info": "^4.2.0",
"graceful-fs": "^4.2.11",
"picomatch": "^4.0.2"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/jest-validate": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.2.0.tgz",
"integrity": "sha512-FBGWi7dP2hpdi8nBoWxSsLvBFewKAg0+uSQwBaof4Y4DPgBabXgpSYC5/lR7VmnIlSpASmCi/ntRWPbv7089Pw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/get-type": "30.1.0",
"@jest/types": "30.2.0",
"camelcase": "^6.3.0",
"chalk": "^4.1.2",
"leven": "^3.1.0",
"pretty-format": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/pretty-format": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz",
"integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/schemas": "30.0.5",
"ansi-styles": "^5.2.0",
"react-is": "^18.3.1"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
@ -25828,6 +26024,25 @@
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-runner/node_modules/jest-environment-node": {
"version": "30.3.0",
"resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz",
"integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/environment": "30.3.0",
"@jest/fake-timers": "30.3.0",
"@jest/types": "30.3.0",
"@types/node": "*",
"jest-mock": "30.3.0",
"jest-util": "30.3.0",
"jest-validate": "30.3.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-runner/node_modules/source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",

1
package.json
View File

@ -204,6 +204,7 @@
"husky": "9.1.7",
"jest": "30.3.0",
"jest-environment-jsdom": "30.2.0",
"jest-environment-node": "30.2.0",
"jest-preset-angular": "17.0.0",
"nx": "23.1.1",
"prettier": "3.9.6",

Write Preview
Loading…
Cancel
Save