topaLE
/
ghostfolio
mirror of https://github.com/ghostfolio/ghostfolio
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Simplify recent market data gathering

task/simplify-recent-market-data-gathering
Thomas Kaul 4 days ago
parent
24c117720f
commit
9468841ae1
9 changed files with 48 additions and 243 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      apps/api/jest-environment-tz.js
  2. 3
      apps/api/src/app/admin/admin.controller.ts
  3. 4
      apps/api/src/services/data-provider/manual/manual.service.ts
  4. 9
      apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
  5. 3
      apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
  6. 28
      apps/api/src/services/queues/data-gathering/data-gathering.service.ts
  7. 5
      libs/common/src/lib/helper.ts
  8. 234
      package-lock.json
  9. 2
      package.json

3
apps/api/jest-environment-tz.js
View File

@ -7,12 +7,11 @@ class TimeZoneEnvironment extends NodeEnvironment {
constructor(config, context) {
super(config, context);
this.previousTimeZone = process.env.TZ;
this.timeZone = config.projectConfig.testEnvironmentOptions?.timeZone;
}
async setup() {
this.previousTimeZone = process.env.TZ;
if (this.timeZone) {
process.env.TZ = this.timeZone;
}

3
apps/api/src/app/admin/admin.controller.ts
View File

@ -37,6 +37,7 @@ import type {
RequestWithUser
} from '@ghostfolio/common/types';
import { utc } from '@date-fns/utc';
import {
Body,
Controller,
@ -204,7 +205,7 @@ export class AdminController {
@Param('dateString') dateString: string,
@Param('symbol') symbol: string
): Promise<MarketData> {
const date = parseISO(dateString);
const date = parseISO(dateString, { in: utc });
if (!isDate(date)) {
throw new HttpException(

4
apps/api/src/services/data-provider/manual/manual.service.ts
View File

@ -14,7 +14,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/sy
import {
DATE_FORMAT,
extractNumberFromString,
getYesterday
getStartOfUtcDateOfYesterday
} from '@ghostfolio/common/helper';
import {
DataProviderHistoricalResponse,
@ -115,7 +115,7 @@ export class ManualService implements DataProviderInterface {
});
return {
[format(getYesterday(), DATE_FORMAT, { in: utc })]: {
[format(getStartOfUtcDateOfYesterday(), DATE_FORMAT, { in: utc })]: {
marketPrice: value
}
};

9
apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
View File

@ -9,7 +9,10 @@ import {
} from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service';
import { ghostfolioFearAndGreedIndexSymbolStocks } from '@ghostfolio/common/config';
import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper';
import {
DATE_FORMAT,
getStartOfUtcDateOfYesterday
} from '@ghostfolio/common/helper';
import {
DataProviderHistoricalResponse,
DataProviderInfo,
@ -67,7 +70,9 @@ export class RapidApiService implements DataProviderInterface {
if (fgi) {
return {
[format(getYesterday(), DATE_FORMAT, { in: utc })]: {
[format(getStartOfUtcDateOfYesterday(), DATE_FORMAT, {
in: utc
})]: {
marketPrice: fgi.previousClose.value
}
};

3
apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
View File

@ -118,9 +118,10 @@ export class DataGatheringProcessor {
});
const data: Prisma.MarketDataUpdateInput[] = [];
const startOfUtcDateOfToday = getStartOfUtcDate(new Date());
let lastMarketPrice: number;
while (isBefore(currentDate, getStartOfUtcDate(new Date()))) {
while (isBefore(currentDate, startOfUtcDateOfToday)) {
const marketPriceOfDataProvider =
historicalData[assetProfileIdentifier]?.[
format(currentDate, DATE_FORMAT, { in: utc })

28
apps/api/src/services/queues/data-gathering/data-gathering.service.ts
View File

@ -20,7 +20,7 @@ import {
DATE_FORMAT,
getAssetProfileIdentifier,
getStartOfUtcDate,
resetHours
getStartOfUtcDateOfYesterday
} from '@ghostfolio/common/helper';
import {
AssetProfileIdentifier,
@ -323,29 +323,37 @@ export class DataGatheringService {
date,
symbol
}: { date: Date } & AssetProfileIdentifier) {
const startOfUtcDate = getStartOfUtcDate(date);
try {
const historicalData = await this.dataProviderService.getHistoricalRaw({
assetProfileIdentifiers: [{ dataSource, symbol }],
from: date,
to: date
from: startOfUtcDate,
to: startOfUtcDate
});
const marketPrice =
historicalData[getAssetProfileIdentifier({ dataSource, symbol })]?.[
format(date, DATE_FORMAT)
format(startOfUtcDate, DATE_FORMAT, { in: utc })
]?.marketPrice;
if (marketPrice) {
return await this.prismaService.marketData.upsert({
create: {
dataSource,
date,
marketPrice,
symbol,
date: startOfUtcDate,
isCarriedForward: false
},
update: { marketPrice, isCarriedForward: false },
where: { dataSource_date_symbol: { dataSource, date, symbol } }
where: {
dataSource_date_symbol: {
dataSource,
symbol,
date: startOfUtcDate
}
}
});
}
} catch (error) {
@ -383,7 +391,7 @@ export class DataGatheringService {
jobId: `${getAssetProfileIdentifier({
dataSource,
symbol
})}-${format(date, DATE_FORMAT)}`
})}-${format(date, DATE_FORMAT, { in: utc })}`
}
};
})
@ -430,7 +438,7 @@ export class DataGatheringService {
by: ['dataSource', 'symbol'],
where: {
date: {
gte: getStartOfUtcDate(subDays(new Date(), 1, { in: utc }))
gte: getStartOfUtcDateOfYesterday()
},
isCarriedForward: false,
state: 'CLOSE'
@ -457,7 +465,7 @@ export class DataGatheringService {
return {
dataSource,
symbol,
date: subDays(resetHours(new Date()), 7)
date: subDays(getStartOfUtcDate(new Date()), 7, { in: utc })
};
});
}
@ -525,7 +533,7 @@ export class DataGatheringService {
.map((symbolProfile) => {
return {
...symbolProfile,
date: subDays(resetHours(new Date()), 7)
date: subDays(getStartOfUtcDate(new Date()), 7, { in: utc })
};
});
}

5
libs/common/src/lib/helper.ts
View File

@ -1,3 +1,4 @@
import { utc } from '@date-fns/utc';
import { NumberParser } from '@internationalized/number';
import {
AccessType,
@ -465,6 +466,10 @@ export function getStartOfUtcDate(aDate: Date) {
return date;
}
export function getStartOfUtcDateOfYesterday() {
return subDays(getStartOfUtcDate(new Date()), 1, { in: utc });
}
export function getStringOrNull(aString: string | null | undefined) {
const trimmedString = aString?.trim();

234
package-lock.json
View File

@ -160,7 +160,7 @@
"husky": "9.1.7",
"jest": "30.3.0",
"jest-environment-jsdom": "30.2.0",
"jest-environment-node": "30.2.0",
"jest-environment-node": "30.3.0",
"jest-preset-angular": "17.0.0",
"nx": "23.1.1",
"prettier": "3.9.6",
@ -25013,25 +25013,6 @@
}
}
},
"node_modules/jest-config/node_modules/jest-environment-node": {
"version": "30.3.0",
"resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz",
"integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/environment": "30.3.0",
"@jest/fake-timers": "30.3.0",
"@jest/types": "30.3.0",
"@types/node": "*",
"jest-mock": "30.3.0",
"jest-util": "30.3.0",
"jest-validate": "30.3.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-diff": {
"version": "30.3.0",
"resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.3.0.tgz",
@ -25412,195 +25393,19 @@
}
},
"node_modules/jest-environment-node": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.2.0.tgz",
"integrity": "sha512-ElU8v92QJ9UrYsKrxDIKCxu6PfNj4Hdcktcn0JX12zqNdqWHB0N+hwOnnBBXvjLd2vApZtuLUGs1QSY+MsXoNA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/environment": "30.2.0",
"@jest/fake-timers": "30.2.0",
"@jest/types": "30.2.0",
"@types/node": "*",
"jest-mock": "30.2.0",
"jest-util": "30.2.0",
"jest-validate": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/@jest/environment": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.2.0.tgz",
"integrity": "sha512-/QPTL7OBJQ5ac09UDRa3EQes4gt1FTEG/8jZ/4v5IVzx+Cv7dLxlVIvfvSVRiiX2drWyXeBjkMSR8hvOWSog5g==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/fake-timers": "30.2.0",
"@jest/types": "30.2.0",
"@types/node": "*",
"jest-mock": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/@jest/fake-timers": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.2.0.tgz",
"integrity": "sha512-HI3tRLjRxAbBy0VO8dqqm7Hb2mIa8d5bg/NJkyQcOk7V118ObQML8RC5luTF/Zsg4474a+gDvhce7eTnP4GhYw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/types": "30.2.0",
"@sinonjs/fake-timers": "^13.0.0",
"@types/node": "*",
"jest-message-util": "30.2.0",
"jest-mock": "30.2.0",
"jest-util": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/@jest/types": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/@jest/types/-/types-30.2.0.tgz",
"integrity": "sha512-H9xg1/sfVvyfU7o3zMfBEjQ1gcsdeTMgqHoYdN79tuLqfTtuu7WckRA1R5whDwOzxaZAeMKTYWqP+WCAi0CHsg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/pattern": "30.0.1",
"@jest/schemas": "30.0.5",
"@types/istanbul-lib-coverage": "^2.0.6",
"@types/istanbul-reports": "^3.0.4",
"@types/node": "*",
"@types/yargs": "^17.0.33",
"chalk": "^4.1.2"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/@sinonjs/fake-timers": {
"version": "13.0.5",
"resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz",
"integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==",
"dev": true,
"license": "BSD-3-Clause",
"dependencies": {
"@sinonjs/commons": "^3.0.1"
}
},
"node_modules/jest-environment-node/node_modules/ansi-styles": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
"integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/jest-environment-node/node_modules/camelcase": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
"integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/jest-environment-node/node_modules/jest-message-util": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.2.0.tgz",
"integrity": "sha512-y4DKFLZ2y6DxTWD4cDe07RglV88ZiNEdlRfGtqahfbIjfsw1nMCPx49Uev4IA/hWn3sDKyAnSPwoYSsAEdcimw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/code-frame": "^7.27.1",
"@jest/types": "30.2.0",
"@types/stack-utils": "^2.0.3",
"chalk": "^4.1.2",
"graceful-fs": "^4.2.11",
"micromatch": "^4.0.8",
"pretty-format": "30.2.0",
"slash": "^3.0.0",
"stack-utils": "^2.0.6"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/jest-mock": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.2.0.tgz",
"integrity": "sha512-JNNNl2rj4b5ICpmAcq+WbLH83XswjPbjH4T7yvGzfAGCPh1rw+xVNbtk+FnRslvt9lkCcdn9i1oAoKUuFsOxRw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/types": "30.2.0",
"@types/node": "*",
"jest-util": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/jest-util": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.2.0.tgz",
"integrity": "sha512-QKNsM0o3Xe6ISQU869e+DhG+4CK/48aHYdJZGlFQVTjnbvgpcKyxpzk29fGiO7i/J8VENZ+d2iGnSsvmuHywlA==",
"version": "30.3.0",
"resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz",
"integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/types": "30.2.0",
"@jest/environment": "30.3.0",
"@jest/fake-timers": "30.3.0",
"@jest/types": "30.3.0",
"@types/node": "*",
"chalk": "^4.1.2",
"ci-info": "^4.2.0",
"graceful-fs": "^4.2.11",
"picomatch": "^4.0.2"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/jest-validate": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.2.0.tgz",
"integrity": "sha512-FBGWi7dP2hpdi8nBoWxSsLvBFewKAg0+uSQwBaof4Y4DPgBabXgpSYC5/lR7VmnIlSpASmCi/ntRWPbv7089Pw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/get-type": "30.1.0",
"@jest/types": "30.2.0",
"camelcase": "^6.3.0",
"chalk": "^4.1.2",
"leven": "^3.1.0",
"pretty-format": "30.2.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-environment-node/node_modules/pretty-format": {
"version": "30.2.0",
"resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz",
"integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/schemas": "30.0.5",
"ansi-styles": "^5.2.0",
"react-is": "^18.3.1"
"jest-mock": "30.3.0",
"jest-util": "30.3.0",
"jest-validate": "30.3.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
@ -26024,25 +25829,6 @@
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-runner/node_modules/jest-environment-node": {
"version": "30.3.0",
"resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.3.0.tgz",
"integrity": "sha512-4i6HItw/JSiJVsC5q0hnKIe/hbYfZLVG9YJ/0pU9Hz2n/9qZe3Rhn5s5CUZA5ORZlcdT/vmAXRMyONXJwPrmYQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jest/environment": "30.3.0",
"@jest/fake-timers": "30.3.0",
"@jest/types": "30.3.0",
"@types/node": "*",
"jest-mock": "30.3.0",
"jest-util": "30.3.0",
"jest-validate": "30.3.0"
},
"engines": {
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
}
},
"node_modules/jest-runner/node_modules/source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",

2
package.json
View File

@ -204,7 +204,7 @@
"husky": "9.1.7",
"jest": "30.3.0",
"jest-environment-jsdom": "30.2.0",
"jest-environment-node": "30.2.0",
"jest-environment-node": "30.3.0",
"jest-preset-angular": "17.0.0",
"nx": "23.1.1",
"prettier": "3.9.6",

Write Preview
Loading…
Cancel
Save