From 5f4cbe3af72a1cd3422be09f3ee07a0161085bfc Mon Sep 17 00:00:00 2001 From: Madhab Chandra Sahoo <30400985+madhab-chandra-sahoo@users.noreply.github.com> Date: Fri, 11 Oct 2024 23:29:08 +0530 Subject: [PATCH 1/8] Bugfix/considered language of user settings on login with Security Token (#3828) * Consider language of user settings on login with Security Token * Update changelog --- CHANGELOG.md | 6 ++++++ .../src/app/components/header/header.component.ts | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e66630572..d334d6b08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed + +- Considered the language of the user settings on login with _Security Token_ + ## 2.114.0 - 2024-10-10 ### Added diff --git a/apps/client/src/app/components/header/header.component.ts b/apps/client/src/app/components/header/header.component.ts index be90fbc8e..33069aa23 100644 --- a/apps/client/src/app/components/header/header.component.ts +++ b/apps/client/src/app/components/header/header.component.ts @@ -261,7 +261,18 @@ export class HeaderComponent implements OnChanges { this.settingsStorageService.getSetting(KEY_STAY_SIGNED_IN) === 'true' ); - this.router.navigate(['/']); + this.userService + .get() + .pipe(takeUntil(this.unsubscribeSubject)) + .subscribe((user) => { + const userLanguage = user?.settings?.language; + + if (userLanguage && document.documentElement.lang !== userLanguage) { + window.location.href = `../${userLanguage}`; + } else { + this.router.navigate(['/']); + } + }); } public ngOnDestroy() { From d158d0c326567600dbddfed0c90b1d18de33a28c Mon Sep 17 00:00:00 2001 From: Uday R <82779467+uday-rana@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:13:40 -0400 Subject: [PATCH 2/8] Feature/Extend tooltip in treemap chart component by name (#3907) * Extend tooltip in treemap chart component by name * Update changelog --- CHANGELOG.md | 4 ++++ .../treemap-chart/treemap-chart.component.ts | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d334d6b08..388fd9772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added the name to the tooltip of the chart of the holdings tab on the home page (experimental) + ### Fixed - Considered the language of the user settings on login with _Security Token_ diff --git a/libs/ui/src/lib/treemap-chart/treemap-chart.component.ts b/libs/ui/src/lib/treemap-chart/treemap-chart.component.ts index b278180ea..0e694f6dc 100644 --- a/libs/ui/src/lib/treemap-chart/treemap-chart.component.ts +++ b/libs/ui/src/lib/treemap-chart/treemap-chart.component.ts @@ -227,16 +227,24 @@ export class GfTreemapChartComponent }), callbacks: { label: (context) => { + const name = context.raw._data.name; + const symbol = context.raw._data.symbol; + if (context.raw._data.valueInBaseCurrency !== null) { const value = context.raw._data.valueInBaseCurrency; - return `${value.toLocaleString(this.locale, { - maximumFractionDigits: 2, - minimumFractionDigits: 2 - })} ${this.baseCurrency}`; + + return [ + `${name ?? symbol}`, + `${value.toLocaleString(this.locale, { + maximumFractionDigits: 2, + minimumFractionDigits: 2 + })} ${this.baseCurrency}` + ]; } else { const percentage = context.raw._data.allocationInPercentage * 100; - return `${percentage.toFixed(2)}%`; + + return [`${name ?? symbol}`, `${percentage.toFixed(2)}%`]; } }, title: () => { From 7a11bb93d5badddc69651b5d6fe8f5b20421c58c Mon Sep 17 00:00:00 2001 From: Dhaneshwari Tendle <110600266+dhaneshwaritendle@users.noreply.github.com> Date: Sat, 12 Oct 2024 00:41:39 +0530 Subject: [PATCH 3/8] Feature/Set up unit test that loads activity from exported json file (#3901) * Set up unit test that loads activity from exported json file * Update changelog --- CHANGELOG.md | 4 + .../portfolio-calculator-test-utils.ts | 6 + ...folio-calculator-novn-buy-and-sell.spec.ts | 512 +++++++++--------- test/import/ok-novn-buy-and-sell.json | 4 +- 4 files changed, 265 insertions(+), 261 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 388fd9772..d83247b12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the name to the tooltip of the chart of the holdings tab on the home page (experimental) +### Changed + +- Improved the portfolio unit tests to work with exported activity files + ### Fixed - Considered the language of the user settings on login with _Security Token_ diff --git a/apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts b/apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts index d458be708..217ec499b 100644 --- a/apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts +++ b/apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts @@ -1,3 +1,5 @@ +import { readFileSync } from 'fs'; + export const activityDummyData = { accountId: undefined, accountUserId: undefined, @@ -29,3 +31,7 @@ export const symbolProfileDummyData = { export const userDummyData = { id: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' }; + +export function loadActivityExportFile(filePath: string) { + return JSON.parse(readFileSync(filePath, 'utf8')).activities; +} diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts index db5aaf6bc..66cdb9e8e 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts @@ -1,259 +1,253 @@ -import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface'; -import { - activityDummyData, - symbolProfileDummyData, - userDummyData -} from '@ghostfolio/api/app/portfolio/calculator/portfolio-calculator-test-utils'; -import { - PerformanceCalculationType, - PortfolioCalculatorFactory -} from '@ghostfolio/api/app/portfolio/calculator/portfolio-calculator.factory'; -import { CurrentRateService } from '@ghostfolio/api/app/portfolio/current-rate.service'; -import { CurrentRateServiceMock } from '@ghostfolio/api/app/portfolio/current-rate.service.mock'; -import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service'; -import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; -import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; -import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; -import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; -import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; -import { parseDate } from '@ghostfolio/common/helper'; - -import { Big } from 'big.js'; -import { last } from 'lodash'; - -jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { - return { - // eslint-disable-next-line @typescript-eslint/naming-convention - CurrentRateService: jest.fn().mockImplementation(() => { - return CurrentRateServiceMock; - }) - }; -}); - -jest.mock( - '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', - () => { - return { - // eslint-disable-next-line @typescript-eslint/naming-convention - PortfolioSnapshotService: jest.fn().mockImplementation(() => { - return PortfolioSnapshotServiceMock; - }) - }; - } -); - -jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { - return { - // eslint-disable-next-line @typescript-eslint/naming-convention - RedisCacheService: jest.fn().mockImplementation(() => { - return RedisCacheServiceMock; - }) - }; -}); - -describe('PortfolioCalculator', () => { - let configurationService: ConfigurationService; - let currentRateService: CurrentRateService; - let exchangeRateDataService: ExchangeRateDataService; - let portfolioCalculatorFactory: PortfolioCalculatorFactory; - let portfolioSnapshotService: PortfolioSnapshotService; - let redisCacheService: RedisCacheService; - - beforeEach(() => { - configurationService = new ConfigurationService(); - - currentRateService = new CurrentRateService(null, null, null, null); - - exchangeRateDataService = new ExchangeRateDataService( - null, - null, - null, - null - ); - - portfolioSnapshotService = new PortfolioSnapshotService(null); - - redisCacheService = new RedisCacheService(null, null); - - portfolioCalculatorFactory = new PortfolioCalculatorFactory( - configurationService, - currentRateService, - exchangeRateDataService, - portfolioSnapshotService, - redisCacheService - ); - }); - - describe('get current positions', () => { - it.only('with NOVN.SW buy and sell', async () => { - jest.useFakeTimers().setSystemTime(parseDate('2022-04-11').getTime()); - - const activities: Activity[] = [ - { - ...activityDummyData, - date: new Date('2022-03-07'), - fee: 0, - quantity: 2, - SymbolProfile: { - ...symbolProfileDummyData, - currency: 'CHF', - dataSource: 'YAHOO', - name: 'Novartis AG', - symbol: 'NOVN.SW' - }, - type: 'BUY', - unitPrice: 75.8 - }, - { - ...activityDummyData, - date: new Date('2022-04-08'), - fee: 0, - quantity: 2, - SymbolProfile: { - ...symbolProfileDummyData, - currency: 'CHF', - dataSource: 'YAHOO', - name: 'Novartis AG', - symbol: 'NOVN.SW' - }, - type: 'SELL', - unitPrice: 85.73 - } - ]; - - const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ - activities, - calculationType: PerformanceCalculationType.TWR, - currency: 'CHF', - userId: userDummyData.id - }); - - const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); - - const investments = portfolioCalculator.getInvestments(); - - const investmentsByMonth = portfolioCalculator.getInvestmentsByGroup({ - data: portfolioSnapshot.historicalData, - groupBy: 'month' - }); - - expect(portfolioSnapshot.historicalData[0]).toEqual({ - date: '2022-03-06', - investmentValueWithCurrencyEffect: 0, - netPerformance: 0, - netPerformanceInPercentage: 0, - netPerformanceInPercentageWithCurrencyEffect: 0, - netPerformanceWithCurrencyEffect: 0, - netWorth: 0, - totalAccountBalance: 0, - totalInvestment: 0, - totalInvestmentValueWithCurrencyEffect: 0, - value: 0, - valueWithCurrencyEffect: 0 - }); - - expect(portfolioSnapshot.historicalData[1]).toEqual({ - date: '2022-03-07', - investmentValueWithCurrencyEffect: 151.6, - netPerformance: 0, - netPerformanceInPercentage: 0, - netPerformanceInPercentageWithCurrencyEffect: 0, - netPerformanceWithCurrencyEffect: 0, - netWorth: 151.6, - totalAccountBalance: 0, - totalInvestment: 151.6, - totalInvestmentValueWithCurrencyEffect: 151.6, - value: 151.6, - valueWithCurrencyEffect: 151.6 - }); - - expect( - portfolioSnapshot.historicalData[ - portfolioSnapshot.historicalData.length - 1 - ] - ).toEqual({ - date: '2022-04-11', - investmentValueWithCurrencyEffect: 0, - netPerformance: 19.86, - netPerformanceInPercentage: 0.13100263852242744, - netPerformanceInPercentageWithCurrencyEffect: 0.13100263852242744, - netPerformanceWithCurrencyEffect: 19.86, - netWorth: 0, - totalAccountBalance: 0, - totalInvestment: 0, - totalInvestmentValueWithCurrencyEffect: 0, - value: 0, - valueWithCurrencyEffect: 0 - }); - - expect(portfolioSnapshot).toMatchObject({ - currentValueInBaseCurrency: new Big('0'), - errors: [], - hasErrors: false, - positions: [ - { - averagePrice: new Big('0'), - currency: 'CHF', - dataSource: 'YAHOO', - dividend: new Big('0'), - dividendInBaseCurrency: new Big('0'), - fee: new Big('0'), - feeInBaseCurrency: new Big('0'), - firstBuyDate: '2022-03-07', - grossPerformance: new Big('19.86'), - grossPerformancePercentage: new Big('0.13100263852242744063'), - grossPerformancePercentageWithCurrencyEffect: new Big( - '0.13100263852242744063' - ), - grossPerformanceWithCurrencyEffect: new Big('19.86'), - investment: new Big('0'), - investmentWithCurrencyEffect: new Big('0'), - netPerformance: new Big('19.86'), - netPerformancePercentage: new Big('0.13100263852242744063'), - netPerformancePercentageWithCurrencyEffectMap: { - max: new Big('0.13100263852242744063') - }, - netPerformanceWithCurrencyEffectMap: { - max: new Big('19.86') - }, - marketPrice: 87.8, - marketPriceInBaseCurrency: 87.8, - quantity: new Big('0'), - symbol: 'NOVN.SW', - tags: [], - timeWeightedInvestment: new Big('151.6'), - timeWeightedInvestmentWithCurrencyEffect: new Big('151.6'), - transactionCount: 2, - valueInBaseCurrency: new Big('0') - } - ], - totalFeesWithCurrencyEffect: new Big('0'), - totalInterestWithCurrencyEffect: new Big('0'), - totalInvestment: new Big('0'), - totalInvestmentWithCurrencyEffect: new Big('0'), - totalLiabilitiesWithCurrencyEffect: new Big('0'), - totalValuablesWithCurrencyEffect: new Big('0') - }); - - expect(last(portfolioSnapshot.historicalData)).toMatchObject( - expect.objectContaining({ - netPerformance: 19.86, - netPerformanceInPercentage: 0.13100263852242744063, - netPerformanceInPercentageWithCurrencyEffect: 0.13100263852242744063, - netPerformanceWithCurrencyEffect: 19.86, - totalInvestmentValueWithCurrencyEffect: 0 - }) - ); - - expect(investments).toEqual([ - { date: '2022-03-07', investment: new Big('151.6') }, - { date: '2022-04-08', investment: new Big('0') } - ]); - - expect(investmentsByMonth).toEqual([ - { date: '2022-03-01', investment: 151.6 }, - { date: '2022-04-01', investment: -151.6 } - ]); - }); - }); -}); +import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto'; +import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface'; +import { + activityDummyData, + loadActivityExportFile, + symbolProfileDummyData, + userDummyData +} from '@ghostfolio/api/app/portfolio/calculator/portfolio-calculator-test-utils'; +import { + PerformanceCalculationType, + PortfolioCalculatorFactory +} from '@ghostfolio/api/app/portfolio/calculator/portfolio-calculator.factory'; +import { CurrentRateService } from '@ghostfolio/api/app/portfolio/current-rate.service'; +import { CurrentRateServiceMock } from '@ghostfolio/api/app/portfolio/current-rate.service.mock'; +import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service'; +import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; +import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; +import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; +import { parseDate } from '@ghostfolio/common/helper'; + +import { Big } from 'big.js'; +import { last } from 'lodash'; +import { join } from 'path'; + +jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + CurrentRateService: jest.fn().mockImplementation(() => { + return CurrentRateServiceMock; + }) + }; +}); + +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + +jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + RedisCacheService: jest.fn().mockImplementation(() => { + return RedisCacheServiceMock; + }) + }; +}); + +describe('PortfolioCalculator', () => { + let activityDtos: CreateOrderDto[]; + + let configurationService: ConfigurationService; + let currentRateService: CurrentRateService; + let exchangeRateDataService: ExchangeRateDataService; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; + let redisCacheService: RedisCacheService; + + beforeAll(() => { + activityDtos = loadActivityExportFile( + join( + __dirname, + '../../../../../../../test/import/ok-novn-buy-and-sell.json' + ) + ); + }); + + beforeEach(() => { + configurationService = new ConfigurationService(); + + currentRateService = new CurrentRateService(null, null, null, null); + + exchangeRateDataService = new ExchangeRateDataService( + null, + null, + null, + null + ); + + portfolioSnapshotService = new PortfolioSnapshotService(null); + + redisCacheService = new RedisCacheService(null, null); + + portfolioCalculatorFactory = new PortfolioCalculatorFactory( + configurationService, + currentRateService, + exchangeRateDataService, + portfolioSnapshotService, + redisCacheService + ); + }); + + describe('get current positions', () => { + it.only('with NOVN.SW buy and sell', async () => { + jest.useFakeTimers().setSystemTime(parseDate('2022-04-11').getTime()); + + const activities: Activity[] = activityDtos.map((activity) => ({ + ...activityDummyData, + ...activity, + date: parseDate(activity.date), + SymbolProfile: { + ...symbolProfileDummyData, + currency: activity.currency, + dataSource: activity.dataSource, + name: 'Novartis AG', + symbol: activity.symbol + } + })); + + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ + activities, + calculationType: PerformanceCalculationType.TWR, + currency: 'CHF', + userId: userDummyData.id + }); + + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); + + const investments = portfolioCalculator.getInvestments(); + + const investmentsByMonth = portfolioCalculator.getInvestmentsByGroup({ + data: portfolioSnapshot.historicalData, + groupBy: 'month' + }); + + expect(portfolioSnapshot.historicalData[0]).toEqual({ + date: '2022-03-06', + investmentValueWithCurrencyEffect: 0, + netPerformance: 0, + netPerformanceInPercentage: 0, + netPerformanceInPercentageWithCurrencyEffect: 0, + netPerformanceWithCurrencyEffect: 0, + netWorth: 0, + totalAccountBalance: 0, + totalInvestment: 0, + totalInvestmentValueWithCurrencyEffect: 0, + value: 0, + valueWithCurrencyEffect: 0 + }); + + expect(portfolioSnapshot.historicalData[1]).toEqual({ + date: '2022-03-07', + investmentValueWithCurrencyEffect: 151.6, + netPerformance: 0, + netPerformanceInPercentage: 0, + netPerformanceInPercentageWithCurrencyEffect: 0, + netPerformanceWithCurrencyEffect: 0, + netWorth: 151.6, + totalAccountBalance: 0, + totalInvestment: 151.6, + totalInvestmentValueWithCurrencyEffect: 151.6, + value: 151.6, + valueWithCurrencyEffect: 151.6 + }); + + expect( + portfolioSnapshot.historicalData[ + portfolioSnapshot.historicalData.length - 1 + ] + ).toEqual({ + date: '2022-04-11', + investmentValueWithCurrencyEffect: 0, + netPerformance: 19.86, + netPerformanceInPercentage: 0.13100263852242744, + netPerformanceInPercentageWithCurrencyEffect: 0.13100263852242744, + netPerformanceWithCurrencyEffect: 19.86, + netWorth: 0, + totalAccountBalance: 0, + totalInvestment: 0, + totalInvestmentValueWithCurrencyEffect: 0, + value: 0, + valueWithCurrencyEffect: 0 + }); + + expect(portfolioSnapshot).toMatchObject({ + currentValueInBaseCurrency: new Big('0'), + errors: [], + hasErrors: false, + positions: [ + { + averagePrice: new Big('0'), + currency: 'CHF', + dataSource: 'YAHOO', + dividend: new Big('0'), + dividendInBaseCurrency: new Big('0'), + fee: new Big('0'), + feeInBaseCurrency: new Big('0'), + firstBuyDate: '2022-03-07', + grossPerformance: new Big('19.86'), + grossPerformancePercentage: new Big('0.13100263852242744063'), + grossPerformancePercentageWithCurrencyEffect: new Big( + '0.13100263852242744063' + ), + grossPerformanceWithCurrencyEffect: new Big('19.86'), + investment: new Big('0'), + investmentWithCurrencyEffect: new Big('0'), + netPerformance: new Big('19.86'), + netPerformancePercentage: new Big('0.13100263852242744063'), + netPerformancePercentageWithCurrencyEffectMap: { + max: new Big('0.13100263852242744063') + }, + netPerformanceWithCurrencyEffectMap: { + max: new Big('19.86') + }, + marketPrice: 87.8, + marketPriceInBaseCurrency: 87.8, + quantity: new Big('0'), + symbol: 'NOVN.SW', + tags: [], + timeWeightedInvestment: new Big('151.6'), + timeWeightedInvestmentWithCurrencyEffect: new Big('151.6'), + transactionCount: 2, + valueInBaseCurrency: new Big('0') + } + ], + totalFeesWithCurrencyEffect: new Big('0'), + totalInterestWithCurrencyEffect: new Big('0'), + totalInvestment: new Big('0'), + totalInvestmentWithCurrencyEffect: new Big('0'), + totalLiabilitiesWithCurrencyEffect: new Big('0'), + totalValuablesWithCurrencyEffect: new Big('0') + }); + + expect(last(portfolioSnapshot.historicalData)).toMatchObject( + expect.objectContaining({ + netPerformance: 19.86, + netPerformanceInPercentage: 0.13100263852242744063, + netPerformanceInPercentageWithCurrencyEffect: 0.13100263852242744063, + netPerformanceWithCurrencyEffect: 19.86, + totalInvestmentValueWithCurrencyEffect: 0 + }) + ); + + expect(investments).toEqual([ + { date: '2022-03-07', investment: new Big('151.6') }, + { date: '2022-04-08', investment: new Big('0') } + ]); + + expect(investmentsByMonth).toEqual([ + { date: '2022-03-01', investment: 151.6 }, + { date: '2022-04-01', investment: -151.6 } + ]); + }); + }); +}); diff --git a/test/import/ok-novn-buy-and-sell.json b/test/import/ok-novn-buy-and-sell.json index b8a62279d..b7ab6aee1 100644 --- a/test/import/ok-novn-buy-and-sell.json +++ b/test/import/ok-novn-buy-and-sell.json @@ -11,7 +11,7 @@ "unitPrice": 85.73, "currency": "CHF", "dataSource": "YAHOO", - "date": "2022-04-07T22:00:00.000Z", + "date": "2022-04-08T00:00:00.000Z", "symbol": "NOVN.SW" }, { @@ -21,7 +21,7 @@ "unitPrice": 75.8, "currency": "CHF", "dataSource": "YAHOO", - "date": "2022-03-06T23:00:00.000Z", + "date": "2022-03-07T00:00:00.000Z", "symbol": "NOVN.SW" } ] From c44da0bce3934aa3448b96b0442d013337506267 Mon Sep 17 00:00:00 2001 From: ceroma <678940+ceroma@users.noreply.github.com> Date: Sat, 12 Oct 2024 04:04:58 -0300 Subject: [PATCH 4/8] Feature/expose portfolio snapshot computation timeout as environment variable (#3894) * Expose portfolio snapshot computation timeout as environment variable * Update changelog --- CHANGELOG.md | 1 + .../configuration/configuration.service.ts | 4 ++++ .../portfolio-snapshot.module.ts | 14 ++++++++++++-- libs/common/src/lib/config.ts | 1 + 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d83247b12..84b641513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Exposed the timeout of the portfolio snapshot computation as an environment variable (`PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT`) - Improved the portfolio unit tests to work with exported activity files ### Fixed diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts index cca393a2a..dafd4803c 100644 --- a/apps/api/src/services/configuration/configuration.service.ts +++ b/apps/api/src/services/configuration/configuration.service.ts @@ -4,6 +4,7 @@ import { DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE, DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA, DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT, + DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT, DEFAULT_ROOT_URL } from '@ghostfolio/common/config'; @@ -59,6 +60,9 @@ export class ConfigurationService { PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT: num({ default: DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT }), + PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: num({ + default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT + }), REDIS_DB: num({ default: 0 }), REDIS_HOST: str({ default: 'localhost' }), REDIS_PASSWORD: str({ default: '' }), diff --git a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts index 620feda53..058d971d8 100644 --- a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts +++ b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts @@ -8,7 +8,10 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data- import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; -import { PORTFOLIO_SNAPSHOT_QUEUE } from '@ghostfolio/common/config'; +import { + DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT, + PORTFOLIO_SNAPSHOT_QUEUE +} from '@ghostfolio/common/config'; import { BullModule } from '@nestjs/bull'; import { Module } from '@nestjs/common'; @@ -20,7 +23,14 @@ import { PortfolioSnapshotProcessor } from './portfolio-snapshot.processor'; imports: [ AccountBalanceModule, BullModule.registerQueue({ - name: PORTFOLIO_SNAPSHOT_QUEUE + name: PORTFOLIO_SNAPSHOT_QUEUE, + settings: { + lockDuration: parseInt( + process.env.PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT ?? + DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT.toString(), + 10 + ) + } }), ConfigurationModule, DataProviderModule, diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 19ec965fa..4a5079d28 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -51,6 +51,7 @@ export const DEFAULT_PAGE_SIZE = 50; export const DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE = 1; export const DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA = 1; export const DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT = 1; +export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000; export const DEFAULT_ROOT_URL = 'https://localhost:4200'; // USX is handled separately From b0cfb2b6fd4c90bf4bd6062489a234ac18829151 Mon Sep 17 00:00:00 2001 From: Dan Date: Sat, 12 Oct 2024 17:08:41 +0200 Subject: [PATCH 5/8] Add Datagathering process for missing values only --- apps/api/src/app/admin/admin.controller.ts | 17 +- .../data-gathering.processor.ts | 159 +++++++++++++++++- .../data-gathering/data-gathering.service.ts | 52 +++++- .../asset-profile-dialog.component.ts | 10 ++ .../asset-profile-dialog.html | 13 ++ apps/client/src/app/services/admin.service.ts | 16 ++ libs/common/src/lib/config.ts | 12 ++ 7 files changed, 271 insertions(+), 8 deletions(-) diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 0cf8d78bd..2c469612e 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -158,7 +158,22 @@ export class AdminController { @Param('dataSource') dataSource: DataSource, @Param('symbol') symbol: string ): Promise { - this.dataGatheringService.gatherSymbol({ dataSource, symbol }); + await this.dataGatheringService.gatherSymbol({ dataSource, symbol }); + + return; + } + + @Post('gatherMissing/:dataSource/:symbol') + @UseGuards(AuthGuard('jwt'), HasPermissionGuard) + @HasPermission(permissions.accessAdminControl) + public async gatherSymbolMissingOnly( + @Param('dataSource') dataSource: DataSource, + @Param('symbol') symbol: string + ): Promise { + await this.dataGatheringService.gatherSymbolMissingOnly({ + dataSource, + symbol + }); return; } diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts index 2745aa288..d925b2294 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts @@ -1,20 +1,25 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; -import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; +import { + IDataGatheringItem, + IDataProviderHistoricalResponse +} from '@ghostfolio/api/services/interfaces/interfaces'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { DATA_GATHERING_QUEUE, DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE, DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA, GATHER_ASSET_PROFILE_PROCESS, - GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME + GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, + GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME } from '@ghostfolio/common/config'; import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { Process, Processor } from '@nestjs/bull'; import { Injectable, Logger } from '@nestjs/common'; -import { Prisma } from '@prisma/client'; +import { DataSource, Prisma } from '@prisma/client'; import { Job } from 'bull'; +import { isNumber } from 'class-validator'; import { addDays, format, @@ -22,7 +27,9 @@ import { getMonth, getYear, isBefore, - parseISO + parseISO, + eachDayOfInterval, + isEqual } from 'date-fns'; import { DataGatheringService } from './data-gathering.service'; @@ -150,4 +157,148 @@ export class DataGatheringProcessor { throw new Error(error); } } + @Process({ + concurrency: parseInt( + process.env.PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA ?? + DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA.toString(), + 10 + ), + name: GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME + }) + public async gatherMissingHistoricalMarketData(job: Job) { + try { + const { dataSource, date, symbol } = job.data; + + Logger.log( + `Historical market data gathering for missing values has been started for ${symbol} (${dataSource}) at ${format( + date, + DATE_FORMAT + )}`, + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` + ); + const entries = await this.marketDataService.marketDataItems({ + where: { + AND: { + symbol: { + equals: symbol + }, + dataSource: { + equals: dataSource + } + } + }, + orderBy: { + date: 'asc' + }, + take: 1 + }); + const firstEntry = entries[0]; + const marketData = await this.marketDataService + .getRange({ + assetProfileIdentifiers: [{ dataSource, symbol }], + dateQuery: { + gte: addDays(firstEntry.date, -10) + } + }) + .then((md) => md.map((m) => m.date)); + + let dates = eachDayOfInterval( + { + start: firstEntry.date, + end: new Date() + }, + { + step: 1 + } + ); + dates = dates.filter((d) => !marketData.some((md) => isEqual(md,d))); + + const historicalData = await this.dataProviderService.getHistoricalRaw({ + dataGatheringItems: [{ dataSource, symbol }], + from: firstEntry.date, + to: new Date() + }); + + const data: Prisma.MarketDataUpdateInput[] = + this.mapToMarketUpsertDataInputs( + dates, + historicalData, + symbol, + dataSource + ); + + await this.marketDataService.updateMany({ data }); + + Logger.log( + `Historical market data gathering for missing values has been completed for ${symbol} (${dataSource}) at ${format( + date, + DATE_FORMAT + )}`, + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` + ); + } catch (error) { + Logger.error( + error, + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` + ); + + throw new Error(error); + } + } + + private mapToMarketUpsertDataInputs( + missingMarketData: Date[], + historicalData: Record< + string, + Record + >, + symbol: string, + dataSource: DataSource + ): Prisma.MarketDataUpdateInput[] { + return missingMarketData.map((date) => { + if ( + isNumber( + historicalData[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice + ) + ) { + return { + date, + symbol, + dataSource, + marketPrice: + historicalData[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice + }; + } else { + let earlierDate = date; + let index = 0; + while ( + !isNumber( + historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)] + ?.marketPrice + ) + ) { + earlierDate = addDays(earlierDate, -1); + index++; + if (index > 10) { + break; + } + } + if ( + isNumber( + historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)] + ?.marketPrice + ) + ) { + return { + date, + symbol, + dataSource, + marketPrice: + historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)] + ?.marketPrice + }; + } + } + }); + } } diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts index a66e05b72..24b174785 100644 --- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -13,6 +13,8 @@ import { DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS, + GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, + GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS, PROPERTY_BENCHMARKS } from '@ghostfolio/common/config'; import { @@ -28,7 +30,6 @@ import { import { InjectQueue } from '@nestjs/bull'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { DataSource } from '@prisma/client'; -import AwaitLock from 'await-lock'; import { JobOptions, Queue } from 'bull'; import { format, min, subDays, subYears } from 'date-fns'; import { isEmpty } from 'lodash'; @@ -48,8 +49,6 @@ export class DataGatheringService { private readonly symbolProfileService: SymbolProfileService ) {} - lock = new AwaitLock(); - public async addJobToQueue({ data, name, @@ -114,6 +113,24 @@ export class DataGatheringService { }); } + public async gatherSymbolMissingOnly({ + dataSource, + symbol + }: AssetProfileIdentifier) { + const dataGatheringItems = (await this.getSymbolsMax()).filter( + (dataGatheringItem) => { + return ( + dataGatheringItem.dataSource === dataSource && + dataGatheringItem.symbol === symbol + ); + } + ); + await this.gatherMissingDataSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); + } + public async gatherSymbolForDate({ dataSource, date, @@ -296,6 +313,35 @@ export class DataGatheringService { ); } + public async gatherMissingDataSymbols({ + dataGatheringItems, + priority + }: { + dataGatheringItems: IDataGatheringItem[]; + priority: number; + }) { + await this.addJobsToQueue( + dataGatheringItems.map(({ dataSource, date, symbol }) => { + return { + data: { + dataSource, + date, + symbol + }, + name: GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, + opts: { + ...GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS, + priority, + jobId: `${getAssetProfileIdentifier({ + dataSource, + symbol + })}-missing-${format(date, DATE_FORMAT)}` + } + }; + }) + ); + } + public async getAllAssetProfileIdentifiers(): Promise< AssetProfileIdentifier[] > { diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts index d79ba44f3..4eb65a70d 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts @@ -223,6 +223,16 @@ export class AssetProfileDialog implements OnDestroy, OnInit { .subscribe(() => {}); } + public onGatherSymbolMissingOnly({ + dataSource, + symbol + }: AssetProfileIdentifier) { + this.adminService + .gatherSymbolMissingOnly({ dataSource, symbol }) + .pipe(takeUntil(this.unsubscribeSubject)) + .subscribe(() => {}); + } + public onImportHistoricalData() { try { const marketData = csvToJson( diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html index 76bce400b..6ea0a1a03 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html @@ -31,6 +31,19 @@ > Gather Historical Data +