From 4ba493fd0c0c4699d6424421b264b1c2be8aa05e Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 1 Feb 2026 21:02:07 +0100 Subject: [PATCH] Introduce fast-redact Co-Authored-By: Valentin Zickner --- .../src/app/portfolio/portfolio.controller.ts | 2 + apps/api/src/helper/object.helper.spec.ts | 111 ++++++------------ apps/api/src/helper/object.helper.ts | 69 +++-------- .../redact-values-in-response.interceptor.ts | 42 ++----- ...orm-data-source-in-response.interceptor.ts | 19 +-- libs/common/src/lib/config.ts | 52 ++++++++ package-lock.json | 18 +++ package.json | 2 + 8 files changed, 151 insertions(+), 164 deletions(-) diff --git a/apps/api/src/app/portfolio/portfolio.controller.ts b/apps/api/src/app/portfolio/portfolio.controller.ts index a5a1d95ee..b8aefe0ac 100644 --- a/apps/api/src/app/portfolio/portfolio.controller.ts +++ b/apps/api/src/app/portfolio/portfolio.controller.ts @@ -195,11 +195,13 @@ export class PortfolioController { 'excludedAccountsAndActivities', 'fees', 'filteredValueInBaseCurrency', + 'fireWealth', 'grossPerformance', 'grossPerformanceWithCurrencyEffect', 'interestInBaseCurrency', 'items', 'liabilities', + 'liabilitiesInBaseCurrency', 'netPerformance', 'netPerformanceWithCurrencyEffect', 'totalBuy', diff --git a/apps/api/src/helper/object.helper.spec.ts b/apps/api/src/helper/object.helper.spec.ts index e1ec81b8f..3657dc554 100644 --- a/apps/api/src/helper/object.helper.spec.ts +++ b/apps/api/src/helper/object.helper.spec.ts @@ -1,4 +1,6 @@ -import { query, redactAttributes } from './object.helper'; +import { DEFAULT_REDACTED_PATHS } from '@ghostfolio/common/config'; + +import { query, redactPaths } from './object.helper'; describe('query', () => { it('should get market price from stock API response', () => { @@ -22,46 +24,38 @@ describe('query', () => { describe('redactAttributes', () => { it('should redact provided attributes', () => { - expect(redactAttributes({ object: {}, options: [] })).toStrictEqual({}); + expect(redactPaths({ object: {}, paths: [] })).toStrictEqual({}); - expect( - redactAttributes({ object: { value: 1000 }, options: [] }) - ).toStrictEqual({ value: 1000 }); + expect(redactPaths({ object: { value: 1000 }, paths: [] })).toStrictEqual({ + value: 1000 + }); expect( - redactAttributes({ + redactPaths({ object: { value: 1000 }, - options: [{ attribute: 'value', valueMap: { '*': null } }] + paths: ['value'] }) ).toStrictEqual({ value: null }); expect( - redactAttributes({ + redactPaths({ object: { value: 'abc' }, - options: [{ attribute: 'value', valueMap: { abc: 'xyz' } }] + paths: ['value'], + valueMap: { abc: 'xyz' } }) ).toStrictEqual({ value: 'xyz' }); expect( - redactAttributes({ + redactPaths({ object: { data: [{ value: 'a' }, { value: 'b' }] }, - options: [{ attribute: 'value', valueMap: { a: 1, b: 2 } }] + paths: ['data[*].value'], + valueMap: { a: 1, b: 2 } }) ).toStrictEqual({ data: [{ value: 1 }, { value: 2 }] }); - expect( - redactAttributes({ - object: { value1: 'a', value2: 'b' }, - options: [ - { attribute: 'value1', valueMap: { a: 'x' } }, - { attribute: 'value2', valueMap: { '*': 'y' } } - ] - }) - ).toStrictEqual({ value1: 'x', value2: 'y' }); - console.time('redactAttributes execution time'); expect( - redactAttributes({ + redactPaths({ object: { accounts: { '2e937c05-657c-4de9-8fb3-0813a2245f26': { @@ -150,7 +144,7 @@ describe('redactAttributes', () => { ], dataSource: 'EOD_HISTORICAL_DATA', dateOfFirstActivity: '2021-11-30T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: 2665.5, grossPerformancePercent: 0.3183066634822068, grossPerformancePercentWithCurrencyEffect: 0.3183066634822068, @@ -249,7 +243,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2018-09-30T22:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: 8689.05, grossPerformancePercent: 0.8594552890963852, grossPerformancePercentWithCurrencyEffect: 0.8594552890963852, @@ -300,7 +294,7 @@ describe('redactAttributes', () => { countries: [], dataSource: 'COINGECKO', dateOfFirstActivity: '2017-08-15T22:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: 34985.0332704, grossPerformancePercent: 17.4925166352, grossPerformancePercentWithCurrencyEffect: 17.4925166352, @@ -475,7 +469,7 @@ describe('redactAttributes', () => { ], dataSource: 'MANUAL', dateOfFirstActivity: '2021-03-31T22:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: 3533.389614611676, grossPerformancePercent: 0.27579517683678895, grossPerformancePercentWithCurrencyEffect: 0.458553421589667, @@ -527,7 +521,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2023-01-02T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: 5653.2, grossPerformancePercent: 0.7865431171216295, grossPerformancePercentWithCurrencyEffect: 0.7865431171216295, @@ -579,7 +573,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2017-01-02T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: 36920.500000005, grossPerformancePercent: 17.184314638161936, grossPerformancePercentWithCurrencyEffect: 17.184314638161936, @@ -661,7 +655,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2019-02-28T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: 5856.3, grossPerformancePercent: 0.8832083851170418, grossPerformancePercentWithCurrencyEffect: 0.8832083851170418, @@ -1061,7 +1055,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2018-02-28T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: 4534.60577952194, grossPerformancePercent: 0.3683200415015591, grossPerformancePercentWithCurrencyEffect: 0.5806366182968891, @@ -1343,7 +1337,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2021-08-18T22:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: 2281.298817228297, grossPerformancePercent: 0.3474381850624522, grossPerformancePercentWithCurrencyEffect: 0.28744846894552306, @@ -1461,7 +1455,7 @@ describe('redactAttributes', () => { assetClass: 'LIQUIDITY', assetSubClass: 'CASH', countries: [], - dividend: 0, + dividend: null, grossPerformance: 0, grossPerformancePercent: 0, grossPerformancePercentWithCurrencyEffect: 0, @@ -1564,34 +1558,7 @@ describe('redactAttributes', () => { currentNetWorth: null } }, - options: [ - 'balance', - 'balanceInBaseCurrency', - 'comment', - 'convertedBalance', - 'dividendInBaseCurrency', - 'fee', - 'feeInBaseCurrency', - 'grossPerformance', - 'grossPerformanceWithCurrencyEffect', - 'investment', - 'netPerformance', - 'netPerformanceWithCurrencyEffect', - 'quantity', - 'symbolMapping', - 'totalBalanceInBaseCurrency', - 'totalValueInBaseCurrency', - 'unitPrice', - 'value', - 'valueInBaseCurrency' - ].map((attribute) => { - return { - attribute, - valueMap: { - '*': null - } - }; - }) + paths: DEFAULT_REDACTED_PATHS }) ).toStrictEqual({ accounts: { @@ -1681,7 +1648,7 @@ describe('redactAttributes', () => { ], dataSource: 'EOD_HISTORICAL_DATA', dateOfFirstActivity: '2021-11-30T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 0.3183066634822068, grossPerformancePercentWithCurrencyEffect: 0.3183066634822068, @@ -1728,7 +1695,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2021-04-22T22:00:00.000Z', - dividend: 192, + dividend: null, grossPerformance: null, grossPerformancePercent: 0.3719230057375532, grossPerformancePercentWithCurrencyEffect: 0.2650716044872953, @@ -1780,7 +1747,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2018-09-30T22:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 0.8594552890963852, grossPerformancePercentWithCurrencyEffect: 0.8594552890963852, @@ -1831,7 +1798,7 @@ describe('redactAttributes', () => { countries: [], dataSource: 'COINGECKO', dateOfFirstActivity: '2017-08-15T22:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 17.4925166352, grossPerformancePercentWithCurrencyEffect: 17.4925166352, @@ -1882,7 +1849,7 @@ describe('redactAttributes', () => { countries: [], dataSource: 'MANUAL', dateOfFirstActivity: '2021-01-31T23:00:00.000Z', - dividend: 11.45, + dividend: null, grossPerformance: null, grossPerformancePercent: 0, grossPerformancePercentWithCurrencyEffect: -0.06153834320225245, @@ -1986,7 +1953,7 @@ describe('redactAttributes', () => { ], dataSource: 'MANUAL', dateOfFirstActivity: '2021-03-31T22:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 0.27579517683678895, grossPerformancePercentWithCurrencyEffect: 0.458553421589667, @@ -2038,7 +2005,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2023-01-02T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 0.7865431171216295, grossPerformancePercentWithCurrencyEffect: 0.7865431171216295, @@ -2090,7 +2057,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2017-01-02T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 17.184314638161936, grossPerformancePercentWithCurrencyEffect: 17.184314638161936, @@ -2172,7 +2139,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2019-02-28T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 0.8832083851170418, grossPerformancePercentWithCurrencyEffect: 0.8832083851170418, @@ -2567,7 +2534,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2018-02-28T23:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 0.3683200415015591, grossPerformancePercentWithCurrencyEffect: 0.5806366182968891, @@ -2846,7 +2813,7 @@ describe('redactAttributes', () => { ], dataSource: 'YAHOO', dateOfFirstActivity: '2021-08-18T22:00:00.000Z', - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 0.3474381850624522, grossPerformancePercentWithCurrencyEffect: 0.28744846894552306, @@ -2964,7 +2931,7 @@ describe('redactAttributes', () => { assetClass: 'LIQUIDITY', assetSubClass: 'CASH', countries: [], - dividend: 0, + dividend: null, grossPerformance: null, grossPerformancePercent: 0, grossPerformancePercentWithCurrencyEffect: 0, diff --git a/apps/api/src/helper/object.helper.ts b/apps/api/src/helper/object.helper.ts index 6bb6579d2..350d5fe04 100644 --- a/apps/api/src/helper/object.helper.ts +++ b/apps/api/src/helper/object.helper.ts @@ -1,6 +1,6 @@ -import { Big } from 'big.js'; +import fastRedact from 'fast-redact'; import jsonpath from 'jsonpath'; -import { cloneDeep, isArray, isObject } from 'lodash'; +import { cloneDeep, isObject } from 'lodash'; export function hasNotDefinedValuesInObject(aObject: Object): boolean { for (const key in aObject) { @@ -42,60 +42,29 @@ export function query({ return jsonpath.query(object, pathExpression); } -export function redactAttributes({ - isFirstRun = true, +export function redactPaths({ object, - options + paths, + valueMap }: { - isFirstRun?: boolean; object: any; - options: { attribute: string; valueMap: { [key: string]: any } }[]; + paths: fastRedact.RedactOptions['paths']; + valueMap?: { [key: string]: any }; }): any { - if (!object || !options?.length) { - return object; - } - - // Create deep clone - const redactedObject = isFirstRun - ? JSON.parse(JSON.stringify(object)) - : object; - - for (const option of options) { - if (redactedObject.hasOwnProperty(option.attribute)) { - if (option.valueMap['*'] || option.valueMap['*'] === null) { - redactedObject[option.attribute] = option.valueMap['*']; - } else if (option.valueMap[redactedObject[option.attribute]]) { - redactedObject[option.attribute] = - option.valueMap[redactedObject[option.attribute]]; - } - } else { - // If the attribute is not present on the current object, - // check if it exists on any nested objects - for (const property in redactedObject) { - if (isArray(redactedObject[property])) { - redactedObject[property] = redactedObject[property].map( - (currentObject) => { - return redactAttributes({ - options, - isFirstRun: false, - object: currentObject - }); - } - ); - } else if ( - isObject(redactedObject[property]) && - !(redactedObject[property] instanceof Big) - ) { - // Recursively call the function on the nested object - redactedObject[property] = redactAttributes({ - options, - isFirstRun: false, - object: redactedObject[property] - }); + const redact = fastRedact({ + paths, + censor: (value) => { + if (valueMap) { + if (valueMap[value]) { + return valueMap[value]; + } else { + return value; } + } else { + return null; } } - } + }); - return redactedObject; + return JSON.parse(redact(object)); } diff --git a/apps/api/src/interceptors/redact-values-in-response/redact-values-in-response.interceptor.ts b/apps/api/src/interceptors/redact-values-in-response/redact-values-in-response.interceptor.ts index 5ecf7c48d..60b994cac 100644 --- a/apps/api/src/interceptors/redact-values-in-response/redact-values-in-response.interceptor.ts +++ b/apps/api/src/interceptors/redact-values-in-response/redact-values-in-response.interceptor.ts @@ -1,5 +1,8 @@ -import { redactAttributes } from '@ghostfolio/api/helper/object.helper'; -import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config'; +import { redactPaths } from '@ghostfolio/api/helper/object.helper'; +import { + DEFAULT_REDACTED_PATHS, + HEADER_KEY_IMPERSONATION +} from '@ghostfolio/common/config'; import { hasReadRestrictedAccessPermission, isRestrictedView @@ -39,40 +42,9 @@ export class RedactValuesInResponseInterceptor implements NestInterceptor< }) || isRestrictedView(user) ) { - data = redactAttributes({ + data = redactPaths({ object: data, - options: [ - 'balance', - 'balanceInBaseCurrency', - 'comment', - 'convertedBalance', - 'dividendInBaseCurrency', - 'fee', - 'feeInBaseCurrency', - 'grossPerformance', - 'grossPerformanceWithCurrencyEffect', - 'interestInBaseCurrency', - 'investment', - 'netPerformance', - 'netPerformanceWithCurrencyEffect', - 'quantity', - 'symbolMapping', - 'totalBalanceInBaseCurrency', - 'totalDividendInBaseCurrency', - 'totalInterestInBaseCurrency', - 'totalValueInBaseCurrency', - 'unitPrice', - 'unitPriceInAssetProfileCurrency', - 'value', - 'valueInBaseCurrency' - ].map((attribute) => { - return { - attribute, - valueMap: { - '*': null - } - }; - }) + paths: DEFAULT_REDACTED_PATHS }); } diff --git a/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts b/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts index 9af256671..eaa6dd08c 100644 --- a/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts +++ b/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts @@ -1,4 +1,4 @@ -import { redactAttributes } from '@ghostfolio/api/helper/object.helper'; +import { redactPaths } from '@ghostfolio/api/helper/object.helper'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { encodeDataSource } from '@ghostfolio/common/helper'; @@ -58,13 +58,18 @@ export class TransformDataSourceInResponseInterceptor< } } - data = redactAttributes({ + data = redactPaths({ + valueMap, object: data, - options: [ - { - valueMap, - attribute: 'dataSource' - } + paths: [ + 'activities[*].SymbolProfile.dataSource', + 'benchmarks[*].dataSource', + 'fearAndGreedIndex.CRYPTOCURRENCIES.dataSource', + 'fearAndGreedIndex.STOCKS.dataSource', + 'holdings[*].dataSource', + 'items[*].dataSource', + 'SymbolProfile.dataSource', + 'watchlist[*].dataSource' ] }); } diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index a10a828e1..b558ccc42 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -78,6 +78,58 @@ export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY = 1; export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY = 1; export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000; +export const DEFAULT_REDACTED_PATHS = [ + 'accounts[*].balance', + 'accounts[*].valueInBaseCurrency', + 'activities[*].account.balance', + 'activities[*].account.comment', + 'activities[*].comment', + 'activities[*].fee', + 'activities[*].feeInAssetProfileCurrency', + 'activities[*].feeInBaseCurrency', + 'activities[*].quantity', + 'activities[*].SymbolProfile.symbolMapping', + 'activities[*].SymbolProfile.watchedByCount', + 'activities[*].value', + 'activities[*].valueInBaseCurrency', + 'balance', + 'balanceInBaseCurrency', + 'balances[*].account.balance', + 'balances[*].account.comment', + 'balances[*].value', + 'balances[*].valueInBaseCurrency', + 'comment', + 'dividendInBaseCurrency', + 'feeInBaseCurrency', + 'grossPerformance', + 'grossPerformanceWithCurrencyEffect', + 'historicalData[*].quantity', + 'holdings[*].dividend', + 'holdings[*].grossPerformance', + 'holdings[*].grossPerformanceWithCurrencyEffect', + 'holdings[*].holdings[*].valueInBaseCurrency', + 'holdings[*].investment', + 'holdings[*].netPerformance', + 'holdings[*].netPerformanceWithCurrencyEffect', + 'holdings[*].quantity', + 'holdings[*].valueInBaseCurrency', + 'interestInBaseCurrency', + 'investmentInBaseCurrencyWithCurrencyEffect', + 'netPerformance', + 'netPerformanceWithCurrencyEffect', + 'platforms[*].balance', + 'platforms[*].valueInBaseCurrency', + 'quantity', + 'SymbolProfile.symbolMapping', + 'SymbolProfile.watchedByCount', + 'totalBalanceInBaseCurrency', + 'totalDividendInBaseCurrency', + 'totalInterestInBaseCurrency', + 'totalValueInBaseCurrency', + 'value', + 'valueInBaseCurrency' +]; + // USX is handled separately export const DERIVED_CURRENCIES = [ { diff --git a/package-lock.json b/package-lock.json index b8d343edd..cbb1fb53d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -62,6 +62,7 @@ "dotenv": "17.2.3", "dotenv-expand": "12.0.3", "envalid": "8.1.1", + "fast-redact": "3.5.0", "fuse.js": "7.1.0", "google-spreadsheet": "3.2.0", "helmet": "7.0.0", @@ -122,6 +123,7 @@ "@storybook/angular": "10.1.10", "@trivago/prettier-plugin-sort-imports": "5.2.2", "@types/big.js": "6.2.2", + "@types/fast-redact": "3.0.4", "@types/google-spreadsheet": "3.1.5", "@types/jest": "30.0.0", "@types/jsonpath": "0.2.4", @@ -12936,6 +12938,13 @@ "@types/send": "*" } }, + "node_modules/@types/fast-redact": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/fast-redact/-/fast-redact-3.0.4.tgz", + "integrity": "sha512-tgGJaXucrCH4Yx2l/AI6e/JQksZhKGIQsVwBMTh+nxUhQDv5tXScTs5DHTw+qSKDXnHL2dTAh1e2rd5pcFQyNQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/geojson": { "version": "7946.0.16", "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", @@ -19940,6 +19949,15 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "license": "MIT" }, + "node_modules/fast-redact": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz", + "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/fast-safe-stringify": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", diff --git a/package.json b/package.json index 44df5228a..aee2de03f 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ "dotenv": "17.2.3", "dotenv-expand": "12.0.3", "envalid": "8.1.1", + "fast-redact": "3.5.0", "fuse.js": "7.1.0", "google-spreadsheet": "3.2.0", "helmet": "7.0.0", @@ -166,6 +167,7 @@ "@storybook/angular": "10.1.10", "@trivago/prettier-plugin-sort-imports": "5.2.2", "@types/big.js": "6.2.2", + "@types/fast-redact": "3.0.4", "@types/google-spreadsheet": "3.1.5", "@types/jest": "30.0.0", "@types/jsonpath": "0.2.4",