Introduce fast-redact

Co-Authored-By: Valentin Zickner <valentin@coderworks.de>

apps/api/src/app/portfolio/portfolio.controller.ts

@@ -195,11 +195,13 @@ export class PortfolioController {
         'excludedAccountsAndActivities',
         'fees',
         'filteredValueInBaseCurrency',
+        'fireWealth',
         'grossPerformance',
         'grossPerformanceWithCurrencyEffect',
         'interestInBaseCurrency',
         'items',
         'liabilities',
+        'liabilitiesInBaseCurrency',
         'netPerformance',
         'netPerformanceWithCurrencyEffect',
         'totalBuy',

apps/api/src/helper/object.helper.spec.ts

@@ -1,4 +1,6 @@
-import { query, redactAttributes } from './object.helper';
+import { DEFAULT_REDACTED_PATHS } from '@ghostfolio/common/config';
+
+import { query, redactPaths } from './object.helper';
 
 describe('query', () => {
   it('should get market price from stock API response', () => {
@@ -22,46 +24,38 @@ describe('query', () => {
 
 describe('redactAttributes', () => {
   it('should redact provided attributes', () => {
-    expect(redactAttributes({ object: {}, options: [] })).toStrictEqual({});
+    expect(redactPaths({ object: {}, paths: [] })).toStrictEqual({});
 
-    expect(
-      redactAttributes({ object: { value: 1000 }, options: [] })
-    ).toStrictEqual({ value: 1000 });
+    expect(redactPaths({ object: { value: 1000 }, paths: [] })).toStrictEqual({
+      value: 1000
+    });
 
     expect(
-      redactAttributes({
+      redactPaths({
         object: { value: 1000 },
-        options: [{ attribute: 'value', valueMap: { '*': null } }]
+        paths: ['value']
       })
     ).toStrictEqual({ value: null });
 
     expect(
-      redactAttributes({
+      redactPaths({
         object: { value: 'abc' },
-        options: [{ attribute: 'value', valueMap: { abc: 'xyz' } }]
+        paths: ['value'],
+        valueMap: { abc: 'xyz' }
       })
     ).toStrictEqual({ value: 'xyz' });
 
     expect(
-      redactAttributes({
+      redactPaths({
         object: { data: [{ value: 'a' }, { value: 'b' }] },
-        options: [{ attribute: 'value', valueMap: { a: 1, b: 2 } }]
+        paths: ['data[*].value'],
+        valueMap: { a: 1, b: 2 }
       })
     ).toStrictEqual({ data: [{ value: 1 }, { value: 2 }] });
 
-    expect(
-      redactAttributes({
-        object: { value1: 'a', value2: 'b' },
-        options: [
-          { attribute: 'value1', valueMap: { a: 'x' } },
-          { attribute: 'value2', valueMap: { '*': 'y' } }
-        ]
-      })
-    ).toStrictEqual({ value1: 'x', value2: 'y' });
-
     console.time('redactAttributes execution time');
     expect(
-      redactAttributes({
+      redactPaths({
         object: {
           accounts: {
             '2e937c05-657c-4de9-8fb3-0813a2245f26': {
@@ -150,7 +144,7 @@ describe('redactAttributes', () => {
               ],
               dataSource: 'EOD_HISTORICAL_DATA',
               dateOfFirstActivity: '2021-11-30T23:00:00.000Z',
-              dividend: 0,
+              dividend: null,
               grossPerformance: 2665.5,
               grossPerformancePercent: 0.3183066634822068,
               grossPerformancePercentWithCurrencyEffect: 0.3183066634822068,
@@ -249,7 +243,7 @@ describe('redactAttributes', () => {
               ],
               dataSource: 'YAHOO',
               dateOfFirstActivity: '2018-09-30T22:00:00.000Z',
-              dividend: 0,
+              dividend: null,
               grossPerformance: 8689.05,
               grossPerformancePercent: 0.8594552890963852,
               grossPerformancePercentWithCurrencyEffect: 0.8594552890963852,
@@ -300,7 +294,7 @@ describe('redactAttributes', () => {
               countries: [],
               dataSource: 'COINGECKO',
               dateOfFirstActivity: '2017-08-15T22:00:00.000Z',
-              dividend: 0,
+              dividend: null,
               grossPerformance: 34985.0332704,
               grossPerformancePercent: 17.4925166352,
               grossPerformancePercentWithCurrencyEffect: 17.4925166352,
@@ -475,7 +469,7 @@ describe('redactAttributes', () => {
               ],
               dataSource: 'MANUAL',
               dateOfFirstActivity: '2021-03-31T22:00:00.000Z',
-              dividend: 0,
+              dividend: null,
               grossPerformance: 3533.389614611676,
               grossPerformancePercent: 0.27579517683678895,
               grossPerformancePercentWithCurrencyEffect: 0.458553421589667,
@@ -527,7 +521,7 @@ describe('redactAttributes', () => {
               ],
               dataSource: 'YAHOO',
               dateOfFirstActivity: '2023-01-02T23:00:00.000Z',
-              dividend: 0,
+              dividend: null,
               grossPerformance: 5653.2,
               grossPerformancePercent: 0.7865431171216295,
               grossPerformancePercentWithCurrencyEffect: 0.7865431171216295,
@@ -579,7 +573,7 @@ describe('redactAttributes', () => {
               ],
               dataSource: 'YAHOO',
               dateOfFirstActivity: '2017-01-02T23:00:00.000Z',
-              dividend: 0,
+              dividend: null,
               grossPerformance: 36920.500000005,
               grossPerformancePercent: 17.184314638161936,
               grossPerformancePercentWithCurrencyEffect: 17.184314638161936,
@@ -661,7 +655,7 @@ describe('redactAttributes', () => {
               ],
               dataSource: 'YAHOO',
               dateOfFirstActivity: '2019-02-28T23:00:00.000Z',
-              dividend: 0,
+              dividend: null,
               grossPerformance: 5856.3,
               grossPerformancePercent: 0.8832083851170418,
               grossPerformancePercentWithCurrencyEffect: 0.8832083851170418,
@@ -1061,7 +1055,7 @@ describe('redactAttributes', () => {
               ],
               dataSource: 'YAHOO',
               dateOfFirstActivity: '2018-02-28T23:00:00.000Z',
-              dividend: 0,
+              dividend: null,
               grossPerformance: 4534.60577952194,
               grossPerformancePercent: 0.3683200415015591,
               grossPerformancePercentWithCurrencyEffect: 0.5806366182968891,
@@ -1343,7 +1337,7 @@ describe('redactAttributes', () => {
               ],
               dataSource: 'YAHOO',
               dateOfFirstActivity: '2021-08-18T22:00:00.000Z',
-              dividend: 0,
+              dividend: null,
               grossPerformance: 2281.298817228297,
               grossPerformancePercent: 0.3474381850624522,
               grossPerformancePercentWithCurrencyEffect: 0.28744846894552306,
@@ -1461,7 +1455,7 @@ describe('redactAttributes', () => {
               assetClass: 'LIQUIDITY',
               assetSubClass: 'CASH',
               countries: [],
-              dividend: 0,
+              dividend: null,
               grossPerformance: 0,
               grossPerformancePercent: 0,
               grossPerformancePercentWithCurrencyEffect: 0,
@@ -1564,34 +1558,7 @@ describe('redactAttributes', () => {
             currentNetWorth: null
           }
         },
-        options: [
-          'balance',
-          'balanceInBaseCurrency',
-          'comment',
-          'convertedBalance',
-          'dividendInBaseCurrency',
-          'fee',
-          'feeInBaseCurrency',
-          'grossPerformance',
-          'grossPerformanceWithCurrencyEffect',
-          'investment',
-          'netPerformance',
-          'netPerformanceWithCurrencyEffect',
-          'quantity',
-          'symbolMapping',
-          'totalBalanceInBaseCurrency',
-          'totalValueInBaseCurrency',
-          'unitPrice',
-          'value',
-          'valueInBaseCurrency'
-        ].map((attribute) => {
-          return {
-            attribute,
-            valueMap: {
-              '*': null
-            }
-          };
-        })
+        paths: DEFAULT_REDACTED_PATHS
       })
     ).toStrictEqual({
       accounts: {
@@ -1681,7 +1648,7 @@ describe('redactAttributes', () => {
           ],
           dataSource: 'EOD_HISTORICAL_DATA',
           dateOfFirstActivity: '2021-11-30T23:00:00.000Z',
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0.3183066634822068,
           grossPerformancePercentWithCurrencyEffect: 0.3183066634822068,
@@ -1728,7 +1695,7 @@ describe('redactAttributes', () => {
           ],
           dataSource: 'YAHOO',
           dateOfFirstActivity: '2021-04-22T22:00:00.000Z',
-          dividend: 192,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0.3719230057375532,
           grossPerformancePercentWithCurrencyEffect: 0.2650716044872953,
@@ -1780,7 +1747,7 @@ describe('redactAttributes', () => {
           ],
           dataSource: 'YAHOO',
           dateOfFirstActivity: '2018-09-30T22:00:00.000Z',
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0.8594552890963852,
           grossPerformancePercentWithCurrencyEffect: 0.8594552890963852,
@@ -1831,7 +1798,7 @@ describe('redactAttributes', () => {
           countries: [],
           dataSource: 'COINGECKO',
           dateOfFirstActivity: '2017-08-15T22:00:00.000Z',
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 17.4925166352,
           grossPerformancePercentWithCurrencyEffect: 17.4925166352,
@@ -1882,7 +1849,7 @@ describe('redactAttributes', () => {
           countries: [],
           dataSource: 'MANUAL',
           dateOfFirstActivity: '2021-01-31T23:00:00.000Z',
-          dividend: 11.45,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0,
           grossPerformancePercentWithCurrencyEffect: -0.06153834320225245,
@@ -1986,7 +1953,7 @@ describe('redactAttributes', () => {
           ],
           dataSource: 'MANUAL',
           dateOfFirstActivity: '2021-03-31T22:00:00.000Z',
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0.27579517683678895,
           grossPerformancePercentWithCurrencyEffect: 0.458553421589667,
@@ -2038,7 +2005,7 @@ describe('redactAttributes', () => {
           ],
           dataSource: 'YAHOO',
           dateOfFirstActivity: '2023-01-02T23:00:00.000Z',
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0.7865431171216295,
           grossPerformancePercentWithCurrencyEffect: 0.7865431171216295,
@@ -2090,7 +2057,7 @@ describe('redactAttributes', () => {
           ],
           dataSource: 'YAHOO',
           dateOfFirstActivity: '2017-01-02T23:00:00.000Z',
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 17.184314638161936,
           grossPerformancePercentWithCurrencyEffect: 17.184314638161936,
@@ -2172,7 +2139,7 @@ describe('redactAttributes', () => {
           ],
           dataSource: 'YAHOO',
           dateOfFirstActivity: '2019-02-28T23:00:00.000Z',
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0.8832083851170418,
           grossPerformancePercentWithCurrencyEffect: 0.8832083851170418,
@@ -2567,7 +2534,7 @@ describe('redactAttributes', () => {
           ],
           dataSource: 'YAHOO',
           dateOfFirstActivity: '2018-02-28T23:00:00.000Z',
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0.3683200415015591,
           grossPerformancePercentWithCurrencyEffect: 0.5806366182968891,
@@ -2846,7 +2813,7 @@ describe('redactAttributes', () => {
           ],
           dataSource: 'YAHOO',
           dateOfFirstActivity: '2021-08-18T22:00:00.000Z',
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0.3474381850624522,
           grossPerformancePercentWithCurrencyEffect: 0.28744846894552306,
@@ -2964,7 +2931,7 @@ describe('redactAttributes', () => {
           assetClass: 'LIQUIDITY',
           assetSubClass: 'CASH',
           countries: [],
-          dividend: 0,
+          dividend: null,
           grossPerformance: null,
           grossPerformancePercent: 0,
           grossPerformancePercentWithCurrencyEffect: 0,

apps/api/src/helper/object.helper.ts

@@ -1,6 +1,6 @@
-import { Big } from 'big.js';
+import fastRedact from 'fast-redact';
 import jsonpath from 'jsonpath';
-import { cloneDeep, isArray, isObject } from 'lodash';
+import { cloneDeep, isObject } from 'lodash';
 
 export function hasNotDefinedValuesInObject(aObject: Object): boolean {
   for (const key in aObject) {
@@ -42,60 +42,29 @@ export function query({
   return jsonpath.query(object, pathExpression);
 }
 
-export function redactAttributes({
-  isFirstRun = true,
+export function redactPaths({
   object,
-  options
+  paths,
+  valueMap
 }: {
-  isFirstRun?: boolean;
   object: any;
-  options: { attribute: string; valueMap: { [key: string]: any } }[];
+  paths: fastRedact.RedactOptions['paths'];
+  valueMap?: { [key: string]: any };
 }): any {
-  if (!object || !options?.length) {
-    return object;
-  }
-
-  // Create deep clone
-  const redactedObject = isFirstRun
-    ? JSON.parse(JSON.stringify(object))
-    : object;
-
-  for (const option of options) {
-    if (redactedObject.hasOwnProperty(option.attribute)) {
-      if (option.valueMap['*'] || option.valueMap['*'] === null) {
-        redactedObject[option.attribute] = option.valueMap['*'];
-      } else if (option.valueMap[redactedObject[option.attribute]]) {
-        redactedObject[option.attribute] =
-          option.valueMap[redactedObject[option.attribute]];
-      }
-    } else {
-      // If the attribute is not present on the current object,
-      // check if it exists on any nested objects
-      for (const property in redactedObject) {
-        if (isArray(redactedObject[property])) {
-          redactedObject[property] = redactedObject[property].map(
-            (currentObject) => {
-              return redactAttributes({
-                options,
-                isFirstRun: false,
-                object: currentObject
-              });
-            }
-          );
-        } else if (
-          isObject(redactedObject[property]) &&
-          !(redactedObject[property] instanceof Big)
-        ) {
-          // Recursively call the function on the nested object
-          redactedObject[property] = redactAttributes({
-            options,
-            isFirstRun: false,
-            object: redactedObject[property]
-          });
+  const redact = fastRedact({
+    paths,
+    censor: (value) => {
+      if (valueMap) {
+        if (valueMap[value]) {
+          return valueMap[value];
+        } else {
+          return value;
         }
+      } else {
+        return null;
       }
     }
-  }
+  });
 
-  return redactedObject;
+  return JSON.parse(redact(object));
 }

apps/api/src/interceptors/redact-values-in-response/redact-values-in-response.interceptor.ts

@@ -1,5 +1,8 @@
-import { redactAttributes } from '@ghostfolio/api/helper/object.helper';
-import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
+import { redactPaths } from '@ghostfolio/api/helper/object.helper';
+import {
+  DEFAULT_REDACTED_PATHS,
+  HEADER_KEY_IMPERSONATION
+} from '@ghostfolio/common/config';
 import {
   hasReadRestrictedAccessPermission,
   isRestrictedView
@@ -39,40 +42,9 @@ export class RedactValuesInResponseInterceptor<T> implements NestInterceptor<
           }) ||
           isRestrictedView(user)
         ) {
-          data = redactAttributes({
+          data = redactPaths({
             object: data,
-            options: [
-              'balance',
-              'balanceInBaseCurrency',
-              'comment',
-              'convertedBalance',
-              'dividendInBaseCurrency',
-              'fee',
-              'feeInBaseCurrency',
-              'grossPerformance',
-              'grossPerformanceWithCurrencyEffect',
-              'interestInBaseCurrency',
-              'investment',
-              'netPerformance',
-              'netPerformanceWithCurrencyEffect',
-              'quantity',
-              'symbolMapping',
-              'totalBalanceInBaseCurrency',
-              'totalDividendInBaseCurrency',
-              'totalInterestInBaseCurrency',
-              'totalValueInBaseCurrency',
-              'unitPrice',
-              'unitPriceInAssetProfileCurrency',
-              'value',
-              'valueInBaseCurrency'
-            ].map((attribute) => {
-              return {
-                attribute,
-                valueMap: {
-                  '*': null
-                }
-              };
-            })
+            paths: DEFAULT_REDACTED_PATHS
           });
         }
 

apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts

@@ -1,4 +1,4 @@
-import { redactAttributes } from '@ghostfolio/api/helper/object.helper';
+import { redactPaths } from '@ghostfolio/api/helper/object.helper';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { encodeDataSource } from '@ghostfolio/common/helper';
 
@@ -58,13 +58,18 @@ export class TransformDataSourceInResponseInterceptor<
             }
           }
 
-          data = redactAttributes({
+          data = redactPaths({
+            valueMap,
             object: data,
-            options: [
-              {
-                valueMap,
-                attribute: 'dataSource'
-              }
+            paths: [
+              'activities[*].SymbolProfile.dataSource',
+              'benchmarks[*].dataSource',
+              'fearAndGreedIndex.CRYPTOCURRENCIES.dataSource',
+              'fearAndGreedIndex.STOCKS.dataSource',
+              'holdings[*].dataSource',
+              'items[*].dataSource',
+              'SymbolProfile.dataSource',
+              'watchlist[*].dataSource'
             ]
           });
         }

libs/common/src/lib/config.ts

@@ -78,6 +78,58 @@ export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY = 1;
 export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY = 1;
 export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000;
 
+export const DEFAULT_REDACTED_PATHS = [
+  'accounts[*].balance',
+  'accounts[*].valueInBaseCurrency',
+  'activities[*].account.balance',
+  'activities[*].account.comment',
+  'activities[*].comment',
+  'activities[*].fee',
+  'activities[*].feeInAssetProfileCurrency',
+  'activities[*].feeInBaseCurrency',
+  'activities[*].quantity',
+  'activities[*].SymbolProfile.symbolMapping',
+  'activities[*].SymbolProfile.watchedByCount',
+  'activities[*].value',
+  'activities[*].valueInBaseCurrency',
+  'balance',
+  'balanceInBaseCurrency',
+  'balances[*].account.balance',
+  'balances[*].account.comment',
+  'balances[*].value',
+  'balances[*].valueInBaseCurrency',
+  'comment',
+  'dividendInBaseCurrency',
+  'feeInBaseCurrency',
+  'grossPerformance',
+  'grossPerformanceWithCurrencyEffect',
+  'historicalData[*].quantity',
+  'holdings[*].dividend',
+  'holdings[*].grossPerformance',
+  'holdings[*].grossPerformanceWithCurrencyEffect',
+  'holdings[*].holdings[*].valueInBaseCurrency',
+  'holdings[*].investment',
+  'holdings[*].netPerformance',
+  'holdings[*].netPerformanceWithCurrencyEffect',
+  'holdings[*].quantity',
+  'holdings[*].valueInBaseCurrency',
+  'interestInBaseCurrency',
+  'investmentInBaseCurrencyWithCurrencyEffect',
+  'netPerformance',
+  'netPerformanceWithCurrencyEffect',
+  'platforms[*].balance',
+  'platforms[*].valueInBaseCurrency',
+  'quantity',
+  'SymbolProfile.symbolMapping',
+  'SymbolProfile.watchedByCount',
+  'totalBalanceInBaseCurrency',
+  'totalDividendInBaseCurrency',
+  'totalInterestInBaseCurrency',
+  'totalValueInBaseCurrency',
+  'value',
+  'valueInBaseCurrency'
+];
+
 // USX is handled separately
 export const DERIVED_CURRENCIES = [
   {

package-lock.json

@@ -62,6 +62,7 @@
         "dotenv": "17.2.3",
         "dotenv-expand": "12.0.3",
         "envalid": "8.1.1",
+        "fast-redact": "3.5.0",
         "fuse.js": "7.1.0",
         "google-spreadsheet": "3.2.0",
         "helmet": "7.0.0",
@@ -122,6 +123,7 @@
         "@storybook/angular": "10.1.10",
         "@trivago/prettier-plugin-sort-imports": "5.2.2",
         "@types/big.js": "6.2.2",
+        "@types/fast-redact": "3.0.4",
         "@types/google-spreadsheet": "3.1.5",
         "@types/jest": "30.0.0",
         "@types/jsonpath": "0.2.4",
@@ -12936,6 +12938,13 @@
         "@types/send": "*"
       }
     },
+    "node_modules/@types/fast-redact": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/fast-redact/-/fast-redact-3.0.4.tgz",
+      "integrity": "sha512-tgGJaXucrCH4Yx2l/AI6e/JQksZhKGIQsVwBMTh+nxUhQDv5tXScTs5DHTw+qSKDXnHL2dTAh1e2rd5pcFQyNQ==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/geojson": {
       "version": "7946.0.16",
       "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz",
@@ -19940,6 +19949,15 @@
       "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
       "license": "MIT"
     },
+    "node_modules/fast-redact": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz",
+      "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/fast-safe-stringify": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",

package.json

@@ -106,6 +106,7 @@
     "dotenv": "17.2.3",
     "dotenv-expand": "12.0.3",
     "envalid": "8.1.1",
+    "fast-redact": "3.5.0",
     "fuse.js": "7.1.0",
     "google-spreadsheet": "3.2.0",
     "helmet": "7.0.0",
@@ -166,6 +167,7 @@
     "@storybook/angular": "10.1.10",
     "@trivago/prettier-plugin-sort-imports": "5.2.2",
     "@types/big.js": "6.2.2",
+    "@types/fast-redact": "3.0.4",
     "@types/google-spreadsheet": "3.1.5",
     "@types/jest": "30.0.0",
     "@types/jsonpath": "0.2.4",