Task/improve data source encoding (#7138)

Improve data source encoding

apps/api/src/app/info/info.service.ts

@@ -1,6 +1,7 @@
 import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service';
 import { SubscriptionService } from '@ghostfolio/api/app/subscription/subscription.service';
 import { UserService } from '@ghostfolio/api/app/user/user.service';
+import { encodeDataSource } from '@ghostfolio/api/helper/data-source.helper';
 import { BenchmarkService } from '@ghostfolio/api/services/benchmark/benchmark.service';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
@@ -17,7 +18,6 @@ import {
   PROPERTY_UPTIME,
   ghostfolioFearAndGreedIndexDataSourceStocks
 } from '@ghostfolio/common/config';
-import { encodeDataSource } from '@ghostfolio/common/helper';
 import { InfoItem, Statistics } from '@ghostfolio/common/interfaces';
 import { permissions } from '@ghostfolio/common/permissions';
 

apps/api/src/helper/data-source.helper.ts

@@ -1,4 +1,58 @@
 import { DataSource } from '@prisma/client';
+import { createHash } from 'node:crypto';
+
+const encodedDataSourceByDataSource = new Map<DataSource, string>(
+  Object.values(DataSource).map((dataSource) => {
+    return [dataSource, hashDataSource(dataSource)];
+  })
+);
+
+const dataSourceByEncodedDataSource = new Map<string, DataSource>(
+  [...encodedDataSourceByDataSource].map(([dataSource, encodedDataSource]) => {
+    return [encodedDataSource, dataSource];
+  })
+);
+
+/**
+ * @deprecated Backward compatibility to support importing data that was
+ * exported using the previous data source encoding
+ */
+const dataSourceByDeprecatedEncodedDataSource = new Map<string, DataSource>(
+  Object.values(DataSource).map((dataSource) => {
+    return [deprecatedHashDataSource(dataSource), dataSource];
+  })
+);
+
+/**
+ * @deprecated Backward compatibility (see above)
+ */
+function deprecatedHashDataSource(dataSource: DataSource) {
+  return Buffer.from(dataSource, 'utf-8').toString('hex');
+}
+
+function hashDataSource(dataSource: DataSource) {
+  return createHash('sha256').update(dataSource).digest('hex').slice(0, 8);
+}
+
+export function decodeDataSource(encodedDataSource: string) {
+  if (!encodedDataSource) {
+    return undefined;
+  }
+
+  return (
+    dataSourceByEncodedDataSource.get(encodedDataSource) ??
+    dataSourceByDeprecatedEncodedDataSource.get(encodedDataSource) ??
+    encodedDataSource
+  );
+}
+
+export function encodeDataSource(dataSource: DataSource) {
+  if (!dataSource) {
+    return undefined;
+  }
+
+  return encodedDataSourceByDataSource.get(dataSource);
+}
 
 export function getMaskedGhostfolioDataSource({
   dataSource,

apps/api/src/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor.ts

@@ -1,5 +1,5 @@
+import { decodeDataSource } from '@ghostfolio/api/helper/data-source.helper';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { decodeDataSource } from '@ghostfolio/common/helper';
 
 import {
   CallHandler,

apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts

@@ -1,7 +1,10 @@
-import { getMaskedGhostfolioDataSource } from '@ghostfolio/api/helper/data-source.helper';
+import {
+  encodeDataSource,
+  getMaskedGhostfolioDataSource
+} from '@ghostfolio/api/helper/data-source.helper';
 import { redactPaths } from '@ghostfolio/api/helper/object.helper';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { encodeDataSource } from '@ghostfolio/common/helper';
+import { hasRole } from '@ghostfolio/common/permissions';
 
 import {
   CallHandler,
@@ -45,11 +48,14 @@ export class TransformDataSourceInResponseInterceptor<
     next: CallHandler<T>
   ): Observable<any> {
     const isExportMode = context.getClass().name === 'ExportController';
+    const { user } = context.switchToHttp().getRequest();
 
     return next.handle().pipe(
       map((data: any) => {
         if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) {
-          const valueMap = this.encodedDataSourceMap;
+          const valueMap = hasRole(user, 'ADMIN')
+            ? {}
+            : { ...this.encodedDataSourceMap };
 
           if (isExportMode) {
             const ghostfolioDataSources = this.configurationService.get(
@@ -64,6 +70,10 @@ export class TransformDataSourceInResponseInterceptor<
             }
           }
 
+          if (Object.keys(valueMap).length === 0) {
+            return data;
+          }
+
           data = redactPaths({
             valueMap,
             object: data,

libs/common/src/lib/helper.ts

@@ -1,7 +1,6 @@
 import { NumberParser } from '@internationalized/number';
 import {
   Type as ActivityType,
-  DataSource,
   MarketData,
   Prisma,
   SymbolProfile,
@@ -166,14 +165,6 @@ export function capitalize(aString: string) {
   return aString.charAt(0).toUpperCase() + aString.slice(1).toLowerCase();
 }
 
-export function decodeDataSource(encodedDataSource: string) {
-  if (encodedDataSource) {
-    return Buffer.from(encodedDataSource, 'hex').toString();
-  }
-
-  return undefined;
-}
-
 export function downloadAsFile({
   content,
   contentType = 'text/plain',
@@ -199,14 +190,6 @@ export function downloadAsFile({
   a.click();
 }
 
-export function encodeDataSource(aDataSource: DataSource) {
-  if (aDataSource) {
-    return Buffer.from(aDataSource, 'utf-8').toString('hex');
-  }
-
-  return undefined;
-}
-
 export function extractNumberFromString({
   locale = 'en-US',
   value