Fix missing asset profiles and historical data

apps/api/src/app/admin/admin.service.ts

@@ -11,7 +11,10 @@ import {
   PROPERTY_IS_READ_ONLY_MODE,
   PROPERTY_IS_USER_SIGNUP_ENABLED
 } from '@ghostfolio/common/config';
-import { getCurrencyFromSymbol } from '@ghostfolio/common/helper';
+import {
+  getAssetProfileIdentifier,
+  getCurrencyFromSymbol
+} from '@ghostfolio/common/helper';
 import {
   AdminData,
   AdminUserResponse,
@@ -68,14 +71,17 @@ export class AdminService {
         { dataSource, symbol }
       ]);
 
-      if (!assetProfiles[symbol]?.currency) {
+      const assetProfile =
+        assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })];
+
+      if (!assetProfile?.currency) {
         throw new BadRequestException(
           `Asset profile not found for ${symbol} (${dataSource})`
         );
       }
 
       return this.symbolProfileService.add(
-        assetProfiles[symbol] as Prisma.SymbolProfileCreateInput
+        assetProfile as Prisma.SymbolProfileCreateInput
       );
     } catch (error) {
       if (

apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts

@@ -16,6 +16,7 @@ import {
   DERIVED_CURRENCIES
 } from '@ghostfolio/common/config';
 import { PROPERTY_DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER_MAX_REQUESTS } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import {
   DataProviderGhostfolioAssetProfileResponse,
   DataProviderHistoricalResponse,
@@ -60,7 +61,13 @@ export class GhostfolioService {
               }
             ])
             .then(async (assetProfiles) => {
-              const assetProfile = assetProfiles[symbol];
+              const assetProfile =
+                assetProfiles[
+                  getAssetProfileIdentifier({
+                    symbol,
+                    dataSource: dataProviderService.getName()
+                  })
+                ];
               const dataSourceOrigin = DataSource.GHOSTFOLIO;
 
               if (assetProfile) {

apps/api/src/app/endpoints/watchlist/watchlist.service.ts

@@ -41,14 +41,17 @@ export class WatchlistService {
         { dataSource, symbol }
       ]);
 
-      if (!assetProfiles[symbol]?.currency) {
+      const assetProfile =
+        assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })];
+
+      if (!assetProfile?.currency) {
         throw new BadRequestException(
           `Asset profile not found for ${symbol} (${dataSource})`
         );
       }
 
       await this.symbolProfileService.add(
-        assetProfiles[symbol] as Prisma.SymbolProfileCreateInput
+        assetProfile as Prisma.SymbolProfileCreateInput
       );
     }
 

apps/api/src/app/portfolio/portfolio.service.ts

@@ -890,10 +890,13 @@ export class PortfolioService {
       marketPrice
     );
 
-    if (historicalData[symbol]) {
+    const historicalDataItems =
+      historicalData[getAssetProfileIdentifier({ dataSource, symbol })];
+
+    if (historicalDataItems) {
       let j = -1;
       for (const [date, { marketPrice }] of Object.entries(
-        historicalData[symbol]
+        historicalDataItems
       )) {
         while (
           j + 1 < transactionPoints.length &&

apps/api/src/app/symbol/symbol.service.ts

@@ -94,12 +94,17 @@ export class SymbolService {
     date = new Date(),
     symbol
   }: DataGatheringItem): Promise<DataProviderHistoricalResponse> {
+    const assetProfileIdentifier = getAssetProfileIdentifier({
+      dataSource,
+      symbol
+    });
+
     let historicalData: {
-      [symbol: string]: {
+      [assetProfileIdentifier: string]: {
         [date: string]: DataProviderHistoricalResponse;
       };
     } = {
-      [symbol]: {}
+      [assetProfileIdentifier]: {}
     };
 
     try {
@@ -112,7 +117,8 @@ export class SymbolService {
 
     return {
       marketPrice:
-        historicalData?.[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice
+        historicalData?.[assetProfileIdentifier]?.[format(date, DATE_FORMAT)]
+          ?.marketPrice
     };
   }
 

apps/api/src/services/data-provider/data-provider.service.ts

@@ -86,12 +86,11 @@ export class DataProviderService implements OnModuleInit {
     return false;
   }
 
-  // TODO: Change symbol in response to assetProfileIdentifier
   public async getAssetProfiles(items: AssetProfileIdentifier[]): Promise<{
-    [symbol: string]: Partial<SymbolProfile>;
+    [assetProfileIdentifier: string]: Partial<SymbolProfile>;
   }> {
     const response: {
-      [symbol: string]: Partial<SymbolProfile>;
+      [assetProfileIdentifier: string]: Partial<SymbolProfile>;
     } = {};
 
     const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => {
@@ -117,7 +116,12 @@ export class DataProviderService implements OnModuleInit {
         promises.push(
           promise.then((assetProfile) => {
             if (isCurrency(assetProfile?.currency)) {
-              response[symbol] = assetProfile;
+              response[
+                getAssetProfileIdentifier({
+                  symbol,
+                  dataSource: DataSource[dataSource]
+                })
+              ] = { ...assetProfile, symbol };
             }
           })
         );
@@ -335,17 +339,20 @@ export class DataProviderService implements OnModuleInit {
     });
   }
 
-  // TODO: Change symbol in response to assetProfileIdentifier
   public async getHistorical(
     aItems: AssetProfileIdentifier[],
     aGranularity: Granularity = 'month',
     from: Date,
     to: Date
   ): Promise<{
-    [symbol: string]: { [date: string]: DataProviderHistoricalResponse };
+    [assetProfileIdentifier: string]: {
+      [date: string]: DataProviderHistoricalResponse;
+    };
   }> {
     let response: {
-      [symbol: string]: { [date: string]: DataProviderHistoricalResponse };
+      [assetProfileIdentifier: string]: {
+        [date: string]: DataProviderHistoricalResponse;
+      };
     } = {};
 
     if (isEmpty(aItems) || !isValid(from) || !isValid(to)) {
@@ -385,13 +392,20 @@ export class DataProviderService implements OnModuleInit {
           ORDER BY date;`;
 
       response = marketDataByGranularity.reduce((r, marketData) => {
-        const { date, marketPrice, symbol } = marketData;
+        const { dataSource, date, marketPrice, symbol } = marketData;
 
-        if (!r[symbol]) {
+        const assetProfileIdentifier = getAssetProfileIdentifier({
-          r[symbol] = {};
+          dataSource,
+          symbol
+        });
+
+        if (!r[assetProfileIdentifier]) {
+          r[assetProfileIdentifier] = {};
         }
 
-        r[symbol][format(new Date(date), DATE_FORMAT)] = { marketPrice };
+        r[assetProfileIdentifier][format(new Date(date), DATE_FORMAT)] = {
+          marketPrice
+        };
 
         return r;
       }, {});
@@ -402,7 +416,6 @@ export class DataProviderService implements OnModuleInit {
     }
   }
 
-  // TODO: Change symbol in response to assetProfileIdentifier
   public async getHistoricalRaw({
     assetProfileIdentifiers,
     from,
@@ -412,7 +425,9 @@ export class DataProviderService implements OnModuleInit {
     from: Date;
     to: Date;
   }): Promise<{
-    [symbol: string]: { [date: string]: DataProviderHistoricalResponse };
+    [assetProfileIdentifier: string]: {
+      [date: string]: DataProviderHistoricalResponse;
+    };
   }> {
     for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
       if (
@@ -445,11 +460,14 @@ export class DataProviderService implements OnModuleInit {
     );
 
     const result: {
-      [symbol: string]: { [date: string]: DataProviderHistoricalResponse };
+      [assetProfileIdentifier: string]: {
+        [date: string]: DataProviderHistoricalResponse;
+      };
     } = {};
 
     const promises: Promise<{
       data: { [date: string]: DataProviderHistoricalResponse };
+      dataSource: DataSource;
       symbol: string;
     }>[] = [];
     for (const { dataSource, symbol } of assetProfileIdentifiers) {
@@ -467,6 +485,7 @@ export class DataProviderService implements OnModuleInit {
           promises.push(
             Promise.resolve({
               data,
+              dataSource,
               symbol
             })
           );
@@ -480,7 +499,7 @@ export class DataProviderService implements OnModuleInit {
                 requestTimeout: ms('30 seconds')
               })
               .then((data) => {
-                return { symbol, data: data?.[symbol] };
+                return { dataSource, symbol, data: data?.[symbol] };
               })
           );
         }
@@ -490,22 +509,26 @@ export class DataProviderService implements OnModuleInit {
     try {
       const allData = await Promise.all(promises);
 
-      for (const { data, symbol } of allData) {
+      for (const { data, dataSource, symbol } of allData) {
         const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => {
           return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol;
         });
 
         if (currency) {
           // Add derived currency
-          result[`${DEFAULT_CURRENCY}${currency.currency}`] =
+          result[
-            this.transformHistoricalData({
+            getAssetProfileIdentifier({
-              allData,
+              dataSource,
-              currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`,
+              symbol: `${DEFAULT_CURRENCY}${currency.currency}`
-              factor: currency.factor
+            })
-            });
+          ] = this.transformHistoricalData({
+            allData,
+            currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`,
+            factor: currency.factor
+          });
         }
 
-        result[symbol] = data;
+        result[getAssetProfileIdentifier({ dataSource, symbol })] = data;
       }
     } catch (error) {
       this.logger.error(error);

apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts

@@ -15,6 +15,7 @@ import {
   getYesterday,
   resetHours
 } from '@ghostfolio/common/helper';
+import { DataProviderHistoricalResponse } from '@ghostfolio/common/interfaces';
 
 import { Injectable, Logger } from '@nestjs/common';
 import {
@@ -163,7 +164,7 @@ export class ExchangeRateDataService {
   }
 
   public async loadCurrencies() {
-    const result = await this.dataProviderService.getHistorical(
+    const historicalData = await this.dataProviderService.getHistorical(
       this.currencyPairs,
       'day',
       getYesterday(),
@@ -177,8 +178,21 @@ export class ExchangeRateDataService {
       requestTimeout: ms('30 seconds')
     });
 
+    const result: {
+      [symbol: string]: { [date: string]: DataProviderHistoricalResponse };
+    } = {};
+
     for (const { dataSource, symbol } of this.currencyPairs) {
-      const quote = quotes[getAssetProfileIdentifier({ dataSource, symbol })];
+      const assetProfileIdentifier = getAssetProfileIdentifier({
+        dataSource,
+        symbol
+      });
+
+      if (historicalData[assetProfileIdentifier]) {
+        result[symbol] = historicalData[assetProfileIdentifier];
+      }
+
+      const quote = quotes[assetProfileIdentifier];
 
       if (isNumber(quote?.marketPrice)) {
         result[symbol] = {

apps/api/src/services/queues/data-gathering/data-gathering.processor.ts

@@ -10,7 +10,11 @@ import {
   GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
 } from '@ghostfolio/common/config';
-import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
+import {
+  DATE_FORMAT,
+  getAssetProfileIdentifier,
+  getStartOfUtcDate
+} from '@ghostfolio/common/helper';
 import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
 
 import { Process, Processor } from '@nestjs/bull';
@@ -114,6 +118,11 @@ export class DataGatheringProcessor {
         to: new Date()
       });
 
+      const assetProfileIdentifier = getAssetProfileIdentifier({
+        dataSource,
+        symbol
+      });
+
       const data: Prisma.MarketDataUpdateInput[] = [];
       let lastMarketPrice: number;
 
@@ -131,12 +140,14 @@ export class DataGatheringProcessor {
         )
       ) {
         if (
-          historicalData[symbol]?.[format(currentDate, DATE_FORMAT)]
+          historicalData[assetProfileIdentifier]?.[
-            ?.marketPrice
+            format(currentDate, DATE_FORMAT)
+          ]?.marketPrice
         ) {
           lastMarketPrice =
-            historicalData[symbol]?.[format(currentDate, DATE_FORMAT)]
+            historicalData[assetProfileIdentifier]?.[
-              ?.marketPrice;
+              format(currentDate, DATE_FORMAT)
+            ]?.marketPrice;
         }
 
         if (lastMarketPrice) {

apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@@ -135,7 +135,9 @@ export class DataGatheringService {
       });
 
       const marketPrice =
-        historicalData[symbol][format(date, DATE_FORMAT)].marketPrice;
+        historicalData[getAssetProfileIdentifier({ dataSource, symbol })][
+          format(date, DATE_FORMAT)
+        ].marketPrice;
 
       if (marketPrice) {
         return await this.prismaService.marketData.upsert({
@@ -180,7 +182,9 @@ export class DataGatheringService {
       assetProfileIdentifiers
     );
 
-    for (const [symbol, assetProfile] of Object.entries(assetProfiles)) {
+    for (const assetProfile of Object.values(assetProfiles)) {
+      const { symbol } = assetProfile;
+
       const symbolProfile = symbolProfiles.find(
         ({ symbol: symbolProfileSymbol }) => {
           return symbolProfileSymbol === symbol;