Merge pull request #129 from dandevaud/feature/Add-Gather-Missing-data-only

Add Datagathering process for missing values only

CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+- Added the name to the tooltip of the chart of the holdings tab on the home page (experimental)
+
+### Changed
+
+- Exposed the timeout of the portfolio snapshot computation as an environment variable (`PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT`)
+- Improved the portfolio unit tests to work with exported activity files
+
+### Fixed
+
+- Considered the language of the user settings on login with _Security Token_
+
 ## 2.114.0 - 2024-10-10
 
 ### Added

apps/api/src/app/admin/admin.controller.ts

@@ -158,7 +158,22 @@ export class AdminController {
     @Param('dataSource') dataSource: DataSource,
     @Param('symbol') symbol: string
   ): Promise<void> {
-    this.dataGatheringService.gatherSymbol({ dataSource, symbol });
+    await this.dataGatheringService.gatherSymbol({ dataSource, symbol });
+
+    return;
+  }
+
+  @Post('gatherMissing/:dataSource/:symbol')
+  @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
+  @HasPermission(permissions.accessAdminControl)
+  public async gatherSymbolMissingOnly(
+    @Param('dataSource') dataSource: DataSource,
+    @Param('symbol') symbol: string
+  ): Promise<void> {
+    await this.dataGatheringService.gatherSymbolMissingOnly({
+      dataSource,
+      symbol
+    });
 
     return;
   }

apps/api/src/app/admin/admin.service.ts

@@ -41,9 +41,7 @@ import {
   PrismaClient,
   Property,
   SymbolProfile,
-  DataSource,
-  Tag,
-  SymbolProfileOverrides
+  DataSource
 } from '@prisma/client';
 import { differenceInDays } from 'date-fns';
 import { groupBy } from 'lodash';

apps/api/src/app/portfolio/calculator/constantPortfolioReturn/portfolio-calculator.ts

@@ -7,16 +7,13 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service';
-import { getIntervalFromDateRange } from '@ghostfolio/common/calculation-helper';
 import { DATE_FORMAT, parseDate, resetHours } from '@ghostfolio/common/helper';
 import { Filter, HistoricalDataItem } from '@ghostfolio/common/interfaces';
-import { DateRange } from '@ghostfolio/common/types';
 
 import { Inject, Logger } from '@nestjs/common';
 import { Big } from 'big.js';
 import {
   addDays,
-  differenceInDays,
   eachDayOfInterval,
   endOfDay,
   format,
@@ -83,19 +80,19 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     start: Date;
     end: Date;
   }): Promise<{ chart: HistoricalDataItem[] }> {
-    let item = await super.getPerformance({
+    const item = await super.getPerformance({
       end,
       start
     });
 
-    let itemResult = item.chart;
-    let dates = itemResult.map((item) => parseDate(item.date));
-    let timeWeighted = await this.getTimeWeightedChartData({
+    const itemResult = item.chart;
+    const dates = itemResult.map((item) => parseDate(item.date));
+    const timeWeighted = await this.getTimeWeightedChartData({
       dates
     });
 
     item.chart = itemResult.map((itemInt) => {
-      let timeWeightedItem = timeWeighted.find(
+      const timeWeightedItem = timeWeighted.find(
         (timeWeightedItem) => timeWeightedItem.date === itemInt.date
       );
       if (timeWeightedItem) {
@@ -135,11 +132,11 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
       dateQuery: { in: [end] }
     });
     const endString = format(end, DATE_FORMAT);
-    let exchangeRates = await Promise.all(
+    const exchangeRates = await Promise.all(
       Object.keys(holdings[endString]).map(async (holding) => {
-        let symbol = marketMap.values.find((m) => m.symbol === holding);
-        let symbolCurrency = this.getCurrencyFromActivities(orders, holding);
-        let exchangeRate = await this.exchangeRateDataService.toCurrencyAtDate(
+        const symbolCurrency = this.getCurrencyFromActivities(orders, holding);
+        const exchangeRate =
+          await this.exchangeRateDataService.toCurrencyAtDate(
             1,
             symbolCurrency,
             this.currency,
@@ -148,7 +145,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
         return { symbolCurrency, exchangeRate };
       })
     );
-    let currencyRates = exchangeRates.reduce<{ [currency: string]: number }>(
+    const currencyRates = exchangeRates.reduce<{ [currency: string]: number }>(
       (all, currency): { [currency: string]: number } => {
         all[currency.symbolCurrency] ??= currency.exchangeRate;
         return all;
@@ -156,12 +153,12 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
       {}
     );
 
-    let totalInvestment = await Object.keys(holdings[endString]).reduce(
+    const totalInvestment = await Object.keys(holdings[endString]).reduce(
       (sum, holding) => {
         if (!holdings[endString][holding].toNumber()) {
           return sum;
         }
-        let symbol = marketMap.values.find((m) => m.symbol === holding);
+        const symbol = marketMap.values.find((m) => m.symbol === holding);
 
         if (symbol?.marketPrice === undefined) {
           Logger.warn(
@@ -170,8 +167,8 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
           );
           return sum;
         } else {
-          let symbolCurrency = this.getCurrency(holding);
-          let price = new Big(currencyRates[symbolCurrency]).mul(
+          const symbolCurrency = this.getCurrency(holding);
+          const price = new Big(currencyRates[symbolCurrency]).mul(
             symbol.marketPrice
           );
           return sum.plus(new Big(price).mul(holdings[endString][holding]));
@@ -191,7 +188,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     dates = dates.sort((a, b) => a.getTime() - b.getTime());
     const start = dates[0];
     const end = dates[dates.length - 1];
-    let marketMapTask = this.computeMarketMap({
+    const marketMapTask = this.computeMarketMap({
       gte: start,
       lt: addDays(end, 1)
     });
@@ -201,7 +198,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
       end
     );
 
-    let data: HistoricalDataItem[] = [];
+    const data: HistoricalDataItem[] = [];
     const startString = format(start, DATE_FORMAT);
 
     data.push({
@@ -435,12 +432,12 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
     end: Date
   ) {
     const transactionDates = Object.keys(investmentByDate).sort();
-    let dates = eachDayOfInterval({ start, end }, { step: 1 })
+    const dates = eachDayOfInterval({ start, end }, { step: 1 })
       .map((date) => {
         return resetHours(date);
       })
       .sort((a, b) => a.getTime() - b.getTime());
-    let currentHoldings: { [date: string]: { [symbol: string]: Big } } = {};
+    const currentHoldings: { [date: string]: { [symbol: string]: Big } } = {};
 
     this.calculateInitialHoldings(investmentByDate, start, currentHoldings);
 
@@ -448,7 +445,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
       const dateString = format(dates[i], DATE_FORMAT);
       const previousDateString = format(dates[i - 1], DATE_FORMAT);
       if (transactionDates.some((d) => d === dateString)) {
-        let holdings = { ...currentHoldings[previousDateString] };
+        const holdings = { ...currentHoldings[previousDateString] };
         investmentByDate[dateString].forEach((trade) => {
           holdings[trade.SymbolProfile.symbol] ??= new Big(0);
           holdings[trade.SymbolProfile.symbol] = holdings[
@@ -488,7 +485,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
 
     for (const symbol of Object.keys(preRangeTrades)) {
       const trades: PortfolioOrder[] = preRangeTrades[symbol];
-      let startQuantity = trades.reduce((sum, trade) => {
+      const startQuantity = trades.reduce((sum, trade) => {
         return sum.plus(trade.quantity.mul(getFactor(trade.type)));
       }, new Big(0));
       currentHoldings[format(start, DATE_FORMAT)][symbol] = startQuantity;

apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts

@@ -1,3 +1,5 @@
+import { readFileSync } from 'fs';
+
 export const activityDummyData = {
   accountId: undefined,
   accountUserId: undefined,
@@ -29,3 +31,7 @@ export const symbolProfileDummyData = {
 export const userDummyData = {
   id: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
 };
+
+export function loadActivityExportFile(filePath: string) {
+  return JSON.parse(readFileSync(filePath, 'utf8')).activities;
+}

apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts

@@ -13,7 +13,6 @@ import { OrderService } from '../../order/order.service';
 import { CPRPortfolioCalculator } from './constantPortfolioReturn/portfolio-calculator';
 import { MWRPortfolioCalculator } from './mwr/portfolio-calculator';
 import { PortfolioCalculator } from './portfolio-calculator';
-import { TWRPortfolioCalculator } from './twr/portfolio-calculator';
 
 export enum PerformanceCalculationType {
   MWR = 'MWR', // Money-Weighted Rate of Return

apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts

@@ -156,25 +156,25 @@ describe('PortfolioCalculator', () => {
             dividendInBaseCurrency: new Big('0.62'),
             fee: new Big('19'),
             firstBuyDate: '2021-09-16',
-            grossPerformance: new Big('33.25'),
-            grossPerformancePercentage: new Big('0.11136043941322258691'),
+            grossPerformance: new Big('33.87'),
+            grossPerformancePercentage: new Big('0.11343693482483756447'),
             grossPerformancePercentageWithCurrencyEffect: new Big(
-              '0.11136043941322258691'
+              '0.11343693482483756447'
             ),
-            grossPerformanceWithCurrencyEffect: new Big('33.25'),
+            grossPerformanceWithCurrencyEffect: new Big('33.87'),
             investment: new Big('298.58'),
             investmentWithCurrencyEffect: new Big('298.58'),
             marketPrice: 331.83,
             marketPriceInBaseCurrency: 331.83,
-            netPerformance: new Big('14.25'),
-            netPerformancePercentage: new Big('0.04772590260566682296'),
+            netPerformance: new Big('14.87'),
+            netPerformancePercentage: new Big('0.04980239801728180052'),
             netPerformancePercentageWithCurrencyEffectMap: {
-              max: new Big('0.04772590260566682296')
+              max: new Big('0.04980239801728180052')
             },
             netPerformanceWithCurrencyEffectMap: {
               '1d': new Big('-5.39'),
-              '5y': new Big('14.25'),
-              max: new Big('14.25'),
+              '5y': new Big('14.87'),
+              max: new Big('14.87'),
               wtd: new Big('-5.39')
             },
             quantity: new Big('1'),

apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts

@@ -1,6 +1,8 @@
+import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import {
   activityDummyData,
+  loadActivityExportFile,
   symbolProfileDummyData,
   userDummyData
 } from '@ghostfolio/api/app/portfolio/calculator/portfolio-calculator-test-utils';
@@ -20,6 +22,7 @@ import { parseDate } from '@ghostfolio/common/helper';
 
 import { Big } from 'big.js';
 import { last } from 'lodash';
+import { join } from 'path';
 
 jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => {
   return {
@@ -52,6 +55,8 @@ jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => {
 });
 
 describe('PortfolioCalculator', () => {
+  let activityDtos: CreateOrderDto[];
+
   let configurationService: ConfigurationService;
   let currentRateService: CurrentRateService;
   let exchangeRateDataService: ExchangeRateDataService;
@@ -59,6 +64,15 @@ describe('PortfolioCalculator', () => {
   let portfolioSnapshotService: PortfolioSnapshotService;
   let redisCacheService: RedisCacheService;
 
+  beforeAll(() => {
+    activityDtos = loadActivityExportFile(
+      join(
+        __dirname,
+        '../../../../../../../test/import/ok-novn-buy-and-sell.json'
+      )
+    );
+  });
+
   beforeEach(() => {
     configurationService = new ConfigurationService();
 
@@ -89,38 +103,18 @@ describe('PortfolioCalculator', () => {
     it.only('with NOVN.SW buy and sell', async () => {
       jest.useFakeTimers().setSystemTime(parseDate('2022-04-11').getTime());
 
-      const activities: Activity[] = [
-        {
+      const activities: Activity[] = activityDtos.map((activity) => ({
         ...activityDummyData,
-          date: new Date('2022-03-07'),
-          fee: 0,
-          quantity: 2,
+        ...activity,
+        date: parseDate(activity.date),
         SymbolProfile: {
           ...symbolProfileDummyData,
-            currency: 'CHF',
-            dataSource: 'YAHOO',
+          currency: activity.currency,
+          dataSource: activity.dataSource,
           name: 'Novartis AG',
-            symbol: 'NOVN.SW'
-          },
-          type: 'BUY',
-          unitPrice: 75.8
-        },
-        {
-          ...activityDummyData,
-          date: new Date('2022-04-08'),
-          fee: 0,
-          quantity: 2,
-          SymbolProfile: {
-            ...symbolProfileDummyData,
-            currency: 'CHF',
-            dataSource: 'YAHOO',
-            name: 'Novartis AG',
-            symbol: 'NOVN.SW'
-          },
-          type: 'SELL',
-          unitPrice: 85.73
+          symbol: activity.symbol
         }
-      ];
+      }));
 
       const portfolioCalculator = portfolioCalculatorFactory.createCalculator({
         activities,

apps/api/src/app/portfolio/portfolio.controller.ts

@@ -84,7 +84,6 @@ export class PortfolioController {
     @Query('assetClasses') filterByAssetClasses?: string,
     @Query('range') dateRange: DateRange = 'max',
     @Query('tags') filterByTags?: string,
-    @Query('isAllocation') isAllocation: boolean = false,
     @Query('withMarkets') withMarketsParam = 'false'
   ): Promise<PortfolioDetails & { hasError: boolean }> {
     const withMarkets = withMarketsParam === 'true';
@@ -483,8 +482,7 @@ export class PortfolioController {
     @Query('range') dateRange: DateRange = 'max',
     @Query('tags') filterByTags?: string,
     @Query('withExcludedAccounts') withExcludedAccounts = false,
-    @Query('timeWeightedPerformance') calculateTimeWeightedPerformance = false,
-    @Query('withItems') withItems = false
+    @Query('timeWeightedPerformance') calculateTimeWeightedPerformance = false
   ): Promise<PortfolioPerformanceResponse> {
     const filters = this.apiService.buildFiltersFromQueryParams({
       filterByAccounts,

apps/api/src/app/portfolio/portfolio.service.ts

@@ -15,7 +15,6 @@ import { FeeRatioInitialInvestment } from '@ghostfolio/api/models/rules/fees/fee
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
-import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import {
   getAnnualizedPerformancePercent,
@@ -818,23 +817,6 @@ export class PortfolioService {
           })
       );
 
-      // Convert investment, gross and net performance to currency of user
-      const investment = this.exchangeRateDataService.toCurrency(
-        position.investment?.toNumber(),
-        currency,
-        userCurrency
-      );
-      const grossPerformance = this.exchangeRateDataService.toCurrency(
-        position.grossPerformance?.toNumber(),
-        currency,
-        userCurrency
-      );
-      const netPerformance = this.exchangeRateDataService.toCurrency(
-        position.netPerformance?.toNumber(),
-        currency,
-        userCurrency
-      );
-
       const historicalData = await this.dataProviderService.getHistorical(
         [{ dataSource, symbol: aSymbol }],
         'day',
@@ -1899,7 +1881,7 @@ export class PortfolioService {
       cashDetailsWithExcludedAccounts.balanceInBaseCurrency
     ).minus(balanceInBaseCurrency);
 
-    let excludedAccountsAndActivities = excludedBalanceInBaseCurrency
+    const excludedAccountsAndActivities = excludedBalanceInBaseCurrency
       .plus(totalOfExcludedActivities)
       .toNumber();
 

apps/api/src/helper/dateQueryHelper.ts

@@ -13,8 +13,8 @@ export class DateQueryHelper {
     let query = dateQuery;
     if (dateQuery.in?.length > 0) {
       dates = dateQuery.in;
-      let end = Math.max(...dates.map((d) => d.getTime()));
-      let start = Math.min(...dates.map((d) => d.getTime()));
+      const end = Math.max(...dates.map((d) => d.getTime()));
+      const start = Math.min(...dates.map((d) => d.getTime()));
       query = {
         gte: resetHours(new Date(start)),
         lt: resetHours(addDays(end, 1))

apps/api/src/services/configuration/configuration.service.ts

@@ -4,6 +4,7 @@ import {
   DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE,
   DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA,
   DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT,
+  DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT,
   DEFAULT_ROOT_URL
 } from '@ghostfolio/common/config';
 
@@ -59,6 +60,9 @@ export class ConfigurationService {
       PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT: num({
         default: DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT
       }),
+      PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: num({
+        default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT
+      }),
       REDIS_DB: num({ default: 0 }),
       REDIS_HOST: str({ default: 'localhost' }),
       REDIS_PASSWORD: str({ default: '' }),

apps/api/src/services/data-provider/data-provider.service.ts

@@ -475,7 +475,7 @@ export class DataProviderService {
               }
 
               response[symbol] = dataProviderResponse;
-              let quotesCacheTTL =
+              const quotesCacheTTL =
                 this.getAppropriateCacheTTL(dataProviderResponse);
 
               this.redisCacheService.set(
@@ -573,8 +573,8 @@ export class DataProviderService {
     if (dataProviderResponse.dataSource === 'MANUAL') {
       quotesCacheTTL = 14400; // 4h Cache for Manual Service
     } else if (dataProviderResponse.marketState === 'closed') {
-      let date = new Date();
-      let dayOfWeek = date.getDay();
+      const date = new Date();
+      const dayOfWeek = date.getDay();
       if (dayOfWeek === 0 || dayOfWeek === 6) {
         quotesCacheTTL = 14400;
       } else if (date.getHours() > 16) {

apps/api/src/services/market-data/market-data.service.ts

@@ -3,7 +3,6 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.i
 import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper';
 import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
-import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper';
 import { resetHours } from '@ghostfolio/common/helper';
 import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
 

apps/api/src/services/queues/data-gathering/data-gathering.processor.ts

@@ -1,20 +1,25 @@
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
-import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
+import {
+  IDataGatheringItem,
+  IDataProviderHistoricalResponse
+} from '@ghostfolio/api/services/interfaces/interfaces';
 import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import {
   DATA_GATHERING_QUEUE,
   DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE,
   DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA,
   GATHER_ASSET_PROFILE_PROCESS,
-  GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
+  GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME,
+  GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
 } from '@ghostfolio/common/config';
 import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
 import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
 
 import { Process, Processor } from '@nestjs/bull';
 import { Injectable, Logger } from '@nestjs/common';
-import { Prisma } from '@prisma/client';
+import { DataSource, Prisma } from '@prisma/client';
 import { Job } from 'bull';
+import { isNumber } from 'class-validator';
 import {
   addDays,
   format,
@@ -22,7 +27,9 @@ import {
   getMonth,
   getYear,
   isBefore,
-  parseISO
+  parseISO,
+  eachDayOfInterval,
+  isEqual
 } from 'date-fns';
 
 import { DataGatheringService } from './data-gathering.service';
@@ -150,4 +157,148 @@ export class DataGatheringProcessor {
       throw new Error(error);
     }
   }
+  @Process({
+    concurrency: parseInt(
+      process.env.PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA ??
+        DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA.toString(),
+      10
+    ),
+    name: GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
+  })
+  public async gatherMissingHistoricalMarketData(job: Job<IDataGatheringItem>) {
+    try {
+      const { dataSource, date, symbol } = job.data;
+
+      Logger.log(
+        `Historical market data gathering for missing values has been started for ${symbol} (${dataSource}) at ${format(
+          date,
+          DATE_FORMAT
+        )}`,
+        `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})`
+      );
+      const entries = await this.marketDataService.marketDataItems({
+        where: {
+          AND: {
+            symbol: {
+              equals: symbol
+            },
+            dataSource: {
+              equals: dataSource
+            }
+          }
+        },
+        orderBy: {
+          date: 'asc'
+        },
+        take: 1
+      });
+      const firstEntry = entries[0];
+      const marketData = await this.marketDataService
+        .getRange({
+          assetProfileIdentifiers: [{ dataSource, symbol }],
+          dateQuery: {
+            gte: addDays(firstEntry.date, -10)
+          }
+        })
+        .then((md) => md.map((m) => m.date));
+
+      let dates = eachDayOfInterval(
+        {
+          start: firstEntry.date,
+          end: new Date()
+        },
+        {
+          step: 1
+        }
+      );
+      dates = dates.filter((d) => !marketData.some((md) => isEqual(md, d)));
+
+      const historicalData = await this.dataProviderService.getHistoricalRaw({
+        dataGatheringItems: [{ dataSource, symbol }],
+        from: firstEntry.date,
+        to: new Date()
+      });
+
+      const data: Prisma.MarketDataUpdateInput[] =
+        this.mapToMarketUpsertDataInputs(
+          dates,
+          historicalData,
+          symbol,
+          dataSource
+        );
+
+      await this.marketDataService.updateMany({ data });
+
+      Logger.log(
+        `Historical market data gathering for missing values has been completed for ${symbol} (${dataSource}) at ${format(
+          date,
+          DATE_FORMAT
+        )}`,
+        `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})`
+      );
+    } catch (error) {
+      Logger.error(
+        error,
+        `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})`
+      );
+
+      throw new Error(error);
+    }
+  }
+
+  private mapToMarketUpsertDataInputs(
+    missingMarketData: Date[],
+    historicalData: Record<
+      string,
+      Record<string, IDataProviderHistoricalResponse>
+    >,
+    symbol: string,
+    dataSource: DataSource
+  ): Prisma.MarketDataUpdateInput[] {
+    return missingMarketData.map((date) => {
+      if (
+        isNumber(
+          historicalData[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice
+        )
+      ) {
+        return {
+          date,
+          symbol,
+          dataSource,
+          marketPrice:
+            historicalData[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice
+        };
+      } else {
+        let earlierDate = date;
+        let index = 0;
+        while (
+          !isNumber(
+            historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)]
+              ?.marketPrice
+          )
+        ) {
+          earlierDate = addDays(earlierDate, -1);
+          index++;
+          if (index > 10) {
+            break;
+          }
+        }
+        if (
+          isNumber(
+            historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)]
+              ?.marketPrice
+          )
+        ) {
+          return {
+            date,
+            symbol,
+            dataSource,
+            marketPrice:
+              historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)]
+                ?.marketPrice
+          };
+        }
+      }
+    });
+  }
 }

apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@@ -13,6 +13,8 @@ import {
   DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS,
+  GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME,
+  GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS,
   PROPERTY_BENCHMARKS
 } from '@ghostfolio/common/config';
 import {
@@ -28,7 +30,6 @@ import {
 import { InjectQueue } from '@nestjs/bull';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
-import AwaitLock from 'await-lock';
 import { JobOptions, Queue } from 'bull';
 import { format, min, subDays, subYears } from 'date-fns';
 import { isEmpty } from 'lodash';
@@ -48,8 +49,6 @@ export class DataGatheringService {
     private readonly symbolProfileService: SymbolProfileService
   ) {}
 
-  lock = new AwaitLock();
-
   public async addJobToQueue({
     data,
     name,
@@ -114,6 +113,24 @@ export class DataGatheringService {
     });
   }
 
+  public async gatherSymbolMissingOnly({
+    dataSource,
+    symbol
+  }: AssetProfileIdentifier) {
+    const dataGatheringItems = (await this.getSymbolsMax()).filter(
+      (dataGatheringItem) => {
+        return (
+          dataGatheringItem.dataSource === dataSource &&
+          dataGatheringItem.symbol === symbol
+        );
+      }
+    );
+    await this.gatherMissingDataSymbols({
+      dataGatheringItems,
+      priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+    });
+  }
+
   public async gatherSymbolForDate({
     dataSource,
     date,
@@ -296,6 +313,35 @@ export class DataGatheringService {
     );
   }
 
+  public async gatherMissingDataSymbols({
+    dataGatheringItems,
+    priority
+  }: {
+    dataGatheringItems: IDataGatheringItem[];
+    priority: number;
+  }) {
+    await this.addJobsToQueue(
+      dataGatheringItems.map(({ dataSource, date, symbol }) => {
+        return {
+          data: {
+            dataSource,
+            date,
+            symbol
+          },
+          name: GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME,
+          opts: {
+            ...GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS,
+            priority,
+            jobId: `${getAssetProfileIdentifier({
+              dataSource,
+              symbol
+            })}-missing-${format(date, DATE_FORMAT)}`
+          }
+        };
+      })
+    );
+  }
+
   public async getAllAssetProfileIdentifiers(): Promise<
     AssetProfileIdentifier[]
   > {

apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts

@@ -8,7 +8,10 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-
 import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
 import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
 import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service';
-import { PORTFOLIO_SNAPSHOT_QUEUE } from '@ghostfolio/common/config';
+import {
+  DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT,
+  PORTFOLIO_SNAPSHOT_QUEUE
+} from '@ghostfolio/common/config';
 
 import { BullModule } from '@nestjs/bull';
 import { Module } from '@nestjs/common';
@@ -20,7 +23,14 @@ import { PortfolioSnapshotProcessor } from './portfolio-snapshot.processor';
   imports: [
     AccountBalanceModule,
     BullModule.registerQueue({
-      name: PORTFOLIO_SNAPSHOT_QUEUE
+      name: PORTFOLIO_SNAPSHOT_QUEUE,
+      settings: {
+        lockDuration: parseInt(
+          process.env.PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT ??
+            DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT.toString(),
+          10
+        )
+      }
     }),
     ConfigurationModule,
     DataProviderModule,

apps/api/src/services/symbol-profile/symbol-profile-overwrite.service.ts

@@ -47,7 +47,7 @@ export class SymbolProfileOverwriteService {
     Symbol: string,
     datasource: DataSource
   ): Promise<string> {
-    let SymbolProfileId = await this.prismaService.symbolProfile
+    const SymbolProfileId = await this.prismaService.symbolProfile
       .findFirst({
         where: {
           symbol: Symbol,
@@ -56,7 +56,7 @@ export class SymbolProfileOverwriteService {
       })
       .then((s) => s.id);
 
-    let symbolProfileIdSaved = await this.prismaService.symbolProfileOverrides
+    const symbolProfileIdSaved = await this.prismaService.symbolProfileOverrides
       .findFirst({
         where: {
           symbolProfileId: SymbolProfileId

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts

@@ -223,6 +223,16 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
       .subscribe(() => {});
   }
 
+  public onGatherSymbolMissingOnly({
+    dataSource,
+    symbol
+  }: AssetProfileIdentifier) {
+    this.adminService
+      .gatherSymbolMissingOnly({ dataSource, symbol })
+      .pipe(takeUntil(this.unsubscribeSubject))
+      .subscribe(() => {});
+  }
+
   public onImportHistoricalData() {
     try {
       const marketData = csvToJson(

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html

@@ -31,6 +31,19 @@
       >
         <ng-container i18n>Gather Historical Data</ng-container>
       </button>
+      <button
+        mat-menu-item
+        type="button"
+        [disabled]="assetProfileForm.dirty"
+        (click)="
+          onGatherSymbolMissingOnly({
+            dataSource: data.dataSource,
+            symbol: data.symbol
+          })
+        "
+      >
+        <ng-container i18n>Gather Missing Historical Data</ng-container>
+      </button>
       <button
         mat-menu-item
         type="button"

apps/client/src/app/components/header/header.component.ts

@@ -174,9 +174,9 @@ export class HeaderComponent implements OnChanges {
     const userSetting: UpdateUserSettingDto = {};
 
     for (const filter of filters) {
-      let filtersType = this.getFilterType(filter.type);
+      const filtersType = this.getFilterType(filter.type);
 
-      let userFilters = filters
+      const userFilters = filters
         .filter((f) => f.type === filter.type && filter.id)
         .map((f) => f.id);
 
@@ -267,8 +267,19 @@ export class HeaderComponent implements OnChanges {
       this.settingsStorageService.getSetting(KEY_STAY_SIGNED_IN) === 'true'
     );
 
+    this.userService
+      .get()
+      .pipe(takeUntil(this.unsubscribeSubject))
+      .subscribe((user) => {
+        const userLanguage = user?.settings?.language;
+
+        if (userLanguage && document.documentElement.lang !== userLanguage) {
+          window.location.href = `../${userLanguage}`;
+        } else {
           this.router.navigate(['/']);
         }
+      });
+  }
 
   public ngOnDestroy() {
     this.unsubscribeSubject.next();

apps/client/src/app/pages/portfolio/analysis/analysis-page.component.ts

@@ -15,7 +15,6 @@ import { translate } from '@ghostfolio/ui/i18n';
 
 import { ChangeDetectorRef, Component, OnDestroy, OnInit } from '@angular/core';
 import { SymbolProfile } from '@prisma/client';
-import { Big } from 'big.js';
 import { differenceInDays } from 'date-fns';
 import { isNumber, sortBy } from 'lodash';
 import { DeviceDetectorService } from 'ngx-device-detector';
@@ -249,7 +248,6 @@ export class AnalysisPageComponent implements OnDestroy, OnInit {
             netPerformanceInPercentageWithCurrencyEffect,
             totalInvestmentValueWithCurrencyEffect,
             valueInPercentage,
-            timeWeightedPerformance,
             valueWithCurrencyEffect
           }
         ] of chart.entries()) {

apps/client/src/app/services/admin.service.ts

@@ -198,6 +198,22 @@ export class AdminService {
     return this.http.post<MarketData | void>(url, {});
   }
 
+  public gatherSymbolMissingOnly({
+    dataSource,
+    date,
+    symbol
+  }: AssetProfileIdentifier & {
+    date?: Date;
+  }) {
+    let url = `/api/v1/admin/gatherMissing/${dataSource}/${symbol}`;
+
+    if (date) {
+      url = `${url}/${format(date, DATE_FORMAT)}`;
+    }
+
+    return this.http.post<MarketData | void>(url, {});
+  }
+
   public fetchSymbolForDate({
     dataSource,
     dateString,

libs/common/src/lib/config.ts

@@ -51,6 +51,7 @@ export const DEFAULT_PAGE_SIZE = 50;
 export const DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE = 1;
 export const DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA = 1;
 export const DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT = 1;
+export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000;
 export const DEFAULT_ROOT_URL = 'https://localhost:4200';
 
 // USX is handled separately
@@ -95,6 +96,18 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = {
   removeOnComplete: true
 };
 
+export const GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME =
+  'GATHER_MISSING_HISTORICAL_MARKET_DATA';
+export const GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions =
+  {
+    attempts: 12,
+    backoff: {
+      delay: ms('1 minute'),
+      type: 'exponential'
+    },
+    removeOnComplete: true
+  };
+
 export const PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME = 'PORTFOLIO';
 export const PORTFOLIO_SNAPSHOT_PROCESS_JOB_OPTIONS: JobOptions = {
   removeOnComplete: true

libs/ui/src/lib/assistant/assistant.component.ts

@@ -322,21 +322,21 @@ export class GfAssistantComponent implements OnChanges, OnDestroy, OnInit {
   }
 
   public onApplyFilters() {
-    let accountFilters =
+    const accountFilters =
       this.filterForm
         .get('account')
         .value?.reduce(
           (arr, val) => [...arr, { id: val, type: 'ACCOUNT' }],
           []
         ) ?? [];
-    let assetClassFilters =
+    const assetClassFilters =
       this.filterForm
         .get('assetClass')
         .value?.reduce(
           (arr, val) => [...arr, { id: val, type: 'ASSET_CLASS' }],
           []
         ) ?? [];
-    let tagFilters =
+    const tagFilters =
       this.filterForm
         .get('tag')
         .value?.reduce((arr, val) => [...arr, { id: val, type: 'TAG' }], []) ??

libs/ui/src/lib/treemap-chart/treemap-chart.component.ts

@@ -227,16 +227,24 @@ export class GfTreemapChartComponent
       }),
       callbacks: {
         label: (context) => {
+          const name = context.raw._data.name;
+          const symbol = context.raw._data.symbol;
+
           if (context.raw._data.valueInBaseCurrency !== null) {
             const value = <number>context.raw._data.valueInBaseCurrency;
-            return `${value.toLocaleString(this.locale, {
+
+            return [
+              `${name ?? symbol}`,
+              `${value.toLocaleString(this.locale, {
                 maximumFractionDigits: 2,
                 minimumFractionDigits: 2
-            })} ${this.baseCurrency}`;
+              })} ${this.baseCurrency}`
+            ];
           } else {
             const percentage =
               <number>context.raw._data.allocationInPercentage * 100;
-            return `${percentage.toFixed(2)}%`;
+
+            return [`${name ?? symbol}`, `${percentage.toFixed(2)}%`];
           }
         },
         title: () => {

test/import/ok-novn-buy-and-sell.json

@@ -11,7 +11,7 @@
       "unitPrice": 85.73,
       "currency": "CHF",
       "dataSource": "YAHOO",
-      "date": "2022-04-07T22:00:00.000Z",
+      "date": "2022-04-08T00:00:00.000Z",
       "symbol": "NOVN.SW"
     },
     {
@@ -21,7 +21,7 @@
       "unitPrice": 75.8,
       "currency": "CHF",
       "dataSource": "YAHOO",
-      "date": "2022-03-06T23:00:00.000Z",
+      "date": "2022-03-07T00:00:00.000Z",
       "symbol": "NOVN.SW"
     }
   ]