Browse Source

Merge pull request #129 from dandevaud/feature/Add-Gather-Missing-data-only

Add Datagathering process for missing values only
pull/5027/head
dandevaud 9 months ago
committed by GitHub
parent
commit
2cced2e520
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 15
      CHANGELOG.md
  2. 17
      apps/api/src/app/admin/admin.controller.ts
  3. 4
      apps/api/src/app/admin/admin.service.ts
  4. 43
      apps/api/src/app/portfolio/calculator/constantPortfolioReturn/portfolio-calculator.ts
  5. 6
      apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts
  6. 1
      apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts
  7. 18
      apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts
  8. 48
      apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts
  9. 4
      apps/api/src/app/portfolio/portfolio.controller.ts
  10. 20
      apps/api/src/app/portfolio/portfolio.service.ts
  11. 4
      apps/api/src/helper/dateQueryHelper.ts
  12. 4
      apps/api/src/services/configuration/configuration.service.ts
  13. 6
      apps/api/src/services/data-provider/data-provider.service.ts
  14. 1
      apps/api/src/services/market-data/market-data.service.ts
  15. 159
      apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
  16. 52
      apps/api/src/services/queues/data-gathering/data-gathering.service.ts
  17. 14
      apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts
  18. 4
      apps/api/src/services/symbol-profile/symbol-profile-overwrite.service.ts
  19. 10
      apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
  20. 13
      apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
  21. 15
      apps/client/src/app/components/header/header.component.ts
  22. 2
      apps/client/src/app/pages/portfolio/analysis/analysis-page.component.ts
  23. 16
      apps/client/src/app/services/admin.service.ts
  24. 13
      libs/common/src/lib/config.ts
  25. 6
      libs/ui/src/lib/assistant/assistant.component.ts
  26. 14
      libs/ui/src/lib/treemap-chart/treemap-chart.component.ts
  27. 4
      test/import/ok-novn-buy-and-sell.json

15
CHANGELOG.md

@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## Unreleased
### Added
- Added the name to the tooltip of the chart of the holdings tab on the home page (experimental)
### Changed
- Exposed the timeout of the portfolio snapshot computation as an environment variable (`PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT`)
- Improved the portfolio unit tests to work with exported activity files
### Fixed
- Considered the language of the user settings on login with _Security Token_
## 2.114.0 - 2024-10-10
### Added

17
apps/api/src/app/admin/admin.controller.ts

@ -158,7 +158,22 @@ export class AdminController {
@Param('dataSource') dataSource: DataSource,
@Param('symbol') symbol: string
): Promise<void> {
this.dataGatheringService.gatherSymbol({ dataSource, symbol });
await this.dataGatheringService.gatherSymbol({ dataSource, symbol });
return;
}
@Post('gatherMissing/:dataSource/:symbol')
@UseGuards(AuthGuard('jwt'), HasPermissionGuard)
@HasPermission(permissions.accessAdminControl)
public async gatherSymbolMissingOnly(
@Param('dataSource') dataSource: DataSource,
@Param('symbol') symbol: string
): Promise<void> {
await this.dataGatheringService.gatherSymbolMissingOnly({
dataSource,
symbol
});
return;
}

4
apps/api/src/app/admin/admin.service.ts

@ -41,9 +41,7 @@ import {
PrismaClient,
Property,
SymbolProfile,
DataSource,
Tag,
SymbolProfileOverrides
DataSource
} from '@prisma/client';
import { differenceInDays } from 'date-fns';
import { groupBy } from 'lodash';

43
apps/api/src/app/portfolio/calculator/constantPortfolioReturn/portfolio-calculator.ts

@ -7,16 +7,13 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con
import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service';
import { getIntervalFromDateRange } from '@ghostfolio/common/calculation-helper';
import { DATE_FORMAT, parseDate, resetHours } from '@ghostfolio/common/helper';
import { Filter, HistoricalDataItem } from '@ghostfolio/common/interfaces';
import { DateRange } from '@ghostfolio/common/types';
import { Inject, Logger } from '@nestjs/common';
import { Big } from 'big.js';
import {
addDays,
differenceInDays,
eachDayOfInterval,
endOfDay,
format,
@ -83,19 +80,19 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
start: Date;
end: Date;
}): Promise<{ chart: HistoricalDataItem[] }> {
let item = await super.getPerformance({
const item = await super.getPerformance({
end,
start
});
let itemResult = item.chart;
let dates = itemResult.map((item) => parseDate(item.date));
let timeWeighted = await this.getTimeWeightedChartData({
const itemResult = item.chart;
const dates = itemResult.map((item) => parseDate(item.date));
const timeWeighted = await this.getTimeWeightedChartData({
dates
});
item.chart = itemResult.map((itemInt) => {
let timeWeightedItem = timeWeighted.find(
const timeWeightedItem = timeWeighted.find(
(timeWeightedItem) => timeWeightedItem.date === itemInt.date
);
if (timeWeightedItem) {
@ -135,11 +132,11 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
dateQuery: { in: [end] }
});
const endString = format(end, DATE_FORMAT);
let exchangeRates = await Promise.all(
const exchangeRates = await Promise.all(
Object.keys(holdings[endString]).map(async (holding) => {
let symbol = marketMap.values.find((m) => m.symbol === holding);
let symbolCurrency = this.getCurrencyFromActivities(orders, holding);
let exchangeRate = await this.exchangeRateDataService.toCurrencyAtDate(
const symbolCurrency = this.getCurrencyFromActivities(orders, holding);
const exchangeRate =
await this.exchangeRateDataService.toCurrencyAtDate(
1,
symbolCurrency,
this.currency,
@ -148,7 +145,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
return { symbolCurrency, exchangeRate };
})
);
let currencyRates = exchangeRates.reduce<{ [currency: string]: number }>(
const currencyRates = exchangeRates.reduce<{ [currency: string]: number }>(
(all, currency): { [currency: string]: number } => {
all[currency.symbolCurrency] ??= currency.exchangeRate;
return all;
@ -156,12 +153,12 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
{}
);
let totalInvestment = await Object.keys(holdings[endString]).reduce(
const totalInvestment = await Object.keys(holdings[endString]).reduce(
(sum, holding) => {
if (!holdings[endString][holding].toNumber()) {
return sum;
}
let symbol = marketMap.values.find((m) => m.symbol === holding);
const symbol = marketMap.values.find((m) => m.symbol === holding);
if (symbol?.marketPrice === undefined) {
Logger.warn(
@ -170,8 +167,8 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
);
return sum;
} else {
let symbolCurrency = this.getCurrency(holding);
let price = new Big(currencyRates[symbolCurrency]).mul(
const symbolCurrency = this.getCurrency(holding);
const price = new Big(currencyRates[symbolCurrency]).mul(
symbol.marketPrice
);
return sum.plus(new Big(price).mul(holdings[endString][holding]));
@ -191,7 +188,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
dates = dates.sort((a, b) => a.getTime() - b.getTime());
const start = dates[0];
const end = dates[dates.length - 1];
let marketMapTask = this.computeMarketMap({
const marketMapTask = this.computeMarketMap({
gte: start,
lt: addDays(end, 1)
});
@ -201,7 +198,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
end
);
let data: HistoricalDataItem[] = [];
const data: HistoricalDataItem[] = [];
const startString = format(start, DATE_FORMAT);
data.push({
@ -435,12 +432,12 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
end: Date
) {
const transactionDates = Object.keys(investmentByDate).sort();
let dates = eachDayOfInterval({ start, end }, { step: 1 })
const dates = eachDayOfInterval({ start, end }, { step: 1 })
.map((date) => {
return resetHours(date);
})
.sort((a, b) => a.getTime() - b.getTime());
let currentHoldings: { [date: string]: { [symbol: string]: Big } } = {};
const currentHoldings: { [date: string]: { [symbol: string]: Big } } = {};
this.calculateInitialHoldings(investmentByDate, start, currentHoldings);
@ -448,7 +445,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
const dateString = format(dates[i], DATE_FORMAT);
const previousDateString = format(dates[i - 1], DATE_FORMAT);
if (transactionDates.some((d) => d === dateString)) {
let holdings = { ...currentHoldings[previousDateString] };
const holdings = { ...currentHoldings[previousDateString] };
investmentByDate[dateString].forEach((trade) => {
holdings[trade.SymbolProfile.symbol] ??= new Big(0);
holdings[trade.SymbolProfile.symbol] = holdings[
@ -488,7 +485,7 @@ export class CPRPortfolioCalculator extends TWRPortfolioCalculator {
for (const symbol of Object.keys(preRangeTrades)) {
const trades: PortfolioOrder[] = preRangeTrades[symbol];
let startQuantity = trades.reduce((sum, trade) => {
const startQuantity = trades.reduce((sum, trade) => {
return sum.plus(trade.quantity.mul(getFactor(trade.type)));
}, new Big(0));
currentHoldings[format(start, DATE_FORMAT)][symbol] = startQuantity;

6
apps/api/src/app/portfolio/calculator/portfolio-calculator-test-utils.ts

@ -1,3 +1,5 @@
import { readFileSync } from 'fs';
export const activityDummyData = {
accountId: undefined,
accountUserId: undefined,
@ -29,3 +31,7 @@ export const symbolProfileDummyData = {
export const userDummyData = {
id: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
};
export function loadActivityExportFile(filePath: string) {
return JSON.parse(readFileSync(filePath, 'utf8')).activities;
}

1
apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts

@ -13,7 +13,6 @@ import { OrderService } from '../../order/order.service';
import { CPRPortfolioCalculator } from './constantPortfolioReturn/portfolio-calculator';
import { MWRPortfolioCalculator } from './mwr/portfolio-calculator';
import { PortfolioCalculator } from './portfolio-calculator';
import { TWRPortfolioCalculator } from './twr/portfolio-calculator';
export enum PerformanceCalculationType {
MWR = 'MWR', // Money-Weighted Rate of Return

18
apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts

@ -156,25 +156,25 @@ describe('PortfolioCalculator', () => {
dividendInBaseCurrency: new Big('0.62'),
fee: new Big('19'),
firstBuyDate: '2021-09-16',
grossPerformance: new Big('33.25'),
grossPerformancePercentage: new Big('0.11136043941322258691'),
grossPerformance: new Big('33.87'),
grossPerformancePercentage: new Big('0.11343693482483756447'),
grossPerformancePercentageWithCurrencyEffect: new Big(
'0.11136043941322258691'
'0.11343693482483756447'
),
grossPerformanceWithCurrencyEffect: new Big('33.25'),
grossPerformanceWithCurrencyEffect: new Big('33.87'),
investment: new Big('298.58'),
investmentWithCurrencyEffect: new Big('298.58'),
marketPrice: 331.83,
marketPriceInBaseCurrency: 331.83,
netPerformance: new Big('14.25'),
netPerformancePercentage: new Big('0.04772590260566682296'),
netPerformance: new Big('14.87'),
netPerformancePercentage: new Big('0.04980239801728180052'),
netPerformancePercentageWithCurrencyEffectMap: {
max: new Big('0.04772590260566682296')
max: new Big('0.04980239801728180052')
},
netPerformanceWithCurrencyEffectMap: {
'1d': new Big('-5.39'),
'5y': new Big('14.25'),
max: new Big('14.25'),
'5y': new Big('14.87'),
max: new Big('14.87'),
wtd: new Big('-5.39')
},
quantity: new Big('1'),

48
apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts

@ -1,6 +1,8 @@
import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
import {
activityDummyData,
loadActivityExportFile,
symbolProfileDummyData,
userDummyData
} from '@ghostfolio/api/app/portfolio/calculator/portfolio-calculator-test-utils';
@ -20,6 +22,7 @@ import { parseDate } from '@ghostfolio/common/helper';
import { Big } from 'big.js';
import { last } from 'lodash';
import { join } from 'path';
jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => {
return {
@ -52,6 +55,8 @@ jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => {
});
describe('PortfolioCalculator', () => {
let activityDtos: CreateOrderDto[];
let configurationService: ConfigurationService;
let currentRateService: CurrentRateService;
let exchangeRateDataService: ExchangeRateDataService;
@ -59,6 +64,15 @@ describe('PortfolioCalculator', () => {
let portfolioSnapshotService: PortfolioSnapshotService;
let redisCacheService: RedisCacheService;
beforeAll(() => {
activityDtos = loadActivityExportFile(
join(
__dirname,
'../../../../../../../test/import/ok-novn-buy-and-sell.json'
)
);
});
beforeEach(() => {
configurationService = new ConfigurationService();
@ -89,38 +103,18 @@ describe('PortfolioCalculator', () => {
it.only('with NOVN.SW buy and sell', async () => {
jest.useFakeTimers().setSystemTime(parseDate('2022-04-11').getTime());
const activities: Activity[] = [
{
const activities: Activity[] = activityDtos.map((activity) => ({
...activityDummyData,
date: new Date('2022-03-07'),
fee: 0,
quantity: 2,
...activity,
date: parseDate(activity.date),
SymbolProfile: {
...symbolProfileDummyData,
currency: 'CHF',
dataSource: 'YAHOO',
currency: activity.currency,
dataSource: activity.dataSource,
name: 'Novartis AG',
symbol: 'NOVN.SW'
},
type: 'BUY',
unitPrice: 75.8
},
{
...activityDummyData,
date: new Date('2022-04-08'),
fee: 0,
quantity: 2,
SymbolProfile: {
...symbolProfileDummyData,
currency: 'CHF',
dataSource: 'YAHOO',
name: 'Novartis AG',
symbol: 'NOVN.SW'
},
type: 'SELL',
unitPrice: 85.73
symbol: activity.symbol
}
];
}));
const portfolioCalculator = portfolioCalculatorFactory.createCalculator({
activities,

4
apps/api/src/app/portfolio/portfolio.controller.ts

@ -84,7 +84,6 @@ export class PortfolioController {
@Query('assetClasses') filterByAssetClasses?: string,
@Query('range') dateRange: DateRange = 'max',
@Query('tags') filterByTags?: string,
@Query('isAllocation') isAllocation: boolean = false,
@Query('withMarkets') withMarketsParam = 'false'
): Promise<PortfolioDetails & { hasError: boolean }> {
const withMarkets = withMarketsParam === 'true';
@ -483,8 +482,7 @@ export class PortfolioController {
@Query('range') dateRange: DateRange = 'max',
@Query('tags') filterByTags?: string,
@Query('withExcludedAccounts') withExcludedAccounts = false,
@Query('timeWeightedPerformance') calculateTimeWeightedPerformance = false,
@Query('withItems') withItems = false
@Query('timeWeightedPerformance') calculateTimeWeightedPerformance = false
): Promise<PortfolioPerformanceResponse> {
const filters = this.apiService.buildFiltersFromQueryParams({
filterByAccounts,

20
apps/api/src/app/portfolio/portfolio.service.ts

@ -15,7 +15,6 @@ import { FeeRatioInitialInvestment } from '@ghostfolio/api/models/rules/fees/fee
import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import {
getAnnualizedPerformancePercent,
@ -818,23 +817,6 @@ export class PortfolioService {
})
);
// Convert investment, gross and net performance to currency of user
const investment = this.exchangeRateDataService.toCurrency(
position.investment?.toNumber(),
currency,
userCurrency
);
const grossPerformance = this.exchangeRateDataService.toCurrency(
position.grossPerformance?.toNumber(),
currency,
userCurrency
);
const netPerformance = this.exchangeRateDataService.toCurrency(
position.netPerformance?.toNumber(),
currency,
userCurrency
);
const historicalData = await this.dataProviderService.getHistorical(
[{ dataSource, symbol: aSymbol }],
'day',
@ -1899,7 +1881,7 @@ export class PortfolioService {
cashDetailsWithExcludedAccounts.balanceInBaseCurrency
).minus(balanceInBaseCurrency);
let excludedAccountsAndActivities = excludedBalanceInBaseCurrency
const excludedAccountsAndActivities = excludedBalanceInBaseCurrency
.plus(totalOfExcludedActivities)
.toNumber();

4
apps/api/src/helper/dateQueryHelper.ts

@ -13,8 +13,8 @@ export class DateQueryHelper {
let query = dateQuery;
if (dateQuery.in?.length > 0) {
dates = dateQuery.in;
let end = Math.max(...dates.map((d) => d.getTime()));
let start = Math.min(...dates.map((d) => d.getTime()));
const end = Math.max(...dates.map((d) => d.getTime()));
const start = Math.min(...dates.map((d) => d.getTime()));
query = {
gte: resetHours(new Date(start)),
lt: resetHours(addDays(end, 1))

4
apps/api/src/services/configuration/configuration.service.ts

@ -4,6 +4,7 @@ import {
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE,
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA,
DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT,
DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT,
DEFAULT_ROOT_URL
} from '@ghostfolio/common/config';
@ -59,6 +60,9 @@ export class ConfigurationService {
PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT: num({
default: DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT
}),
PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: num({
default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT
}),
REDIS_DB: num({ default: 0 }),
REDIS_HOST: str({ default: 'localhost' }),
REDIS_PASSWORD: str({ default: '' }),

6
apps/api/src/services/data-provider/data-provider.service.ts

@ -475,7 +475,7 @@ export class DataProviderService {
}
response[symbol] = dataProviderResponse;
let quotesCacheTTL =
const quotesCacheTTL =
this.getAppropriateCacheTTL(dataProviderResponse);
this.redisCacheService.set(
@ -573,8 +573,8 @@ export class DataProviderService {
if (dataProviderResponse.dataSource === 'MANUAL') {
quotesCacheTTL = 14400; // 4h Cache for Manual Service
} else if (dataProviderResponse.marketState === 'closed') {
let date = new Date();
let dayOfWeek = date.getDay();
const date = new Date();
const dayOfWeek = date.getDay();
if (dayOfWeek === 0 || dayOfWeek === 6) {
quotesCacheTTL = 14400;
} else if (date.getHours() > 16) {

1
apps/api/src/services/market-data/market-data.service.ts

@ -3,7 +3,6 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.i
import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper';
import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper';
import { resetHours } from '@ghostfolio/common/helper';
import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';

159
apps/api/src/services/queues/data-gathering/data-gathering.processor.ts

@ -1,20 +1,25 @@
import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
import {
IDataGatheringItem,
IDataProviderHistoricalResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
import {
DATA_GATHERING_QUEUE,
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE,
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME,
GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
} from '@ghostfolio/common/config';
import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
import { Process, Processor } from '@nestjs/bull';
import { Injectable, Logger } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { DataSource, Prisma } from '@prisma/client';
import { Job } from 'bull';
import { isNumber } from 'class-validator';
import {
addDays,
format,
@ -22,7 +27,9 @@ import {
getMonth,
getYear,
isBefore,
parseISO
parseISO,
eachDayOfInterval,
isEqual
} from 'date-fns';
import { DataGatheringService } from './data-gathering.service';
@ -150,4 +157,148 @@ export class DataGatheringProcessor {
throw new Error(error);
}
}
@Process({
concurrency: parseInt(
process.env.PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA ??
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA.toString(),
10
),
name: GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
})
public async gatherMissingHistoricalMarketData(job: Job<IDataGatheringItem>) {
try {
const { dataSource, date, symbol } = job.data;
Logger.log(
`Historical market data gathering for missing values has been started for ${symbol} (${dataSource}) at ${format(
date,
DATE_FORMAT
)}`,
`DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})`
);
const entries = await this.marketDataService.marketDataItems({
where: {
AND: {
symbol: {
equals: symbol
},
dataSource: {
equals: dataSource
}
}
},
orderBy: {
date: 'asc'
},
take: 1
});
const firstEntry = entries[0];
const marketData = await this.marketDataService
.getRange({
assetProfileIdentifiers: [{ dataSource, symbol }],
dateQuery: {
gte: addDays(firstEntry.date, -10)
}
})
.then((md) => md.map((m) => m.date));
let dates = eachDayOfInterval(
{
start: firstEntry.date,
end: new Date()
},
{
step: 1
}
);
dates = dates.filter((d) => !marketData.some((md) => isEqual(md, d)));
const historicalData = await this.dataProviderService.getHistoricalRaw({
dataGatheringItems: [{ dataSource, symbol }],
from: firstEntry.date,
to: new Date()
});
const data: Prisma.MarketDataUpdateInput[] =
this.mapToMarketUpsertDataInputs(
dates,
historicalData,
symbol,
dataSource
);
await this.marketDataService.updateMany({ data });
Logger.log(
`Historical market data gathering for missing values has been completed for ${symbol} (${dataSource}) at ${format(
date,
DATE_FORMAT
)}`,
`DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})`
);
} catch (error) {
Logger.error(
error,
`DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})`
);
throw new Error(error);
}
}
private mapToMarketUpsertDataInputs(
missingMarketData: Date[],
historicalData: Record<
string,
Record<string, IDataProviderHistoricalResponse>
>,
symbol: string,
dataSource: DataSource
): Prisma.MarketDataUpdateInput[] {
return missingMarketData.map((date) => {
if (
isNumber(
historicalData[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice
)
) {
return {
date,
symbol,
dataSource,
marketPrice:
historicalData[symbol]?.[format(date, DATE_FORMAT)]?.marketPrice
};
} else {
let earlierDate = date;
let index = 0;
while (
!isNumber(
historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)]
?.marketPrice
)
) {
earlierDate = addDays(earlierDate, -1);
index++;
if (index > 10) {
break;
}
}
if (
isNumber(
historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)]
?.marketPrice
)
) {
return {
date,
symbol,
dataSource,
marketPrice:
historicalData[symbol]?.[format(earlierDate, DATE_FORMAT)]
?.marketPrice
};
}
}
});
}
}

52
apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@ -13,6 +13,8 @@ import {
DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS,
GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME,
GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS,
PROPERTY_BENCHMARKS
} from '@ghostfolio/common/config';
import {
@ -28,7 +30,6 @@ import {
import { InjectQueue } from '@nestjs/bull';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource } from '@prisma/client';
import AwaitLock from 'await-lock';
import { JobOptions, Queue } from 'bull';
import { format, min, subDays, subYears } from 'date-fns';
import { isEmpty } from 'lodash';
@ -48,8 +49,6 @@ export class DataGatheringService {
private readonly symbolProfileService: SymbolProfileService
) {}
lock = new AwaitLock();
public async addJobToQueue({
data,
name,
@ -114,6 +113,24 @@ export class DataGatheringService {
});
}
public async gatherSymbolMissingOnly({
dataSource,
symbol
}: AssetProfileIdentifier) {
const dataGatheringItems = (await this.getSymbolsMax()).filter(
(dataGatheringItem) => {
return (
dataGatheringItem.dataSource === dataSource &&
dataGatheringItem.symbol === symbol
);
}
);
await this.gatherMissingDataSymbols({
dataGatheringItems,
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
});
}
public async gatherSymbolForDate({
dataSource,
date,
@ -296,6 +313,35 @@ export class DataGatheringService {
);
}
public async gatherMissingDataSymbols({
dataGatheringItems,
priority
}: {
dataGatheringItems: IDataGatheringItem[];
priority: number;
}) {
await this.addJobsToQueue(
dataGatheringItems.map(({ dataSource, date, symbol }) => {
return {
data: {
dataSource,
date,
symbol
},
name: GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME,
opts: {
...GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS,
priority,
jobId: `${getAssetProfileIdentifier({
dataSource,
symbol
})}-missing-${format(date, DATE_FORMAT)}`
}
};
})
);
}
public async getAllAssetProfileIdentifiers(): Promise<
AssetProfileIdentifier[]
> {

14
apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts

@ -8,7 +8,10 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-
import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service';
import { PORTFOLIO_SNAPSHOT_QUEUE } from '@ghostfolio/common/config';
import {
DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT,
PORTFOLIO_SNAPSHOT_QUEUE
} from '@ghostfolio/common/config';
import { BullModule } from '@nestjs/bull';
import { Module } from '@nestjs/common';
@ -20,7 +23,14 @@ import { PortfolioSnapshotProcessor } from './portfolio-snapshot.processor';
imports: [
AccountBalanceModule,
BullModule.registerQueue({
name: PORTFOLIO_SNAPSHOT_QUEUE
name: PORTFOLIO_SNAPSHOT_QUEUE,
settings: {
lockDuration: parseInt(
process.env.PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT ??
DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT.toString(),
10
)
}
}),
ConfigurationModule,
DataProviderModule,

4
apps/api/src/services/symbol-profile/symbol-profile-overwrite.service.ts

@ -47,7 +47,7 @@ export class SymbolProfileOverwriteService {
Symbol: string,
datasource: DataSource
): Promise<string> {
let SymbolProfileId = await this.prismaService.symbolProfile
const SymbolProfileId = await this.prismaService.symbolProfile
.findFirst({
where: {
symbol: Symbol,
@ -56,7 +56,7 @@ export class SymbolProfileOverwriteService {
})
.then((s) => s.id);
let symbolProfileIdSaved = await this.prismaService.symbolProfileOverrides
const symbolProfileIdSaved = await this.prismaService.symbolProfileOverrides
.findFirst({
where: {
symbolProfileId: SymbolProfileId

10
apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts

@ -223,6 +223,16 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
.subscribe(() => {});
}
public onGatherSymbolMissingOnly({
dataSource,
symbol
}: AssetProfileIdentifier) {
this.adminService
.gatherSymbolMissingOnly({ dataSource, symbol })
.pipe(takeUntil(this.unsubscribeSubject))
.subscribe(() => {});
}
public onImportHistoricalData() {
try {
const marketData = csvToJson(

13
apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html

@ -31,6 +31,19 @@
>
<ng-container i18n>Gather Historical Data</ng-container>
</button>
<button
mat-menu-item
type="button"
[disabled]="assetProfileForm.dirty"
(click)="
onGatherSymbolMissingOnly({
dataSource: data.dataSource,
symbol: data.symbol
})
"
>
<ng-container i18n>Gather Missing Historical Data</ng-container>
</button>
<button
mat-menu-item
type="button"

15
apps/client/src/app/components/header/header.component.ts

@ -174,9 +174,9 @@ export class HeaderComponent implements OnChanges {
const userSetting: UpdateUserSettingDto = {};
for (const filter of filters) {
let filtersType = this.getFilterType(filter.type);
const filtersType = this.getFilterType(filter.type);
let userFilters = filters
const userFilters = filters
.filter((f) => f.type === filter.type && filter.id)
.map((f) => f.id);
@ -267,8 +267,19 @@ export class HeaderComponent implements OnChanges {
this.settingsStorageService.getSetting(KEY_STAY_SIGNED_IN) === 'true'
);
this.userService
.get()
.pipe(takeUntil(this.unsubscribeSubject))
.subscribe((user) => {
const userLanguage = user?.settings?.language;
if (userLanguage && document.documentElement.lang !== userLanguage) {
window.location.href = `../${userLanguage}`;
} else {
this.router.navigate(['/']);
}
});
}
public ngOnDestroy() {
this.unsubscribeSubject.next();

2
apps/client/src/app/pages/portfolio/analysis/analysis-page.component.ts

@ -15,7 +15,6 @@ import { translate } from '@ghostfolio/ui/i18n';
import { ChangeDetectorRef, Component, OnDestroy, OnInit } from '@angular/core';
import { SymbolProfile } from '@prisma/client';
import { Big } from 'big.js';
import { differenceInDays } from 'date-fns';
import { isNumber, sortBy } from 'lodash';
import { DeviceDetectorService } from 'ngx-device-detector';
@ -249,7 +248,6 @@ export class AnalysisPageComponent implements OnDestroy, OnInit {
netPerformanceInPercentageWithCurrencyEffect,
totalInvestmentValueWithCurrencyEffect,
valueInPercentage,
timeWeightedPerformance,
valueWithCurrencyEffect
}
] of chart.entries()) {

16
apps/client/src/app/services/admin.service.ts

@ -198,6 +198,22 @@ export class AdminService {
return this.http.post<MarketData | void>(url, {});
}
public gatherSymbolMissingOnly({
dataSource,
date,
symbol
}: AssetProfileIdentifier & {
date?: Date;
}) {
let url = `/api/v1/admin/gatherMissing/${dataSource}/${symbol}`;
if (date) {
url = `${url}/${format(date, DATE_FORMAT)}`;
}
return this.http.post<MarketData | void>(url, {});
}
public fetchSymbolForDate({
dataSource,
dateString,

13
libs/common/src/lib/config.ts

@ -51,6 +51,7 @@ export const DEFAULT_PAGE_SIZE = 50;
export const DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE = 1;
export const DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA = 1;
export const DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT = 1;
export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000;
export const DEFAULT_ROOT_URL = 'https://localhost:4200';
// USX is handled separately
@ -95,6 +96,18 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = {
removeOnComplete: true
};
export const GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME =
'GATHER_MISSING_HISTORICAL_MARKET_DATA';
export const GATHER_MISSING_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions =
{
attempts: 12,
backoff: {
delay: ms('1 minute'),
type: 'exponential'
},
removeOnComplete: true
};
export const PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME = 'PORTFOLIO';
export const PORTFOLIO_SNAPSHOT_PROCESS_JOB_OPTIONS: JobOptions = {
removeOnComplete: true

6
libs/ui/src/lib/assistant/assistant.component.ts

@ -322,21 +322,21 @@ export class GfAssistantComponent implements OnChanges, OnDestroy, OnInit {
}
public onApplyFilters() {
let accountFilters =
const accountFilters =
this.filterForm
.get('account')
.value?.reduce(
(arr, val) => [...arr, { id: val, type: 'ACCOUNT' }],
[]
) ?? [];
let assetClassFilters =
const assetClassFilters =
this.filterForm
.get('assetClass')
.value?.reduce(
(arr, val) => [...arr, { id: val, type: 'ASSET_CLASS' }],
[]
) ?? [];
let tagFilters =
const tagFilters =
this.filterForm
.get('tag')
.value?.reduce((arr, val) => [...arr, { id: val, type: 'TAG' }], []) ??

14
libs/ui/src/lib/treemap-chart/treemap-chart.component.ts

@ -227,16 +227,24 @@ export class GfTreemapChartComponent
}),
callbacks: {
label: (context) => {
const name = context.raw._data.name;
const symbol = context.raw._data.symbol;
if (context.raw._data.valueInBaseCurrency !== null) {
const value = <number>context.raw._data.valueInBaseCurrency;
return `${value.toLocaleString(this.locale, {
return [
`${name ?? symbol}`,
`${value.toLocaleString(this.locale, {
maximumFractionDigits: 2,
minimumFractionDigits: 2
})} ${this.baseCurrency}`;
})} ${this.baseCurrency}`
];
} else {
const percentage =
<number>context.raw._data.allocationInPercentage * 100;
return `${percentage.toFixed(2)}%`;
return [`${name ?? symbol}`, `${percentage.toFixed(2)}%`];
}
},
title: () => {

4
test/import/ok-novn-buy-and-sell.json

@ -11,7 +11,7 @@
"unitPrice": 85.73,
"currency": "CHF",
"dataSource": "YAHOO",
"date": "2022-04-07T22:00:00.000Z",
"date": "2022-04-08T00:00:00.000Z",
"symbol": "NOVN.SW"
},
{
@ -21,7 +21,7 @@
"unitPrice": 75.8,
"currency": "CHF",
"dataSource": "YAHOO",
"date": "2022-03-06T23:00:00.000Z",
"date": "2022-03-07T00:00:00.000Z",
"symbol": "NOVN.SW"
}
]

Loading…
Cancel
Save