diff --git a/CHANGELOG.md b/CHANGELOG.md index 90430f5c4..a7b13801f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Ignored nested ETFs when fetching top holdings for ETF and mutual fund assets from _Yahoo Finance_ +- Improved the scraper configuration with more detailed error messages +- Upgraded `cheerio` from version `1.0.0` to `1.2.0` + +### Fixed + +- Added the missing `valueInBaseCurrency` to the response of the import activities endpoint + ## 2.238.0 - 2026-02-12 ### Changed diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 24467c732..8a202a926 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -247,14 +247,17 @@ export class AdminController { @Param('symbol') symbol: string ): Promise<{ price: number }> { try { - const price = await this.manualService.test(data.scraperConfiguration); + const price = await this.manualService.test({ + symbol, + scraperConfiguration: data.scraperConfiguration + }); if (price) { return { price }; } throw new Error( - `Could not parse the current market price for ${symbol} (${dataSource})` + `Could not parse the market price for ${symbol} (${dataSource})` ); } catch (error) { Logger.error(error, 'AdminController'); diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index 7e8e333b9..a787927b5 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -5,6 +5,7 @@ import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.servic import { ApiService } from '@ghostfolio/api/services/api/api.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; +import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; @@ -48,6 +49,7 @@ export class ImportService { private readonly configurationService: ConfigurationService, private readonly dataGatheringService: DataGatheringService, private readonly dataProviderService: DataProviderService, + private readonly exchangeRateDataService: ExchangeRateDataService, private readonly marketDataService: MarketDataService, private readonly orderService: OrderService, private readonly platformService: PlatformService, @@ -590,10 +592,18 @@ export class ImportService { const value = new Big(quantity).mul(unitPrice).toNumber(); + const valueInBaseCurrency = this.exchangeRateDataService.toCurrencyAtDate( + value, + currency ?? assetProfile.currency, + userCurrency, + date + ); + activities.push({ ...order, error, value, + valueInBaseCurrency: await valueInBaseCurrency, // @ts-ignore SymbolProfile: assetProfile }); diff --git a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts index c83e35503..72136dc04 100644 --- a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts @@ -207,14 +207,16 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface { if (['ETF', 'MUTUALFUND'].includes(assetSubClass)) { response.holdings = - assetProfile.topHoldings?.holdings?.map( - ({ holdingName, holdingPercent }) => { + assetProfile.topHoldings?.holdings + ?.filter(({ holdingName }) => { + return !holdingName?.includes('ETF'); + }) + ?.map(({ holdingName, holdingPercent }) => { return { name: this.formatName({ longName: holdingName }), weight: holdingPercent }; - } - ) ?? []; + }) ?? []; response.sectors = ( assetProfile.topHoldings?.sectorWeightings ?? [] diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 7392f0914..51e65e631 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -105,7 +105,10 @@ export class ManualService implements DataProviderInterface { return {}; } - const value = await this.scrape(symbolProfile.scraperConfiguration); + const value = await this.scrape({ + symbol, + scraperConfiguration: symbolProfile.scraperConfiguration + }); return { [symbol]: { @@ -170,7 +173,10 @@ export class ManualService implements DataProviderInterface { symbolProfilesWithScraperConfigurationAndInstantMode.map( async ({ scraperConfiguration, symbol }) => { try { - const marketPrice = await this.scrape(scraperConfiguration); + const marketPrice = await this.scrape({ + scraperConfiguration, + symbol + }); return { marketPrice, symbol }; } catch (error) { Logger.error( @@ -267,13 +273,23 @@ export class ManualService implements DataProviderInterface { }; } - public async test(scraperConfiguration: ScraperConfiguration) { - return this.scrape(scraperConfiguration); + public async test({ + scraperConfiguration, + symbol + }: { + scraperConfiguration: ScraperConfiguration; + symbol: string; + }) { + return this.scrape({ scraperConfiguration, symbol }); } - private async scrape( - scraperConfiguration: ScraperConfiguration - ): Promise { + private async scrape({ + scraperConfiguration, + symbol + }: { + scraperConfiguration: ScraperConfiguration; + symbol: string; + }): Promise { let locale = scraperConfiguration.locale; const response = await fetch(scraperConfiguration.url, { @@ -283,6 +299,12 @@ export class ManualService implements DataProviderInterface { ) }); + if (!response.ok) { + throw new Error( + `Failed to scrape the market price for ${symbol} (${this.getName()}): ${response.status} ${response.statusText} at ${scraperConfiguration.url}` + ); + } + let value: string; if (response.headers.get('content-type')?.includes('application/json')) { diff --git a/package-lock.json b/package-lock.json index 1731f1e7a..046ebf083 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,7 +51,7 @@ "chartjs-chart-treemap": "3.1.0", "chartjs-plugin-annotation": "3.1.0", "chartjs-plugin-datalabels": "2.2.0", - "cheerio": "1.0.0", + "cheerio": "1.2.0", "class-transformer": "0.5.1", "class-validator": "0.14.3", "color": "5.0.3", @@ -16101,25 +16101,25 @@ } }, "node_modules/cheerio": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", - "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", "license": "MIT", "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", - "domutils": "^3.1.0", - "encoding-sniffer": "^0.2.0", - "htmlparser2": "^9.1.0", - "parse5": "^7.1.2", - "parse5-htmlparser2-tree-adapter": "^7.0.0", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", - "undici": "^6.19.5", + "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" }, "engines": { - "node": ">=18.17" + "node": ">=20.18.1" }, "funding": { "url": "https://github.com/cheeriojs/cheerio?sponsor=1" @@ -16142,25 +16142,6 @@ "url": "https://github.com/sponsors/fb55" } }, - "node_modules/cheerio/node_modules/htmlparser2": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", - "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.1.0", - "entities": "^4.5.0" - } - }, "node_modules/cheerio/node_modules/parse5": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", @@ -21656,9 +21637,9 @@ } }, "node_modules/htmlparser2": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", - "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", "funding": [ "https://github.com/fb55/htmlparser2?sponsor=1", { @@ -21670,14 +21651,14 @@ "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", - "domutils": "^3.2.1", - "entities": "^6.0.0" + "domutils": "^3.2.2", + "entities": "^7.0.1" } }, "node_modules/htmlparser2/node_modules/entities": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", - "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", "license": "BSD-2-Clause", "engines": { "node": ">=0.12" @@ -33626,12 +33607,12 @@ "license": "MIT" }, "node_modules/undici": { - "version": "6.21.3", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.3.tgz", - "integrity": "sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==", + "version": "7.22.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.22.0.tgz", + "integrity": "sha512-RqslV2Us5BrllB+JeiZnK4peryVTndy9Dnqq62S3yYRRTj0tFQCwEniUy2167skdGOy3vqRzEvl1Dm4sV2ReDg==", "license": "MIT", "engines": { - "node": ">=18.17" + "node": ">=20.18.1" } }, "node_modules/undici-types": { diff --git a/package.json b/package.json index 4bfad50ff..c70b2c14b 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,7 @@ "chartjs-chart-treemap": "3.1.0", "chartjs-plugin-annotation": "3.1.0", "chartjs-plugin-datalabels": "2.2.0", - "cheerio": "1.0.0", + "cheerio": "1.2.0", "class-transformer": "0.5.1", "class-validator": "0.14.3", "color": "5.0.3",