Browse Source
Feature/improve handling of scraper configuration (#757)
* Improve handling of missing scraper configuration
* Update changelog
pull/758/head
Thomas Kaul
3 years ago
committed by
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with
13 additions and
5 deletions
-
CHANGELOG.md
-
apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
|
@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. |
|
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), |
|
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), |
|
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). |
|
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). |
|
|
|
|
|
|
|
|
|
|
|
## Unreleased |
|
|
|
|
|
|
|
|
|
|
|
### Changed |
|
|
|
|
|
|
|
|
|
|
|
- Improved the error handling in the scraper configuration |
|
|
|
|
|
|
|
|
## 1.126.0 - 14.03.2022 |
|
|
## 1.126.0 - 14.03.2022 |
|
|
|
|
|
|
|
|
### Added |
|
|
### Added |
|
|
|
@ -50,16 +50,18 @@ export class GhostfolioScraperApiService implements DataProviderInterface { |
|
|
const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles( |
|
|
const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles( |
|
|
[symbol] |
|
|
[symbol] |
|
|
); |
|
|
); |
|
|
const scraperConfiguration = symbolProfile?.scraperConfiguration; |
|
|
const { selector, url } = symbolProfile.scraperConfiguration; |
|
|
|
|
|
|
|
|
const get = bent(scraperConfiguration?.url, 'GET', 'string', 200, {}); |
|
|
if (selector === undefined || url === undefined) { |
|
|
|
|
|
return {}; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
const get = bent(url, 'GET', 'string', 200, {}); |
|
|
|
|
|
|
|
|
const html = await get(); |
|
|
const html = await get(); |
|
|
const $ = cheerio.load(html); |
|
|
const $ = cheerio.load(html); |
|
|
|
|
|
|
|
|
const value = this.extractNumberFromString( |
|
|
const value = this.extractNumberFromString($(selector).text()); |
|
|
$(scraperConfiguration?.selector).text() |
|
|
|
|
|
); |
|
|
|
|
|
|
|
|
|
|
|
return { |
|
|
return { |
|
|
[symbol]: { |
|
|
[symbol]: { |
|
|