Browse Source

Expose locale in scraper configuration

pull/2843/head
Thomas Kaul 2 years ago
parent
commit
a968755a0e
  1. 12
      apps/api/src/services/data-provider/manual/manual.service.ts
  2. 1
      apps/api/src/services/symbol-profile/symbol-profile.service.ts
  3. 1
      libs/common/src/lib/interfaces/scraper-configuration.interface.ts

12
apps/api/src/services/data-provider/manual/manual.service.ts

@ -236,6 +236,7 @@ export class ManualService implements DataProviderInterface {
abortController.abort(); abortController.abort();
}, this.configurationService.get('REQUEST_TIMEOUT')); }, this.configurationService.get('REQUEST_TIMEOUT'));
let locale = scraperConfiguration.locale;
const { body, headers } = await got(scraperConfiguration.url, { const { body, headers } = await got(scraperConfiguration.url, {
headers: scraperConfiguration.headers as Headers, headers: scraperConfiguration.headers as Headers,
// @ts-ignore // @ts-ignore
@ -248,14 +249,15 @@ export class ManualService implements DataProviderInterface {
jsonpath.query(data, scraperConfiguration.selector)[0] jsonpath.query(data, scraperConfiguration.selector)[0]
); );
return extractNumberFromString({ value }); return extractNumberFromString({ locale, value });
} else { } else {
const $ = cheerio.load(body); const $ = cheerio.load(body);
let locale: string;
try { if (!locale) {
locale = $('html').attr('lang'); try {
} catch {} locale = $('html').attr('lang');
} catch {}
}
return extractNumberFromString({ return extractNumberFromString({
locale, locale,

1
apps/api/src/services/symbol-profile/symbol-profile.service.ts

@ -202,6 +202,7 @@ export class SymbolProfileService {
defaultMarketPrice: scraperConfiguration.defaultMarketPrice as number, defaultMarketPrice: scraperConfiguration.defaultMarketPrice as number,
headers: headers:
scraperConfiguration.headers as ScraperConfiguration['headers'], scraperConfiguration.headers as ScraperConfiguration['headers'],
locale: scraperConfiguration.locale as string,
selector: scraperConfiguration.selector as string, selector: scraperConfiguration.selector as string,
url: scraperConfiguration.url as string url: scraperConfiguration.url as string
}; };

1
libs/common/src/lib/interfaces/scraper-configuration.interface.ts

@ -1,6 +1,7 @@
export interface ScraperConfiguration { export interface ScraperConfiguration {
defaultMarketPrice?: number; defaultMarketPrice?: number;
headers?: { [key: string]: string }; headers?: { [key: string]: string };
locale?: string;
selector: string; selector: string;
url: string; url: string;
} }

Loading…
Cancel
Save