Browse Source

add postprocessor attribute to ScraperConfiguration

pull/3426/head
gccollect 1 year ago
parent
commit
fd6e5f60a9
  1. 13
      apps/api/src/services/data-provider/manual/manual.service.ts
  2. 1
      apps/api/src/services/symbol-profile/symbol-profile.service.ts
  3. 1
      libs/common/src/lib/interfaces/scraper-configuration.interface.ts

13
apps/api/src/services/data-provider/manual/manual.service.ts

@ -256,13 +256,15 @@ export class ManualService implements DataProviderInterface {
signal: abortController.signal signal: abortController.signal
}); });
let factor = isNaN(+scraperConfiguration.postprocessor)
? 1.0
: +scraperConfiguration.postprocessor;
if (headers['content-type'] === 'application/json') { if (headers['content-type'] === 'application/json') {
const data = JSON.parse(body); const data = JSON.parse(body);
const value = String( const value = String(
jsonpath.query(data, scraperConfiguration.selector)[0] jsonpath.query(data, scraperConfiguration.selector)[0]
); );
return factor * extractNumberFromString({ locale, value });
return extractNumberFromString({ locale, value });
} else { } else {
const $ = cheerio.load(body); const $ = cheerio.load(body);
@ -272,10 +274,13 @@ export class ManualService implements DataProviderInterface {
} catch {} } catch {}
} }
return extractNumberFromString({ return (
factor *
extractNumberFromString({
locale, locale,
value: $(scraperConfiguration.selector).first().text() value: $(scraperConfiguration.selector).first().text()
}); })
);
} }
} catch (error) { } catch (error) {
throw error; throw error;

1
apps/api/src/services/symbol-profile/symbol-profile.service.ts

@ -211,6 +211,7 @@ export class SymbolProfileService {
headers: headers:
scraperConfiguration.headers as ScraperConfiguration['headers'], scraperConfiguration.headers as ScraperConfiguration['headers'],
locale: scraperConfiguration.locale as string, locale: scraperConfiguration.locale as string,
postprocessor: scraperConfiguration.postprocessor as string,
selector: scraperConfiguration.selector as string, selector: scraperConfiguration.selector as string,
url: scraperConfiguration.url as string url: scraperConfiguration.url as string
}; };

1
libs/common/src/lib/interfaces/scraper-configuration.interface.ts

@ -2,6 +2,7 @@ export interface ScraperConfiguration {
defaultMarketPrice?: number; defaultMarketPrice?: number;
headers?: { [key: string]: string }; headers?: { [key: string]: string };
locale?: string; locale?: string;
postprocessor?: string;
selector: string; selector: string;
url: string; url: string;
} }

Loading…
Cancel
Save