Browse Source

add postprocessor attribute to ScraperConfiguration

pull/3426/head
gccollect 1 year ago
parent
commit
fd6e5f60a9
  1. 17
      apps/api/src/services/data-provider/manual/manual.service.ts
  2. 1
      apps/api/src/services/symbol-profile/symbol-profile.service.ts
  3. 1
      libs/common/src/lib/interfaces/scraper-configuration.interface.ts

17
apps/api/src/services/data-provider/manual/manual.service.ts

@ -256,13 +256,15 @@ export class ManualService implements DataProviderInterface {
signal: abortController.signal
});
let factor = isNaN(+scraperConfiguration.postprocessor)
? 1.0
: +scraperConfiguration.postprocessor;
if (headers['content-type'] === 'application/json') {
const data = JSON.parse(body);
const value = String(
jsonpath.query(data, scraperConfiguration.selector)[0]
);
return extractNumberFromString({ locale, value });
return factor * extractNumberFromString({ locale, value });
} else {
const $ = cheerio.load(body);
@ -272,10 +274,13 @@ export class ManualService implements DataProviderInterface {
} catch {}
}
return extractNumberFromString({
locale,
value: $(scraperConfiguration.selector).first().text()
});
return (
factor *
extractNumberFromString({
locale,
value: $(scraperConfiguration.selector).first().text()
})
);
}
} catch (error) {
throw error;

1
apps/api/src/services/symbol-profile/symbol-profile.service.ts

@ -211,6 +211,7 @@ export class SymbolProfileService {
headers:
scraperConfiguration.headers as ScraperConfiguration['headers'],
locale: scraperConfiguration.locale as string,
postprocessor: scraperConfiguration.postprocessor as string,
selector: scraperConfiguration.selector as string,
url: scraperConfiguration.url as string
};

1
libs/common/src/lib/interfaces/scraper-configuration.interface.ts

@ -2,6 +2,7 @@ export interface ScraperConfiguration {
defaultMarketPrice?: number;
headers?: { [key: string]: string };
locale?: string;
postprocessor?: string;
selector: string;
url: string;
}

Loading…
Cancel
Save