Browse Source

Merge be7c92e5e1 into c0ae6f848a

pull/6581/merge
Matt Van Horn 4 days ago
committed by GitHub
parent
commit
7514e81609
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 17
      apps/api/src/services/data-provider/manual/manual.service.ts

17
apps/api/src/services/data-provider/manual/manual.service.ts

@ -27,6 +27,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import * as cheerio from 'cheerio';
import { addDays, format, isBefore } from 'date-fns';
import { ProxyAgent } from 'undici';
@Injectable()
export class ManualService implements DataProviderInterface {
@ -292,12 +293,24 @@ export class ManualService implements DataProviderInterface {
}): Promise<number> {
let locale = scraperConfiguration.locale;
const response = await fetch(scraperConfiguration.url, {
const fetchOptions: RequestInit & { dispatcher?: ProxyAgent } = {
headers: scraperConfiguration.headers as HeadersInit,
signal: AbortSignal.timeout(
this.configurationService.get('REQUEST_TIMEOUT')
)
});
};
const proxyUrl =
process.env.HTTPS_PROXY ??
process.env.https_proxy ??
process.env.HTTP_PROXY ??
process.env.http_proxy;
if (proxyUrl) {
fetchOptions.dispatcher = new ProxyAgent(proxyUrl);
}
const response = await fetch(scraperConfiguration.url, fetchOptions);
if (!response.ok) {
throw new Error(

Loading…
Cancel
Save