Matt Van Horn
4 days ago
committed by
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with
15 additions and
2 deletions
-
apps/api/src/services/data-provider/manual/manual.service.ts
|
|
|
@ -27,6 +27,7 @@ import { Injectable, Logger } from '@nestjs/common'; |
|
|
|
import { DataSource, SymbolProfile } from '@prisma/client'; |
|
|
|
import * as cheerio from 'cheerio'; |
|
|
|
import { addDays, format, isBefore } from 'date-fns'; |
|
|
|
import { ProxyAgent } from 'undici'; |
|
|
|
|
|
|
|
@Injectable() |
|
|
|
export class ManualService implements DataProviderInterface { |
|
|
|
@ -292,12 +293,24 @@ export class ManualService implements DataProviderInterface { |
|
|
|
}): Promise<number> { |
|
|
|
let locale = scraperConfiguration.locale; |
|
|
|
|
|
|
|
const response = await fetch(scraperConfiguration.url, { |
|
|
|
const fetchOptions: RequestInit & { dispatcher?: ProxyAgent } = { |
|
|
|
headers: scraperConfiguration.headers as HeadersInit, |
|
|
|
signal: AbortSignal.timeout( |
|
|
|
this.configurationService.get('REQUEST_TIMEOUT') |
|
|
|
) |
|
|
|
}); |
|
|
|
}; |
|
|
|
|
|
|
|
const proxyUrl = |
|
|
|
process.env.HTTPS_PROXY ?? |
|
|
|
process.env.https_proxy ?? |
|
|
|
process.env.HTTP_PROXY ?? |
|
|
|
process.env.http_proxy; |
|
|
|
|
|
|
|
if (proxyUrl) { |
|
|
|
fetchOptions.dispatcher = new ProxyAgent(proxyUrl); |
|
|
|
} |
|
|
|
|
|
|
|
const response = await fetch(scraperConfiguration.url, fetchOptions); |
|
|
|
|
|
|
|
if (!response.ok) { |
|
|
|
throw new Error( |
|
|
|
|