Browse Source

fix(manual-scraper): respect HTTP_PROXY env vars

Use undici ProxyAgent as the fetch dispatcher when HTTP_PROXY,
HTTPS_PROXY, or their lowercase variants are set. This ensures the
manual scraper routes requests through the configured proxy, matching
the behavior of other data providers.

Closes #6206

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
pull/6581/head
Matt Van Horn 2 weeks ago
parent
commit
be7c92e5e1
  1. 17
      apps/api/src/services/data-provider/manual/manual.service.ts

17
apps/api/src/services/data-provider/manual/manual.service.ts

@ -27,6 +27,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import * as cheerio from 'cheerio';
import { addDays, format, isBefore } from 'date-fns';
import { ProxyAgent } from 'undici';
@Injectable()
export class ManualService implements DataProviderInterface {
@ -292,12 +293,24 @@ export class ManualService implements DataProviderInterface {
}): Promise<number> {
let locale = scraperConfiguration.locale;
const response = await fetch(scraperConfiguration.url, {
const fetchOptions: RequestInit & { dispatcher?: ProxyAgent } = {
headers: scraperConfiguration.headers as HeadersInit,
signal: AbortSignal.timeout(
this.configurationService.get('REQUEST_TIMEOUT')
)
});
};
const proxyUrl =
process.env.HTTPS_PROXY ??
process.env.https_proxy ??
process.env.HTTP_PROXY ??
process.env.http_proxy;
if (proxyUrl) {
fetchOptions.dispatcher = new ProxyAgent(proxyUrl);
}
const response = await fetch(scraperConfiguration.url, fetchOptions);
if (!response.ok) {
throw new Error(

Loading…
Cancel
Save