From 697ef59e3b58bebc5c21a9e482e4f5643390f316 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 31 May 2026 07:17:55 +0200 Subject: [PATCH] Feature/add support for web fetch in fetch service (#6960) * Add support for web fetch * Update changelog --- CHANGELOG.md | 4 + apps/api/src/services/fetch/fetch.module.ts | 2 + apps/api/src/services/fetch/fetch.service.ts | 154 +++++++++++++++++- .../interfaces/web-fetch-route.interface.ts | 19 +++ libs/common/src/lib/config.ts | 1 + 5 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e0c416fb0..d094f394b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added support for routing selected requests through the _OpenRouter_ `web_fetch` tool in the `FetchService` + ### Changed - Extended the countries mapping in the data enhancer for asset profile data via _Trackinsight_ diff --git a/apps/api/src/services/fetch/fetch.module.ts b/apps/api/src/services/fetch/fetch.module.ts index f98f2f45c..16e6f5f5d 100644 --- a/apps/api/src/services/fetch/fetch.module.ts +++ b/apps/api/src/services/fetch/fetch.module.ts @@ -1,9 +1,11 @@ import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service'; +import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; import { Module } from '@nestjs/common'; @Module({ exports: [FetchService], + imports: [PropertyModule], providers: [FetchService] }) export class FetchModule {} diff --git a/apps/api/src/services/fetch/fetch.service.ts b/apps/api/src/services/fetch/fetch.service.ts index b3bd022d9..f32e56a1c 100644 --- a/apps/api/src/services/fetch/fetch.service.ts +++ b/apps/api/src/services/fetch/fetch.service.ts @@ -1,15 +1,35 @@ import { redactPaths } from '@ghostfolio/api/helper/object.helper'; +import { PropertyService } from '@ghostfolio/api/services/property/property.service'; +import { + PROPERTY_API_KEY_OPENROUTER, + PROPERTY_OPENROUTER_MODEL, + PROPERTY_WEB_FETCH_ROUTES +} from '@ghostfolio/common/config'; -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { createOpenRouter } from '@openrouter/ai-sdk-provider'; +import { generateText, jsonSchema, tool } from 'ai'; +import ms from 'ms'; + +import { WebFetchRoute } from './interfaces/web-fetch-route.interface'; @Injectable() -export class FetchService { +export class FetchService implements OnModuleInit { private static readonly REDACTED_QUERY_PARAM_NAMES = ['apikey', 'api_token']; + private static readonly WEB_FETCH_TIMEOUT = ms('30 seconds'); + + private webFetchRoutes: WebFetchRoute[] = []; + + public constructor(private readonly propertyService: PropertyService) {} - public async fetch( - input: RequestInfo | URL, - init?: RequestInit - ): Promise { + public async onModuleInit() { + this.webFetchRoutes = + (await this.propertyService.getByKey( + PROPERTY_WEB_FETCH_ROUTES + )) ?? []; + } + + public async fetch(input: RequestInfo | URL, init?: RequestInit) { const method = ( init?.method ?? (input instanceof Request ? input.method : undefined) ?? @@ -21,6 +41,21 @@ export class FetchService { Logger.debug(`${method} ${urlRedacted}`, 'FetchService'); + if (method === 'GET') { + const webFetchRoute = this.getMatchingWebFetchRoute(url); + + if (webFetchRoute) { + const response = await this.fetchViaWebFetchTool({ + url, + webFetchRoute + }); + + if (response) { + return response; + } + } + } + try { return await globalThis.fetch(input, init); } catch (error) { @@ -40,6 +75,113 @@ export class FetchService { } } + private async fetchViaWebFetchTool({ + url, + webFetchRoute + }: { + url: string; + webFetchRoute: WebFetchRoute; + }) { + const [openRouterApiKey, openRouterModel] = await Promise.all([ + this.propertyService.getByKey(PROPERTY_API_KEY_OPENROUTER), + this.propertyService.getByKey(PROPERTY_OPENROUTER_MODEL) + ]); + + if (!openRouterApiKey || !openRouterModel) { + return undefined; + } + + try { + const openRouterService = createOpenRouter({ apiKey: openRouterApiKey }); + + const { sources, text } = await generateText({ + model: openRouterService.chat(openRouterModel), + prompt: [ + 'You have access to a web_fetch tool. You MUST call it to retrieve the URL below, do not answer from prior knowledge.', + 'Return the fetched response body exactly as received: raw body only, no commentary, no Markdown, and no code fences.', + `URL: ${url}` + ].join('\n'), + timeout: FetchService.WEB_FETCH_TIMEOUT, + tools: { + // Provider-defined tool: lets OpenRouter perform the actual web + // request server-side via its `web_fetch` engine. `id` and `args` + // are the OpenRouter-specific identifiers; the input schema is left + // open as the arguments are supplied by the model. + web_fetch: tool({ + args: { engine: 'openrouter' }, + id: 'openrouter.web_fetch', + inputSchema: jsonSchema({ + additionalProperties: true, + type: 'object' + }), + type: 'provider' + }) + } + }); + + const candidates = [ + ...(sources ?? []).map((source) => { + return source.providerMetadata?.openrouter?.content; + }), + text + ]; + + for (const candidate of candidates) { + if (typeof candidate !== 'string') { + continue; + } + + const body = candidate.trim(); + + if (!body) { + continue; + } + + if (webFetchRoute.responseContentType?.includes('application/json')) { + try { + JSON.parse(body); + } catch { + continue; + } + } + + Logger.debug( + `Routed ${this.redactUrl(url)} via web fetch tool`, + 'FetchService' + ); + + return new Response(body, { + headers: webFetchRoute.responseContentType + ? { 'content-type': webFetchRoute.responseContentType } + : undefined + }); + } + + return undefined; + } catch (error) { + Logger.error( + `Web fetch tool failed for ${this.redactUrl(url)}: ${ + error instanceof Error ? error.message : String(error) + }`, + 'FetchService' + ); + + return undefined; + } + } + + private getMatchingWebFetchRoute(url: string) { + try { + const { hostname } = new URL(url); + + return this.webFetchRoutes.find(({ domain }) => { + return hostname === domain || hostname.endsWith(`.${domain}`); + }); + } catch { + return undefined; + } + } + private redactUrl(rawUrl: string): string { try { const url = new URL(rawUrl); diff --git a/apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts b/apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts new file mode 100644 index 000000000..efff09398 --- /dev/null +++ b/apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts @@ -0,0 +1,19 @@ +/** + * Routes outgoing GET requests for a given domain through the OpenRouter + * `web_fetch` tool instead of a direct network request. + * + * Configured via the `WEB_FETCH_ROUTES` property as a JSON array, e.g. + * + * [ + * { + * "domain": "example.com", + * "responseContentType": "application/json" + * } + * ] + * + * Matches the domain itself and its subdomains (e.g. `api.example.com`). + */ +export interface WebFetchRoute { + domain: string; + responseContentType?: string; +} diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 113dffe4a..28d902d71 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -256,6 +256,7 @@ export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS'; export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG'; export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE'; export const PROPERTY_UPTIME = 'UPTIME'; +export const PROPERTY_WEB_FETCH_ROUTES = 'WEB_FETCH_ROUTES'; export const QUEUE_JOB_STATUS_LIST = [ 'active',