Feature/add support for web fetch in fetch service (#6960)

* Add support for web fetch

* Update changelog

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Added support for routing selected requests through the _OpenRouter_ `web_fetch` tool in the `FetchService`
+
 ### Changed
 
 - Extended the countries mapping in the data enhancer for asset profile data via _Trackinsight_

apps/api/src/services/fetch/fetch.module.ts

@@ -1,9 +1,11 @@
 import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service';
+import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
 
 import { Module } from '@nestjs/common';
 
 @Module({
   exports: [FetchService],
+  imports: [PropertyModule],
   providers: [FetchService]
 })
 export class FetchModule {}

apps/api/src/services/fetch/fetch.service.ts

@@ -1,15 +1,35 @@
 import { redactPaths } from '@ghostfolio/api/helper/object.helper';
+import { PropertyService } from '@ghostfolio/api/services/property/property.service';
+import {
+  PROPERTY_API_KEY_OPENROUTER,
+  PROPERTY_OPENROUTER_MODEL,
+  PROPERTY_WEB_FETCH_ROUTES
+} from '@ghostfolio/common/config';
 
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { createOpenRouter } from '@openrouter/ai-sdk-provider';
+import { generateText, jsonSchema, tool } from 'ai';
+import ms from 'ms';
+
+import { WebFetchRoute } from './interfaces/web-fetch-route.interface';
 
 @Injectable()
-export class FetchService {
+export class FetchService implements OnModuleInit {
   private static readonly REDACTED_QUERY_PARAM_NAMES = ['apikey', 'api_token'];
+  private static readonly WEB_FETCH_TIMEOUT = ms('30 seconds');
+
+  private webFetchRoutes: WebFetchRoute[] = [];
+
+  public constructor(private readonly propertyService: PropertyService) {}
 
-  public async fetch(
-    input: RequestInfo | URL,
-    init?: RequestInit
-  ): Promise<Response> {
+  public async onModuleInit() {
+    this.webFetchRoutes =
+      (await this.propertyService.getByKey<WebFetchRoute[]>(
+        PROPERTY_WEB_FETCH_ROUTES
+      )) ?? [];
+  }
+
+  public async fetch(input: RequestInfo | URL, init?: RequestInit) {
     const method = (
       init?.method ??
       (input instanceof Request ? input.method : undefined) ??
@@ -21,6 +41,21 @@ export class FetchService {
 
     Logger.debug(`${method} ${urlRedacted}`, 'FetchService');
 
+    if (method === 'GET') {
+      const webFetchRoute = this.getMatchingWebFetchRoute(url);
+
+      if (webFetchRoute) {
+        const response = await this.fetchViaWebFetchTool({
+          url,
+          webFetchRoute
+        });
+
+        if (response) {
+          return response;
+        }
+      }
+    }
+
     try {
       return await globalThis.fetch(input, init);
     } catch (error) {
@@ -40,6 +75,113 @@ export class FetchService {
     }
   }
 
+  private async fetchViaWebFetchTool({
+    url,
+    webFetchRoute
+  }: {
+    url: string;
+    webFetchRoute: WebFetchRoute;
+  }) {
+    const [openRouterApiKey, openRouterModel] = await Promise.all([
+      this.propertyService.getByKey<string>(PROPERTY_API_KEY_OPENROUTER),
+      this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL)
+    ]);
+
+    if (!openRouterApiKey || !openRouterModel) {
+      return undefined;
+    }
+
+    try {
+      const openRouterService = createOpenRouter({ apiKey: openRouterApiKey });
+
+      const { sources, text } = await generateText({
+        model: openRouterService.chat(openRouterModel),
+        prompt: [
+          'You have access to a web_fetch tool. You MUST call it to retrieve the URL below, do not answer from prior knowledge.',
+          'Return the fetched response body exactly as received: raw body only, no commentary, no Markdown, and no code fences.',
+          `URL: ${url}`
+        ].join('\n'),
+        timeout: FetchService.WEB_FETCH_TIMEOUT,
+        tools: {
+          // Provider-defined tool: lets OpenRouter perform the actual web
+          // request server-side via its `web_fetch` engine. `id` and `args`
+          // are the OpenRouter-specific identifiers; the input schema is left
+          // open as the arguments are supplied by the model.
+          web_fetch: tool({
+            args: { engine: 'openrouter' },
+            id: 'openrouter.web_fetch',
+            inputSchema: jsonSchema({
+              additionalProperties: true,
+              type: 'object'
+            }),
+            type: 'provider'
+          })
+        }
+      });
+
+      const candidates = [
+        ...(sources ?? []).map((source) => {
+          return source.providerMetadata?.openrouter?.content;
+        }),
+        text
+      ];
+
+      for (const candidate of candidates) {
+        if (typeof candidate !== 'string') {
+          continue;
+        }
+
+        const body = candidate.trim();
+
+        if (!body) {
+          continue;
+        }
+
+        if (webFetchRoute.responseContentType?.includes('application/json')) {
+          try {
+            JSON.parse(body);
+          } catch {
+            continue;
+          }
+        }
+
+        Logger.debug(
+          `Routed ${this.redactUrl(url)} via web fetch tool`,
+          'FetchService'
+        );
+
+        return new Response(body, {
+          headers: webFetchRoute.responseContentType
+            ? { 'content-type': webFetchRoute.responseContentType }
+            : undefined
+        });
+      }
+
+      return undefined;
+    } catch (error) {
+      Logger.error(
+        `Web fetch tool failed for ${this.redactUrl(url)}: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+        'FetchService'
+      );
+
+      return undefined;
+    }
+  }
+
+  private getMatchingWebFetchRoute(url: string) {
+    try {
+      const { hostname } = new URL(url);
+
+      return this.webFetchRoutes.find(({ domain }) => {
+        return hostname === domain || hostname.endsWith(`.${domain}`);
+      });
+    } catch {
+      return undefined;
+    }
+  }
+
   private redactUrl(rawUrl: string): string {
     try {
       const url = new URL(rawUrl);

apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts

@@ -0,0 +1,19 @@
+/**
+ * Routes outgoing GET requests for a given domain through the OpenRouter
+ * `web_fetch` tool instead of a direct network request.
+ *
+ * Configured via the `WEB_FETCH_ROUTES` property as a JSON array, e.g.
+ *
+ * [
+ *   {
+ *     "domain": "example.com",
+ *     "responseContentType": "application/json"
+ *   }
+ * ]
+ *
+ * Matches the domain itself and its subdomains (e.g. `api.example.com`).
+ */
+export interface WebFetchRoute {
+  domain: string;
+  responseContentType?: string;
+}

libs/common/src/lib/config.ts

@@ -256,6 +256,7 @@ export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS';
 export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG';
 export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE';
 export const PROPERTY_UPTIME = 'UPTIME';
+export const PROPERTY_WEB_FETCH_ROUTES = 'WEB_FETCH_ROUTES';
 
 export const QUEUE_JOB_STATUS_LIST = [
   'active',