Improve web_fetch tool

apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts

@@ -64,10 +64,10 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
       symbol
     });
 
-    if (!trackinsightSymbol) {
+    if (!trackinsightSymbol && symbol.includes('.')) {
       trackinsightSymbol = await this.searchTrackinsightSymbol({
         requestTimeout,
-        symbol: symbol.split('.')?.[0]
+        symbol: symbol.split('.')[0]
       });
     }
 
@@ -213,6 +213,12 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
           return jsonRes['results']['docs'][0]['ticker'];
         }
 
+        this.logger.debug(
+          `Could not match a Trackinsight symbol for "${symbol}": ${JSON.stringify(
+            jsonRes
+          )}`
+        );
+
         return undefined;
       })
       .catch(({ message }) => {

apps/api/src/services/fetch/fetch.service.ts

@@ -1,7 +1,9 @@
 import { redactPaths } from '@ghostfolio/api/helper/object.helper';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import {
+  DEFAULT_OPENROUTER_ENGINE_WEB_FETCH,
   PROPERTY_API_KEY_OPENROUTER,
+  PROPERTY_OPENROUTER_ENGINE_WEB_FETCH,
   PROPERTY_OPENROUTER_MODEL,
   PROPERTY_OPENROUTER_MODEL_WEB_FETCH,
   PROPERTY_PROXY_ROUTES,
@@ -20,6 +22,7 @@ import { WebFetchRoute } from './interfaces/web-fetch-route.interface';
 export class FetchService implements OnModuleInit {
   private readonly logger = new Logger(FetchService.name);
 
+  private static readonly BODY_PREVIEW_LENGTH = 500;
   private static readonly REDACTED_QUERY_PARAM_NAMES = ['apikey', 'api_token'];
   private static readonly WEB_FETCH_TIMEOUT = ms('30 seconds');
 
@@ -91,15 +94,22 @@ export class FetchService implements OnModuleInit {
     url: string;
     webFetchRoute: WebFetchRoute;
   }) {
-    const [openRouterApiKey, openRouterModel, openRouterModelWebFetch] =
-      await Promise.all([
-        this.propertyService.getByKey<string>(PROPERTY_API_KEY_OPENROUTER),
-        this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL),
-        this.propertyService.getByKey<string>(
-          PROPERTY_OPENROUTER_MODEL_WEB_FETCH
-        )
-      ]);
-
+    const [
+      openRouterApiKey,
+      openRouterEngineWebFetch,
+      openRouterModel,
+      openRouterModelWebFetch
+    ] = await Promise.all([
+      this.propertyService.getByKey<string>(PROPERTY_API_KEY_OPENROUTER),
+      this.propertyService.getByKey<string>(
+        PROPERTY_OPENROUTER_ENGINE_WEB_FETCH
+      ),
+      this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL),
+      this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL_WEB_FETCH)
+    ]);
+
+    const engine =
+      openRouterEngineWebFetch || DEFAULT_OPENROUTER_ENGINE_WEB_FETCH;
     const model = openRouterModelWebFetch || openRouterModel;
 
     if (!model || !openRouterApiKey) {
@@ -119,11 +129,15 @@ export class FetchService implements OnModuleInit {
         timeout: FetchService.WEB_FETCH_TIMEOUT,
         tools: {
           // Provider-executed tool: lets OpenRouter perform the actual web
-          // request server-side via its `web_fetch` engine. `id` and `args`
-          // are the OpenRouter-specific identifiers. The input schema is left
-          // open as the arguments are supplied by the model.
+          // request server-side via its `web_fetch` engine. `args` and `id`
+          // are the OpenRouter-specific identifiers. The engine must stay
+          // nested in `parameters`, because the provider spreads `args` on the
+          // top level of the tool object, where OpenRouter ignores it. Nested
+          // keys need snake case, as only the top level gets converted. The
+          // input schema is left open as the arguments are supplied by the
+          // model.
           web_fetch: tool({
-            args: { engine: 'openrouter' },
+            args: { parameters: { engine } },
             id: 'openrouter.web_fetch',
             inputSchema: jsonSchema({
               additionalProperties: true,
@@ -142,6 +156,8 @@ export class FetchService implements OnModuleInit {
         text
       ];
 
+      let rejectedBody: string;
+
       for (const candidate of candidates) {
         if (typeof candidate !== 'string') {
           continue;
@@ -157,6 +173,10 @@ export class FetchService implements OnModuleInit {
           try {
             JSON.parse(body);
           } catch {
+            if (!rejectedBody) {
+              rejectedBody = body;
+            }
+
             continue;
           }
         }
@@ -170,6 +190,16 @@ export class FetchService implements OnModuleInit {
         });
       }
 
+      this.logger.warn(
+        `Web fetch tool returned no usable body for ${this.redactUrl(url)}`
+      );
+
+      this.logger.debug(
+        `Web fetch tool response for ${this.redactUrl(
+          url
+        )}: ${this.getBodyPreview(rejectedBody)}`
+      );
+
       return undefined;
     } catch (error) {
       this.logger.error(
@@ -227,6 +257,22 @@ export class FetchService implements OnModuleInit {
       : requestUrl.toString();
   }
 
+  /**
+   * Makes a short single-line preview of a response body for the log. Gives
+   * back a placeholder if the body is empty.
+   */
+  private getBodyPreview(body: string): string {
+    if (!body) {
+      return '(empty)';
+    }
+
+    const singleLineBody = body.replace(/\s+/g, ' ').trim();
+
+    return singleLineBody.length > FetchService.BODY_PREVIEW_LENGTH
+      ? `${singleLineBody.slice(0, FetchService.BODY_PREVIEW_LENGTH)}…`
+      : singleLineBody;
+  }
+
   private getMatchingWebFetchRoute(url: string) {
     try {
       const { hostname } = new URL(url);

libs/common/src/lib/config.ts

@@ -99,6 +99,7 @@ export const DEFAULT_DATE_RANGE: DateRange = 'max';
 export const DEFAULT_HOST = '0.0.0.0';
 export const DEFAULT_LANGUAGE_CODE = 'en';
 export const DEFAULT_LOCALE = 'en-US';
+export const DEFAULT_OPENROUTER_ENGINE_WEB_FETCH = 'openrouter';
 export const DEFAULT_PAGE_SIZE = 50;
 export const DEFAULT_PORT = 3333;
 export const DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY = 1;
@@ -295,6 +296,8 @@ export const PROPERTY_IS_DATA_GATHERING_ENABLED = 'IS_DATA_GATHERING_ENABLED';
 export const PROPERTY_IS_READ_ONLY_MODE = 'IS_READ_ONLY_MODE';
 export const PROPERTY_IS_USER_SIGNUP_ENABLED = 'IS_USER_SIGNUP_ENABLED';
 export const PROPERTY_MAX_DAILY_REQUESTS = 'MAX_DAILY_REQUESTS';
+export const PROPERTY_OPENROUTER_ENGINE_WEB_FETCH =
+  'OPENROUTER_ENGINE_WEB_FETCH';
 export const PROPERTY_OPENROUTER_MODEL = 'OPENROUTER_MODEL';
 export const PROPERTY_OPENROUTER_MODEL_WEB_FETCH = 'OPENROUTER_MODEL_WEB_FETCH';
 export const PROPERTY_PROXY_ROUTES = 'PROXY_ROUTES';